summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAlex Gorrod <alexander.gorrod@mongodb.com>2016-10-14 18:08:49 +1100
committerGitHub <noreply@github.com>2016-10-14 18:08:49 +1100
commitf8bcc58d267e82f085fae3b64f498ee9fecbe424 (patch)
tree8f7bcd5ab1e3b24a01162b093e3ca33e6eae1a62
parente0ce5ab8afb47ca010323d697213d786eae3a08f (diff)
downloadmongo-f8bcc58d267e82f085fae3b64f498ee9fecbe424.tar.gz
WT-1592 Add per-dhandle current cache usage statistics (#3062)
Exposed via a new 'cache_walk' statistics configuration option.
-rw-r--r--dist/api_data.py6
-rw-r--r--dist/filelist1
-rw-r--r--dist/flags.py10
-rw-r--r--dist/s_string.ok1
-rw-r--r--dist/stat.py7
-rw-r--r--dist/stat_data.py55
-rw-r--r--src/btree/bt_stat.c29
-rw-r--r--src/cache/cache_las.c2
-rw-r--r--src/config/config_def.c18
-rw-r--r--src/conn/conn_api.c48
-rw-r--r--src/conn/conn_stat.c10
-rw-r--r--src/cursor/cur_stat.c77
-rw-r--r--src/evict/evict_lru.c2
-rw-r--r--src/evict/evict_stat.c150
-rw-r--r--src/include/btmem.h2
-rw-r--r--src/include/cache.h1
-rw-r--r--src/include/connection.h8
-rw-r--r--src/include/cursor.h2
-rw-r--r--src/include/extern.h1
-rw-r--r--src/include/flags.h8
-rw-r--r--src/include/stat.h18
-rw-r--r--src/include/wiredtiger.in217
-rw-r--r--src/lsm/lsm_stat.c22
-rw-r--r--src/schema/schema_stat.c2
-rw-r--r--src/support/stat.c90
-rw-r--r--test/suite/test_stat02.py70
-rw-r--r--tools/wtstats/stat_data.py39
-rwxr-xr-xtools/wtstats/wtstats.py3
28 files changed, 752 insertions, 147 deletions
diff --git a/dist/api_data.py b/dist/api_data.py
index d02d7e4b985..df3577f56b8 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -502,7 +502,8 @@ connection_runtime_config = [
is used to gather statistics, as well as each time statistics
are logged using the \c statistics_log configuration. See
@ref statistics for more information''',
- type='list', choices=['all', 'fast', 'none', 'clear']),
+ type='list',
+ choices=['all', 'cache_walk', 'fast', 'none', 'clear', 'tree_walk']),
Config('verbose', '', r'''
enable messages for various events. Only available if WiredTiger
is configured with --enable-verbose. Options are given as a
@@ -976,7 +977,8 @@ methods = {
gathering them, where appropriate (for example, a cache size statistic
is not cleared, while the count of cursor insert operations will be
cleared). See @ref statistics for more information''',
- type='list', choices=['all', 'fast', 'clear', 'size']),
+ type='list',
+ choices=['all', 'cache_walk', 'fast', 'clear', 'size', 'tree_walk']),
Config('target', '', r'''
if non-empty, backup the list of objects; valid only for a
backup data source''',
diff --git a/dist/filelist b/dist/filelist
index 32e4231c5f2..fe9a17b7799 100644
--- a/dist/filelist
+++ b/dist/filelist
@@ -90,6 +90,7 @@ src/cursor/cur_table.c
src/evict/evict_file.c
src/evict/evict_lru.c
src/evict/evict_page.c
+src/evict/evict_stat.c
src/log/log.c
src/log/log_auto.c
src/log/log_slot.c
diff --git a/dist/flags.py b/dist/flags.py
index 93b6e0cbbf4..e200f95fba6 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -133,6 +133,16 @@ flags = {
'SESSION_QUIET_CORRUPT_FILE',
'SESSION_SERVER_ASYNC',
],
+ 'stat' : [
+ 'STAT_CLEAR',
+ 'STAT_JSON',
+ 'STAT_ON_CLOSE',
+ 'STAT_TYPE_ALL',
+ 'STAT_TYPE_CACHE_WALK',
+ 'STAT_TYPE_FAST',
+ 'STAT_TYPE_SIZE',
+ 'STAT_TYPE_TREE_WALK',
+ ],
}
flag_cnt = {} # Dictionary [flag] : [reference count]
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 6e629c8cd89..e8723de9810 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -993,6 +993,7 @@ qdown
qrrSS
qsort
quartile
+queueable
qup
rN
rS
diff --git a/dist/stat.py b/dist/stat.py
index c3c85bbe9b4..e42585c1b8c 100644
--- a/dist/stat.py
+++ b/dist/stat.py
@@ -42,8 +42,11 @@ compare_srcfile(tmp_file, '../src/include/stat.h')
def print_defines_one(capname, base, stats):
for v, l in enumerate(stats, base):
desc = l.desc
- if 'all_only' in l.flags:
- desc += ', only reported if statistics=all is set'
+ if 'cache_walk' in l.flags:
+ desc += \
+ ', only reported if cache_walk or all statistics are enabled'
+ if 'tree_walk' in l.flags:
+ desc += ', only reported if tree_walk or all statistics are enabled'
if len(textwrap.wrap(desc, 70)) > 1:
f.write('/*!\n')
f.write(' * %s\n' % '\n * '.join(textwrap.wrap(desc, 70)))
diff --git a/dist/stat_data.py b/dist/stat_data.py
index 5b4d3fa7709..ef70ca89eb0 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -9,7 +9,8 @@
#
# Data-source statistics are normally aggregated across the set of underlying
# objects. Additional optional configuration flags are available:
-# all_only Only gets reported when statistics=all set
+# cache_walk Only reported when statistics=cache_walk is set
+# tree_walk Only reported when statistics=tree_walk is set
# max_aggregate Take the maximum value when aggregating statistics
# no_clear Value not cleared when statistics cleared
# no_scale Don't scale value per second in the logging tool script
@@ -46,6 +47,11 @@ class CacheStat(Stat):
prefix = 'cache'
def __init__(self, name, desc, flags=''):
Stat.__init__(self, name, CacheStat.prefix, desc, flags)
+class CacheWalkStat(Stat):
+ prefix = 'cache_walk'
+ def __init__(self, name, desc, flags=''):
+ flags += ',cache_walk'
+ Stat.__init__(self, name, CacheWalkStat.prefix, desc, flags)
class CompressStat(Stat):
prefix = 'compression'
def __init__(self, name, desc, flags=''):
@@ -109,11 +115,16 @@ groups['cursor'] = [CursorStat.prefix, SessionStat.prefix]
groups['evict'] = [
BlockStat.prefix,
CacheStat.prefix,
+ CacheWalkStat.prefix,
ConnStat.prefix,
ThreadStat.prefix
]
groups['lsm'] = [LSMStat.prefix, TxnStat.prefix]
-groups['memory'] = [CacheStat.prefix, ConnStat.prefix, RecStat.prefix]
+groups['memory'] = [
+ CacheStat.prefix,
+ CacheWalkStat.prefix,
+ ConnStat.prefix,
+ RecStat.prefix]
groups['system'] = [
ConnStat.prefix,
DhandleStat.prefix,
@@ -438,13 +449,13 @@ dsrc_stats = [
# Btree statistics
##########################################
BtreeStat('btree_checkpoint_generation', 'btree checkpoint generation', 'no_clear,no_scale'),
- BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,all_only'),
- BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,all_only'),
- BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,all_only'),
- BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,all_only'),
- BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,all_only'),
+ BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,tree_walk'),
+ BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,tree_walk'),
+ BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,tree_walk'),
+ BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,tree_walk'),
+ BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,tree_walk'),
BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'),
- BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,all_only'),
+ BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,tree_walk'),
BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'),
BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'),
BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'),
@@ -452,9 +463,9 @@ dsrc_stats = [
BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'),
BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'),
BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'),
- BtreeStat('btree_overflow', 'overflow pages', 'no_scale,all_only'),
- BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,all_only'),
- BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,all_only'),
+ BtreeStat('btree_overflow', 'overflow pages', 'no_scale,tree_walk'),
+ BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,tree_walk'),
+ BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,tree_walk'),
##########################################
# Cache and eviction statistics
@@ -483,6 +494,28 @@ dsrc_stats = [
CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'),
##########################################
+ # Cache content statistics
+ ##########################################
+ CacheWalkStat('cache_state_avg_written_size', 'Average on-disk page image size seen', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_gen_avg_gap', 'Average difference between current eviction generation when the page was last considered', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_gen_current', 'Current eviction generation', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_gen_max_gap', 'Maximum difference between current eviction generation when the page was last considered', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_max_pagesize', 'Maximum page size seen', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_memory', 'Pages created in memory and never written', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_min_written_size', 'Minimum on-disk page image size seen', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_not_queueable', 'Pages that could not be queued for eviction', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_pages', 'Total number of pages currently in cache', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_pages_clean', 'Clean pages currently in cache', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_pages_dirty', 'Dirty pages currently in cache', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_pages_internal', 'Internal pages currently in cache', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_pages_leaf', 'Leaf pages currently in cache', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_queued', 'Pages currently queued for eviction', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_refs_skipped', 'Refs skipped during cache traversal', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_root_entries', 'Entries in the root page', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_root_size', 'Size of the root page', 'no_clear,no_scale'),
+ CacheWalkStat('cache_state_smaller_alloc_size', 'On-disk page image sizes smaller than a single allocation unit', 'no_clear,no_scale'),
+
+ ##########################################
# Compression statistics
##########################################
CompressStat('compress_raw_fail', 'raw compression call failed, no additional data available'),
diff --git a/src/btree/bt_stat.c b/src/btree/bt_stat.c
index d3ddf33446e..ad3d94a856f 100644
--- a/src/btree/bt_stat.c
+++ b/src/btree/bt_stat.c
@@ -8,6 +8,7 @@
#include "wt_internal.h"
+static int __stat_tree_walk(WT_SESSION_IMPL *);
static int __stat_page(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **);
static void __stat_page_col_var(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **);
static void __stat_page_row_int(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **);
@@ -23,9 +24,7 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst)
{
WT_BM *bm;
WT_BTREE *btree;
- WT_DECL_RET;
WT_DSRC_STATS **stats;
- WT_REF *next_walk;
btree = S2BT(session);
bm = btree->bm;
@@ -44,9 +43,29 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst)
WT_STAT_SET(session, stats, cache_bytes_inuse,
__wt_btree_bytes_inuse(session));
- /* Everything else is really, really expensive. */
- if (!F_ISSET(cst, WT_CONN_STAT_ALL))
- return (0);
+ if (F_ISSET(cst, WT_STAT_TYPE_CACHE_WALK))
+ WT_RET(__wt_curstat_cache_walk(session));
+
+ if (F_ISSET(cst, WT_STAT_TYPE_TREE_WALK))
+ WT_RET(__stat_tree_walk(session));
+
+ return (0);
+}
+
+/*
+ * __stat_tree_walk --
+ * Gather btree statistics that require traversing the tree.
+ */
+static int
+__stat_tree_walk(WT_SESSION_IMPL *session)
+{
+ WT_BTREE *btree;
+ WT_DECL_RET;
+ WT_DSRC_STATS **stats;
+ WT_REF *next_walk;
+
+ btree = S2BT(session);
+ stats = btree->dhandle->stats;
/*
* Clear the statistics we're about to count.
diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c
index 4c338bc6ad9..41f50957809 100644
--- a/src/cache/cache_las.c
+++ b/src/cache/cache_las.c
@@ -49,7 +49,7 @@ __wt_las_stats_update(WT_SESSION_IMPL *session)
* don't have to worry about users seeing inconsistent data source
* information.
*/
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) {
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_CLEAR)) {
WT_STAT_SET(session, dstats, cursor_insert, 0);
WT_STAT_SET(session, dstats, cursor_remove, 0);
}
diff --git a/src/config/config_def.c b/src/config/config_def.c
index 7bce4bc9cef..018cc7a8ac4 100644
--- a/src/config/config_def.c
+++ b/src/config/config_def.c
@@ -138,7 +138,8 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = {
NULL, NULL,
confchk_wiredtiger_open_shared_cache_subconfigs, 5 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\","
+ "\"clear\",\"tree_walk\"]",
NULL, 0 },
{ "statistics_log", "category",
NULL, NULL,
@@ -334,7 +335,8 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = {
{ "readonly", "boolean", NULL, NULL, NULL, 0 },
{ "skip_sort_check", "boolean", NULL, NULL, NULL, 0 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"clear\",\"size\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"clear\","
+ "\"size\",\"tree_walk\"]",
NULL, 0 },
{ "target", "list", NULL, NULL, NULL, 0 },
{ NULL, NULL, NULL, NULL, NULL, 0 }
@@ -709,7 +711,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = {
NULL, NULL,
confchk_wiredtiger_open_shared_cache_subconfigs, 5 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\","
+ "\"clear\",\"tree_walk\"]",
NULL, 0 },
{ "statistics_log", "category",
NULL, NULL,
@@ -793,7 +796,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = {
NULL, NULL,
confchk_wiredtiger_open_shared_cache_subconfigs, 5 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\","
+ "\"clear\",\"tree_walk\"]",
NULL, 0 },
{ "statistics_log", "category",
NULL, NULL,
@@ -874,7 +878,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = {
NULL, NULL,
confchk_wiredtiger_open_shared_cache_subconfigs, 5 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\","
+ "\"clear\",\"tree_walk\"]",
NULL, 0 },
{ "statistics_log", "category",
NULL, NULL,
@@ -953,7 +958,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = {
NULL, NULL,
confchk_wiredtiger_open_shared_cache_subconfigs, 5 },
{ "statistics", "list",
- NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]",
+ NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\","
+ "\"clear\",\"tree_walk\"]",
NULL, 0 },
{ "statistics_log", "category",
NULL, NULL,
diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c
index 74c675d794d..8a87aa9c7da 100644
--- a/src/conn/conn_api.c
+++ b/src/conn/conn_api.c
@@ -1668,32 +1668,60 @@ __conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[])
if ((ret = __wt_config_subgets(
session, &cval, "fast", &sval)) == 0 && sval.val != 0) {
- LF_SET(WT_CONN_STAT_FAST);
+ LF_SET(WT_STAT_TYPE_FAST);
++set;
}
WT_RET_NOTFOUND_OK(ret);
if ((ret = __wt_config_subgets(
session, &cval, "all", &sval)) == 0 && sval.val != 0) {
- LF_SET(WT_CONN_STAT_ALL | WT_CONN_STAT_FAST);
+ LF_SET(
+ WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK |
+ WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK);
++set;
}
WT_RET_NOTFOUND_OK(ret);
+ if (set > 1)
+ WT_RET_MSG(session, EINVAL,
+ "Only one of all, fast, none configuration values should "
+ "be specified");
+
+ /*
+ * Now that we've parsed general statistics categories, process
+ * sub-categories.
+ */
+ if ((ret = __wt_config_subgets(
+ session, &cval, "cache_walk", &sval)) == 0 && sval.val != 0)
+ /*
+ * Configuring cache walk statistics implies fast statistics.
+ * Keep that knowledge internal for now - it may change in the
+ * future.
+ */
+ LF_SET(WT_STAT_TYPE_FAST | WT_STAT_TYPE_CACHE_WALK);
+ WT_RET_NOTFOUND_OK(ret);
+
+ if ((ret = __wt_config_subgets(
+ session, &cval, "tree_walk", &sval)) == 0 && sval.val != 0)
+ /*
+ * Configuring tree walk statistics implies fast statistics.
+ * Keep that knowledge internal for now - it may change in the
+ * future.
+ */
+ LF_SET(WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK);
+ WT_RET_NOTFOUND_OK(ret);
+
if ((ret = __wt_config_subgets(
session, &cval, "clear", &sval)) == 0 && sval.val != 0) {
- if (!LF_ISSET(WT_CONN_STAT_FAST | WT_CONN_STAT_ALL))
+ if (!LF_ISSET(WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK |
+ WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK))
WT_RET_MSG(session, EINVAL,
- "the value \"clear\" can be specified only if "
- "either \"all\" or \"fast\" is specified");
- LF_SET(WT_CONN_STAT_CLEAR);
+ "the value \"clear\" can only be specified if "
+ "statistics are enabled");
+ LF_SET(WT_STAT_CLEAR);
}
WT_RET_NOTFOUND_OK(ret);
- if (set > 1)
- WT_RET_MSG(session, EINVAL,
- "only one statistics configuration value may be specified");
-
/* Configuring statistics clears any existing values. */
conn->stat_flags = flags;
diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c
index d5a31c671c0..0715a035807 100644
--- a/src/conn/conn_stat.c
+++ b/src/conn/conn_stat.c
@@ -130,12 +130,12 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp)
WT_RET(__wt_config_gets(session, cfg, "statistics_log.json", &cval));
if (cval.val != 0)
- FLD_SET(conn->stat_flags, WT_CONN_STAT_JSON);
+ FLD_SET(conn->stat_flags, WT_STAT_JSON);
WT_RET(__wt_config_gets(
session, cfg, "statistics_log.on_close", &cval));
if (cval.val != 0)
- FLD_SET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE);
+ FLD_SET(conn->stat_flags, WT_STAT_ON_CLOSE);
/*
* We don't allow the log path to be reconfigured for security reasons.
@@ -206,7 +206,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp)
#define WT_TIMESTAMP_JSON_DEFAULT "%Y-%m-%dT%H:%M:%S.000Z"
WT_ERR(__wt_config_gets(
session, cfg, "statistics_log.timestamp", &cval));
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON) &&
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON) &&
WT_STRING_MATCH(WT_TIMESTAMP_DEFAULT, cval.str, cval.len))
WT_ERR(__wt_strdup(
session, WT_TIMESTAMP_JSON_DEFAULT, &conn->stat_format));
@@ -264,7 +264,7 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats)
goto err;
}
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) {
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) {
WT_ERR(__wt_fprintf(session, conn->stat_fs,
"{\"version\":\"%s\",\"localTime\":\"%s\"",
WIREDTIGER_VERSION_STRING, conn->stat_stamp));
@@ -482,7 +482,7 @@ __wt_statlog_log_one(WT_SESSION_IMPL *session)
conn = S2C(session);
- if (!FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE))
+ if (!FLD_ISSET(conn->stat_flags, WT_STAT_ON_CLOSE))
return (0);
if (F_ISSET(conn, WT_CONN_SERVER_RUN) &&
diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c
index 700cc366ff0..b36416debe1 100644
--- a/src/cursor/cur_stat.c
+++ b/src/cursor/cur_stat.c
@@ -354,7 +354,7 @@ __curstat_conn_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst)
*/
__wt_conn_stat_init(session);
__wt_stat_connection_aggregate(conn->stats, &cst->u.conn_stats);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
__wt_stat_connection_clear_all(conn->stats);
cst->stats = (int64_t *)&cst->u.conn_stats;
@@ -380,7 +380,7 @@ __curstat_file_init(WT_SESSION_IMPL *session,
* If we are only getting the size of the file, we don't need to open
* the tree.
*/
- if (F_ISSET(cst, WT_CONN_STAT_SIZE)) {
+ if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) {
filename = uri;
if (!WT_PREFIX_SKIP(filename, "file:"))
return (EINVAL);
@@ -401,7 +401,7 @@ __curstat_file_init(WT_SESSION_IMPL *session,
if ((ret = __wt_btree_stat_init(session, cst)) == 0) {
__wt_stat_dsrc_init_single(&cst->u.dsrc_stats);
__wt_stat_dsrc_aggregate(dhandle->stats, &cst->u.dsrc_stats);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
__wt_stat_dsrc_clear_all(dhandle->stats);
__wt_curstat_dsrc_final(cst);
}
@@ -604,50 +604,79 @@ __wt_curstat_open(WT_SESSION_IMPL *session,
if ((ret = __wt_config_gets(session, cfg, "statistics", &cval)) == 0) {
if ((ret = __wt_config_subgets(
session, &cval, "all", &sval)) == 0 && sval.val != 0) {
- if (!FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ALL))
+ if (!FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_ALL))
goto config_err;
- F_SET(cst, WT_CONN_STAT_ALL | WT_CONN_STAT_FAST);
+ F_SET(cst, WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK |
+ WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK);
}
WT_ERR_NOTFOUND_OK(ret);
if ((ret = __wt_config_subgets(
session, &cval, "fast", &sval)) == 0 && sval.val != 0) {
- if (F_ISSET(cst, WT_CONN_STAT_ALL))
+ if (F_ISSET(cst, WT_STAT_TYPE_ALL))
WT_ERR_MSG(session, EINVAL,
- "only one statistics configuration value "
- "may be specified");
- F_SET(cst, WT_CONN_STAT_FAST);
+ "Only one of all, fast, none "
+ "configuration values should be specified");
+ F_SET(cst, WT_STAT_TYPE_FAST);
}
WT_ERR_NOTFOUND_OK(ret);
+
+ if ((ret = __wt_config_subgets(session,
+ &cval, "cache_walk", &sval)) == 0 && sval.val != 0) {
+ /*
+ * Configuring cache walk statistics implies fast
+ * statistics. Keep that knowledge internal for now -
+ * it may change in the future.
+ */
+ F_SET(cst, WT_STAT_TYPE_CACHE_WALK | WT_STAT_TYPE_FAST);
+ }
+ WT_ERR_NOTFOUND_OK(ret);
+
+ if ((ret = __wt_config_subgets(session,
+ &cval, "tree_walk", &sval)) == 0 && sval.val != 0) {
+ /*
+ * Configuring tree walk statistics implies fast
+ * statistics. Keep that knowledge internal for now -
+ * it may change in the future.
+ */
+ F_SET(cst, WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK);
+ }
+ WT_ERR_NOTFOUND_OK(ret);
+
if ((ret = __wt_config_subgets(
session, &cval, "size", &sval)) == 0 && sval.val != 0) {
- if (F_ISSET(cst, WT_CONN_STAT_FAST | WT_CONN_STAT_ALL))
+ if (F_ISSET(cst, WT_STAT_TYPE_FAST | WT_STAT_TYPE_ALL))
WT_ERR_MSG(session, EINVAL,
- "only one statistics configuration value "
- "may be specified");
- F_SET(cst, WT_CONN_STAT_SIZE);
+ "Only one of all, fast, none "
+ "configuration values should be specified");
+ F_SET(cst, WT_STAT_TYPE_SIZE);
}
WT_ERR_NOTFOUND_OK(ret);
if ((ret = __wt_config_subgets(
session, &cval, "clear", &sval)) == 0 && sval.val != 0) {
- if (F_ISSET(cst, WT_CONN_STAT_SIZE))
+ if (F_ISSET(cst, WT_STAT_TYPE_SIZE))
WT_ERR_MSG(session, EINVAL,
"clear is incompatible with size "
"statistics");
- F_SET(cst, WT_CONN_STAT_CLEAR);
+ F_SET(cst, WT_STAT_CLEAR);
}
WT_ERR_NOTFOUND_OK(ret);
/* If no configuration, use the connection's configuration. */
if (cst->flags == 0) {
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ALL))
- F_SET(cst, WT_CONN_STAT_ALL);
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_FAST))
- F_SET(cst, WT_CONN_STAT_FAST);
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_ALL))
+ F_SET(cst, WT_STAT_TYPE_ALL);
+ if (FLD_ISSET(
+ conn->stat_flags, WT_STAT_TYPE_CACHE_WALK))
+ F_SET(cst, WT_STAT_TYPE_CACHE_WALK);
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_FAST))
+ F_SET(cst, WT_STAT_TYPE_FAST);
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_TREE_WALK))
+ F_SET(cst, WT_STAT_TYPE_TREE_WALK);
}
/* If the connection configures clear, so do we. */
- if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR))
- F_SET(cst, WT_CONN_STAT_CLEAR);
+ if (FLD_ISSET(conn->stat_flags, WT_STAT_CLEAR))
+ F_SET(cst, WT_STAT_CLEAR);
}
/*
@@ -670,9 +699,9 @@ __wt_curstat_open(WT_SESSION_IMPL *session,
/*
* Do the initial statistics snapshot: there won't be cursor operations
- * to trigger initialization when aggregating statistics for upper-level
- * objects like tables, we need to a valid set of statistics when before
- * the open returns.
+ * to trigger initialization with aggregating statistics for upper-level
+ * objects like tables so we need a valid set of statistics before the
+ * open returns.
*/
WT_ERR(__wt_curstat_init(session, uri, other, cst->cfg, cst));
cst->notinitialized = false;
diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c
index db1d19d60d7..8ca73160cd5 100644
--- a/src/evict/evict_lru.c
+++ b/src/evict/evict_lru.c
@@ -548,6 +548,7 @@ __evict_pass(WT_SESSION_IMPL *session)
* does need to do some work.
*/
__wt_cache_read_gen_incr(session);
+ ++cache->evict_pass_gen;
/*
* Update the oldest ID: we use it to decide whether pages are
@@ -1411,6 +1412,7 @@ __evict_walk_file(WT_SESSION_IMPL *session,
page = ref->page;
modified = __wt_page_is_modified(page);
+ page->evict_pass_gen = cache->evict_pass_gen;
/*
* Use the EVICT_LRU flag to avoid putting pages onto the list
diff --git a/src/evict/evict_stat.c b/src/evict/evict_stat.c
new file mode 100644
index 00000000000..cbb21d0bce6
--- /dev/null
+++ b/src/evict/evict_stat.c
@@ -0,0 +1,150 @@
+/*-
+ * Copyright (c) 2014-2016 MongoDB, Inc.
+ * Copyright (c) 2008-2014 WiredTiger, Inc.
+ * All rights reserved.
+ *
+ * See the file LICENSE for redistribution information.
+ */
+
+#include "wt_internal.h"
+
+/*
+ * __evict_stat_walk --
+ * Walk all the pages in cache for a dhandle gathering stats information
+ */
+static int
+__evict_stat_walk(WT_SESSION_IMPL *session)
+{
+ WT_BTREE *btree;
+ WT_PAGE *page;
+ WT_REF *next_walk;
+ uint64_t dsk_size, gen_gap, size;
+ uint64_t bytes_inmem;
+ uint64_t written_size_cnt, written_size_sum;
+ uint64_t gen_gap_cnt, gen_gap_max, gen_gap_sum;
+ uint64_t max_pagesize, min_written_size;
+ uint64_t num_memory, num_queued, num_not_queueable, num_smaller_allocsz;
+ uint64_t pages_clean, pages_dirty, pages_internal, pages_leaf;
+ uint64_t seen_count, walk_count;
+
+ btree = S2BT(session);
+ next_walk = NULL;
+ bytes_inmem = written_size_cnt = written_size_sum = 0;
+ gen_gap_cnt = gen_gap_max = gen_gap_sum = 0;
+ max_pagesize = 0;
+ num_memory = num_queued = num_not_queueable = num_smaller_allocsz = 0;
+ pages_clean = pages_dirty = pages_internal = pages_leaf = 0;
+ seen_count = walk_count = 0;
+ min_written_size = UINT64_MAX;
+
+ while (__wt_tree_walk_count(session, &next_walk, &walk_count,
+ WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 &&
+ next_walk != NULL) {
+ ++seen_count;
+ page = next_walk->page;
+ size = page->memory_footprint;
+
+ bytes_inmem += size;
+
+ if (__wt_page_is_modified(page))
+ ++pages_dirty;
+ else
+ ++pages_clean;
+
+ if (!__wt_ref_is_root(next_walk) &&
+ !__wt_page_can_evict(session, next_walk, NULL))
+ ++num_not_queueable;
+
+ if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU))
+ ++num_queued;
+
+ if (size > max_pagesize)
+ max_pagesize = size;
+
+ dsk_size = page->dsk != NULL ? page->dsk->mem_size : 0;
+ if (dsk_size != 0) {
+ if (dsk_size < btree->allocsize)
+ ++num_smaller_allocsz;
+ if (dsk_size < min_written_size)
+ min_written_size = dsk_size;
+ ++written_size_cnt;
+ written_size_sum += dsk_size;
+ } else
+ ++num_memory;
+
+ if (WT_PAGE_IS_INTERNAL(page)) {
+ ++pages_internal;
+ } else {
+ ++pages_leaf;
+ }
+
+ /* Skip root pages since they are never considered */
+ if (__wt_ref_is_root(next_walk))
+ continue;
+
+ gen_gap =
+ S2C(session)->cache->evict_pass_gen - page->evict_pass_gen;
+ if (gen_gap > gen_gap_max)
+ gen_gap_max = gen_gap;
+ gen_gap_sum += gen_gap;
+ ++gen_gap_cnt;
+ }
+
+ WT_STAT_DATA_SET(session,
+ cache_state_avg_written_size, written_size_cnt == 0 ?
+ 0 : written_size_sum / written_size_cnt);
+ WT_STAT_DATA_SET(session,
+ cache_state_gen_avg_gap, gen_gap_cnt == 0 ?
+ 0 : gen_gap_sum / gen_gap_cnt);
+
+ WT_STAT_DATA_SET(session, cache_state_gen_max_gap, gen_gap_max);
+ WT_STAT_DATA_SET(session, cache_state_max_pagesize, max_pagesize);
+ WT_STAT_DATA_SET(session,
+ cache_state_min_written_size, min_written_size);
+ WT_STAT_DATA_SET(session, cache_state_memory, num_memory);
+ WT_STAT_DATA_SET(session, cache_state_queued, num_queued);
+ WT_STAT_DATA_SET(session,
+ cache_state_not_queueable, num_not_queueable);
+ WT_STAT_DATA_SET(session,
+ cache_state_smaller_alloc_size, num_smaller_allocsz);
+ WT_STAT_DATA_SET(session, cache_state_pages, walk_count);
+ WT_STAT_DATA_SET(session, cache_state_pages_clean, pages_clean);
+ WT_STAT_DATA_SET(session, cache_state_pages_dirty, pages_dirty);
+ WT_STAT_DATA_SET(session, cache_state_pages_internal, pages_internal);
+ WT_STAT_DATA_SET(session, cache_state_pages_leaf, pages_leaf);
+ WT_STAT_DATA_SET(session,
+ cache_state_refs_skipped, walk_count - seen_count);
+
+ return (0);
+}
+
+/*
+ * __wt_curstat_cache_walk --
+ * Initialize the statistics for a cache cache_walk pass.
+ */
+int
+__wt_curstat_cache_walk(WT_SESSION_IMPL *session)
+{
+ WT_BTREE *btree;
+ WT_CONNECTION_IMPL *conn;
+ WT_DECL_RET;
+ WT_PAGE_INDEX *root_idx;
+
+ btree = S2BT(session);
+ conn = S2C(session);
+
+ /* Set statistics that don't require walking the cache. */
+ WT_STAT_DATA_SET(session,
+ cache_state_gen_current, conn->cache->evict_pass_gen);
+
+ /* Root page statistics */
+ root_idx = WT_INTL_INDEX_GET_SAFE(btree->root.page);
+ WT_STAT_DATA_SET(session,
+ cache_state_root_entries, root_idx->entries);
+ WT_STAT_DATA_SET(session,
+ cache_state_root_size, btree->root.page->memory_footprint);
+
+ WT_WITH_HANDLE_LIST_LOCK(session, ret = __evict_stat_walk(session));
+
+ return (ret);
+}
diff --git a/src/include/btmem.h b/src/include/btmem.h
index b4ca937e7ed..278d0e50ee8 100644
--- a/src/include/btmem.h
+++ b/src/include/btmem.h
@@ -619,6 +619,8 @@ struct __wt_page {
#define WT_READGEN_START_VALUE 100
#define WT_READGEN_STEP 100
uint64_t read_gen;
+ /* The evict pass generation for the page */
+ uint64_t evict_pass_gen;
size_t memory_footprint; /* Memory attached to the page */
diff --git a/src/include/cache.h b/src/include/cache.h
index b24b625aec4..9a2b83b5b57 100644
--- a/src/include/cache.h
+++ b/src/include/cache.h
@@ -91,6 +91,7 @@ struct __wt_cache {
uint64_t read_gen; /* Current page read generation */
uint64_t read_gen_oldest; /* Oldest read generation the eviction
* server saw in its last queue load */
+ uint64_t evict_pass_gen; /* Number of eviction passes */
/*
* Eviction thread information.
diff --git a/src/include/connection.h b/src/include/connection.h
index ce81dcf5976..d7c3bf69686 100644
--- a/src/include/connection.h
+++ b/src/include/connection.h
@@ -285,13 +285,7 @@ struct __wt_connection_impl {
uint64_t ckpt_time_recent; /* Checkpoint time recent/total */
uint64_t ckpt_time_total;
-#define WT_CONN_STAT_ALL 0x01 /* "all" statistics configured */
-#define WT_CONN_STAT_CLEAR 0x02 /* clear after gathering */
-#define WT_CONN_STAT_FAST 0x04 /* "fast" statistics configured */
-#define WT_CONN_STAT_JSON 0x08 /* output JSON format */
-#define WT_CONN_STAT_ON_CLOSE 0x10 /* output statistics on close */
-#define WT_CONN_STAT_SIZE 0x20 /* "size" statistics configured */
- uint32_t stat_flags;
+ uint32_t stat_flags; /* Options declared in flags.py */
/* Connection statistics */
WT_CONNECTION_STATS *stats[WT_COUNTER_SLOTS];
diff --git a/src/include/cursor.h b/src/include/cursor.h
index f1fa4d193ac..e322a53a65d 100644
--- a/src/include/cursor.h
+++ b/src/include/cursor.h
@@ -467,7 +467,7 @@ struct __wt_cursor_stat {
uint64_t v; /* Current stats value */
WT_ITEM pv; /* Current stats value (string) */
- /* Uses the same values as WT_CONNECTION::stat_flags field */
+ /* Options declared in flags.py, shared by WT_CONNECTION::stat_flags */
uint32_t flags;
};
diff --git a/src/include/extern.h b/src/include/extern.h
index 9660e641e50..94b1feede47 100644
--- a/src/include/extern.h
+++ b/src/include/extern.h
@@ -353,6 +353,7 @@ extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session);
extern int __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
+extern int __wt_curstat_cache_walk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn);
extern int __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result));
extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn);
diff --git a/src/include/flags.h b/src/include/flags.h
index 5d718da473d..b0d167525b2 100644
--- a/src/include/flags.h
+++ b/src/include/flags.h
@@ -70,6 +70,14 @@
#define WT_SESSION_NO_SCHEMA_LOCK 0x00020000
#define WT_SESSION_QUIET_CORRUPT_FILE 0x00040000
#define WT_SESSION_SERVER_ASYNC 0x00080000
+#define WT_STAT_CLEAR 0x00000001
+#define WT_STAT_JSON 0x00000002
+#define WT_STAT_ON_CLOSE 0x00000004
+#define WT_STAT_TYPE_ALL 0x00000008
+#define WT_STAT_TYPE_CACHE_WALK 0x00000010
+#define WT_STAT_TYPE_FAST 0x00000020
+#define WT_STAT_TYPE_SIZE 0x00000040
+#define WT_STAT_TYPE_TREE_WALK 0x00000080
#define WT_TXN_LOG_CKPT_CLEANUP 0x00000001
#define WT_TXN_LOG_CKPT_PREPARE 0x00000002
#define WT_TXN_LOG_CKPT_START 0x00000004
diff --git a/src/include/stat.h b/src/include/stat.h
index 45c21d89cb3..15edce9ec85 100644
--- a/src/include/stat.h
+++ b/src/include/stat.h
@@ -555,6 +555,24 @@ struct __wt_dsrc_stats {
int64_t cache_write;
int64_t cache_write_restore;
int64_t cache_eviction_clean;
+ int64_t cache_state_gen_avg_gap;
+ int64_t cache_state_avg_written_size;
+ int64_t cache_state_pages_clean;
+ int64_t cache_state_gen_current;
+ int64_t cache_state_pages_dirty;
+ int64_t cache_state_root_entries;
+ int64_t cache_state_pages_internal;
+ int64_t cache_state_pages_leaf;
+ int64_t cache_state_gen_max_gap;
+ int64_t cache_state_max_pagesize;
+ int64_t cache_state_min_written_size;
+ int64_t cache_state_smaller_alloc_size;
+ int64_t cache_state_memory;
+ int64_t cache_state_queued;
+ int64_t cache_state_not_queueable;
+ int64_t cache_state_refs_skipped;
+ int64_t cache_state_root_size;
+ int64_t cache_state_pages;
int64_t compress_read;
int64_t compress_write;
int64_t compress_write_fail;
diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in
index 5f61e86b561..f3156609fed 100644
--- a/src/include/wiredtiger.in
+++ b/src/include/wiredtiger.in
@@ -962,8 +962,9 @@ struct __wt_session {
* where appropriate (for example\, a cache size statistic is not
* cleared\, while the count of cursor insert operations will be
* cleared). See @ref statistics for more information., a list\, with
- * values chosen from the following options: \c "all"\, \c "fast"\, \c
- * "clear"\, \c "size"; default empty.}
+ * values chosen from the following options: \c "all"\, \c
+ * "cache_walk"\, \c "fast"\, \c "clear"\, \c "size"\, \c "tree_walk";
+ * default empty.}
* @config{target, if non-empty\, backup the list of objects; valid only
* for a backup data source., a list of strings; default empty.}
* @configend
@@ -1903,8 +1904,9 @@ struct __wt_connection {
* reset each time a statistics cursor is used to gather statistics\, as
* well as each time statistics are logged using the \c statistics_log
* configuration. See @ref statistics for more information., a list\,
- * with values chosen from the following options: \c "all"\, \c "fast"\,
- * \c "none"\, \c "clear"; default \c none.}
+ * with values chosen from the following options: \c "all"\, \c
+ * "cache_walk"\, \c "fast"\, \c "none"\, \c "clear"\, \c "tree_walk";
+ * default \c none.}
* @config{statistics_log = (, log any statistics the database is
* configured to maintain\, to a file. See @ref statistics for more
* information. Enabling the statistics log server uses a session from
@@ -2406,8 +2408,9 @@ struct __wt_connection {
* statistics are reset each time a statistics cursor is used to gather
* statistics\, as well as each time statistics are logged using the \c
* statistics_log configuration. See @ref statistics for more information., a
- * list\, with values chosen from the following options: \c "all"\, \c "fast"\,
- * \c "none"\, \c "clear"; default \c none.}
+ * list\, with values chosen from the following options: \c "all"\, \c
+ * "cache_walk"\, \c "fast"\, \c "none"\, \c "clear"\, \c "tree_walk"; default
+ * \c none.}
* @config{statistics_log = (, log any statistics the database is configured to
* maintain\, to a file. See @ref statistics for more information. Enabling
* the statistics log server uses a session from the configured session_max., a
@@ -4757,28 +4760,28 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
/*! btree: btree checkpoint generation */
#define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2022
/*!
- * btree: column-store fixed-size leaf pages, only reported if
- * statistics=all is set
+ * btree: column-store fixed-size leaf pages, only reported if tree_walk
+ * or all statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_COLUMN_FIX 2023
/*!
- * btree: column-store internal pages, only reported if statistics=all is
- * set
+ * btree: column-store internal pages, only reported if tree_walk or all
+ * statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2024
/*!
* btree: column-store variable-size RLE encoded values, only reported if
- * statistics=all is set
+ * tree_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_COLUMN_RLE 2025
/*!
* btree: column-store variable-size deleted values, only reported if
- * statistics=all is set
+ * tree_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2026
/*!
* btree: column-store variable-size leaf pages, only reported if
- * statistics=all is set
+ * tree_walk or all statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2027
/*! btree: fixed-record size */
@@ -4796,20 +4799,26 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
/*! btree: maximum tree depth */
#define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2034
/*!
- * btree: number of key/value pairs, only reported if statistics=all is
- * set
+ * btree: number of key/value pairs, only reported if tree_walk or all
+ * statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_ENTRIES 2035
-/*! btree: overflow pages, only reported if statistics=all is set */
+/*!
+ * btree: overflow pages, only reported if tree_walk or all statistics
+ * are enabled
+ */
#define WT_STAT_DSRC_BTREE_OVERFLOW 2036
/*! btree: pages rewritten by compaction */
#define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2037
/*!
- * btree: row-store internal pages, only reported if statistics=all is
- * set
+ * btree: row-store internal pages, only reported if tree_walk or all
+ * statistics are enabled
*/
#define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2038
-/*! btree: row-store leaf pages, only reported if statistics=all is set */
+/*!
+ * btree: row-store leaf pages, only reported if tree_walk or all
+ * statistics are enabled
+ */
#define WT_STAT_DSRC_BTREE_ROW_LEAF 2039
/*! cache: bytes currently in the cache */
#define WT_STAT_DSRC_CACHE_BYTES_INUSE 2040
@@ -4855,87 +4864,179 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection);
#define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060
/*! cache: unmodified pages evicted */
#define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2061
+/*!
+ * cache_walk: Average difference between current eviction generation
+ * when the page was last considered, only reported if cache_walk or all
+ * statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2062
+/*!
+ * cache_walk: Average on-disk page image size seen, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2063
+/*!
+ * cache_walk: Clean pages currently in cache, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2064
+/*!
+ * cache_walk: Current eviction generation, only reported if cache_walk
+ * or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2065
+/*!
+ * cache_walk: Dirty pages currently in cache, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2066
+/*!
+ * cache_walk: Entries in the root page, only reported if cache_walk or
+ * all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2067
+/*!
+ * cache_walk: Internal pages currently in cache, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2068
+/*!
+ * cache_walk: Leaf pages currently in cache, only reported if cache_walk
+ * or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2069
+/*!
+ * cache_walk: Maximum difference between current eviction generation
+ * when the page was last considered, only reported if cache_walk or all
+ * statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2070
+/*!
+ * cache_walk: Maximum page size seen, only reported if cache_walk or all
+ * statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2071
+/*!
+ * cache_walk: Minimum on-disk page image size seen, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2072
+/*!
+ * cache_walk: On-disk page image sizes smaller than a single allocation
+ * unit, only reported if cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2073
+/*!
+ * cache_walk: Pages created in memory and never written, only reported
+ * if cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2074
+/*!
+ * cache_walk: Pages currently queued for eviction, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2075
+/*!
+ * cache_walk: Pages that could not be queued for eviction, only reported
+ * if cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2076
+/*!
+ * cache_walk: Refs skipped during cache traversal, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2077
+/*!
+ * cache_walk: Size of the root page, only reported if cache_walk or all
+ * statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2078
+/*!
+ * cache_walk: Total number of pages currently in cache, only reported if
+ * cache_walk or all statistics are enabled
+ */
+#define WT_STAT_DSRC_CACHE_STATE_PAGES 2079
/*! compression: compressed pages read */
-#define WT_STAT_DSRC_COMPRESS_READ 2062
+#define WT_STAT_DSRC_COMPRESS_READ 2080
/*! compression: compressed pages written */
-#define WT_STAT_DSRC_COMPRESS_WRITE 2063
+#define WT_STAT_DSRC_COMPRESS_WRITE 2081
/*! compression: page written failed to compress */
-#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2064
+#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2082
/*! compression: page written was too small to compress */
-#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2065
+#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2083
/*! compression: raw compression call failed, additional data available */
-#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2066
+#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2084
/*! compression: raw compression call failed, no additional data available */
-#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2067
+#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2085
/*! compression: raw compression call succeeded */
-#define WT_STAT_DSRC_COMPRESS_RAW_OK 2068
+#define WT_STAT_DSRC_COMPRESS_RAW_OK 2086
/*! cursor: bulk-loaded cursor-insert calls */
-#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2069
+#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2087
/*! cursor: create calls */
-#define WT_STAT_DSRC_CURSOR_CREATE 2070
+#define WT_STAT_DSRC_CURSOR_CREATE 2088
/*! cursor: cursor-insert key and value bytes inserted */
-#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2071
+#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2089
/*! cursor: cursor-remove key bytes removed */
-#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2072
+#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2090
/*! cursor: cursor-update value bytes updated */
-#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2073
+#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2091
/*! cursor: insert calls */
-#define WT_STAT_DSRC_CURSOR_INSERT 2074
+#define WT_STAT_DSRC_CURSOR_INSERT 2092
/*! cursor: next calls */
-#define WT_STAT_DSRC_CURSOR_NEXT 2075
+#define WT_STAT_DSRC_CURSOR_NEXT 2093
/*! cursor: prev calls */
-#define WT_STAT_DSRC_CURSOR_PREV 2076
+#define WT_STAT_DSRC_CURSOR_PREV 2094
/*! cursor: remove calls */
-#define WT_STAT_DSRC_CURSOR_REMOVE 2077
+#define WT_STAT_DSRC_CURSOR_REMOVE 2095
/*! cursor: reset calls */
-#define WT_STAT_DSRC_CURSOR_RESET 2078
+#define WT_STAT_DSRC_CURSOR_RESET 2096
/*! cursor: restarted searches */
-#define WT_STAT_DSRC_CURSOR_RESTART 2079
+#define WT_STAT_DSRC_CURSOR_RESTART 2097
/*! cursor: search calls */
-#define WT_STAT_DSRC_CURSOR_SEARCH 2080
+#define WT_STAT_DSRC_CURSOR_SEARCH 2098
/*! cursor: search near calls */
-#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2081
+#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2099
/*! cursor: truncate calls */
-#define WT_STAT_DSRC_CURSOR_TRUNCATE 2082
+#define WT_STAT_DSRC_CURSOR_TRUNCATE 2100
/*! cursor: update calls */
-#define WT_STAT_DSRC_CURSOR_UPDATE 2083
+#define WT_STAT_DSRC_CURSOR_UPDATE 2101
/*! reconciliation: dictionary matches */
-#define WT_STAT_DSRC_REC_DICTIONARY 2084
+#define WT_STAT_DSRC_REC_DICTIONARY 2102
/*! reconciliation: fast-path pages deleted */
-#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2085
+#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2103
/*!
* reconciliation: internal page key bytes discarded using suffix
* compression
*/
-#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2086
+#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2104
/*! reconciliation: internal page multi-block writes */
-#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2087
+#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2105
/*! reconciliation: internal-page overflow keys */
-#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2088
+#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2106
/*! reconciliation: leaf page key bytes discarded using prefix compression */
-#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2089
+#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2107
/*! reconciliation: leaf page multi-block writes */
-#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2090
+#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2108
/*! reconciliation: leaf-page overflow keys */
-#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2091
+#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2109
/*! reconciliation: maximum blocks required for a page */
-#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2092
+#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2110
/*! reconciliation: overflow values written */
-#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2093
+#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2111
/*! reconciliation: page checksum matches */
-#define WT_STAT_DSRC_REC_PAGE_MATCH 2094
+#define WT_STAT_DSRC_REC_PAGE_MATCH 2112
/*! reconciliation: page reconciliation calls */
-#define WT_STAT_DSRC_REC_PAGES 2095
+#define WT_STAT_DSRC_REC_PAGES 2113
/*! reconciliation: page reconciliation calls for eviction */
-#define WT_STAT_DSRC_REC_PAGES_EVICTION 2096
+#define WT_STAT_DSRC_REC_PAGES_EVICTION 2114
/*! reconciliation: pages deleted */
-#define WT_STAT_DSRC_REC_PAGE_DELETE 2097
+#define WT_STAT_DSRC_REC_PAGE_DELETE 2115
/*! session: object compaction */
-#define WT_STAT_DSRC_SESSION_COMPACT 2098
+#define WT_STAT_DSRC_SESSION_COMPACT 2116
/*! session: open cursor count */
-#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2099
+#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2117
/*! transaction: update conflicts */
-#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2100
+#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2118
/*!
* @}
diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c
index f4f5a0acce8..3fe3ca1ba81 100644
--- a/src/lsm/lsm_stat.c
+++ b/src/lsm/lsm_stat.c
@@ -42,11 +42,11 @@ __curstat_lsm_init(
if (cst->flags != 0) {
(void)snprintf(config, sizeof(config),
"statistics=(%s%s%s%s)",
- F_ISSET(cst, WT_CONN_STAT_ALL) ? "all," : "",
- F_ISSET(cst, WT_CONN_STAT_CLEAR) ? "clear," : "",
- !F_ISSET(cst, WT_CONN_STAT_ALL) &&
- F_ISSET(cst, WT_CONN_STAT_FAST) ? "fast," : "",
- F_ISSET(cst, WT_CONN_STAT_SIZE) ? "size," : "");
+ F_ISSET(cst, WT_STAT_TYPE_ALL) ? "all," : "",
+ F_ISSET(cst, WT_STAT_CLEAR) ? "clear," : "",
+ !F_ISSET(cst, WT_STAT_TYPE_ALL) &&
+ F_ISSET(cst, WT_STAT_TYPE_FAST) ? "fast," : "",
+ F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : "");
cfg[1] = disk_cfg[1] = config;
}
@@ -132,26 +132,26 @@ __curstat_lsm_init(
/* Include, and optionally clear, LSM-level specific information. */
WT_STAT_WRITE(session, stats, bloom_miss, lsm_tree->bloom_miss);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->bloom_miss = 0;
WT_STAT_WRITE(session, stats, bloom_hit, lsm_tree->bloom_hit);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->bloom_hit = 0;
WT_STAT_WRITE(session,
stats, bloom_false_positive, lsm_tree->bloom_false_positive);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->bloom_false_positive = 0;
WT_STAT_WRITE(session,
stats, lsm_lookup_no_bloom, lsm_tree->lsm_lookup_no_bloom);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->lsm_lookup_no_bloom = 0;
WT_STAT_WRITE(session,
stats, lsm_checkpoint_throttle, lsm_tree->lsm_checkpoint_throttle);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->lsm_checkpoint_throttle = 0;
WT_STAT_WRITE(session,
stats, lsm_merge_throttle, lsm_tree->lsm_merge_throttle);
- if (F_ISSET(cst, WT_CONN_STAT_CLEAR))
+ if (F_ISSET(cst, WT_STAT_CLEAR))
lsm_tree->lsm_merge_throttle = 0;
__wt_curstat_dsrc_final(cst);
diff --git a/src/schema/schema_stat.c b/src/schema/schema_stat.c
index 1cd39d97364..345f9164e9b 100644
--- a/src/schema/schema_stat.c
+++ b/src/schema/schema_stat.c
@@ -137,7 +137,7 @@ __wt_curstat_table_init(WT_SESSION_IMPL *session,
* If only gathering table size statistics, try a fast path that
* avoids the schema and table list locks.
*/
- if (F_ISSET(cst, WT_CONN_STAT_SIZE)) {
+ if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) {
WT_RET(__curstat_size_only(session, uri, &was_fast, cst));
if (was_fast)
return (0);
diff --git a/src/support/stat.c b/src/support/stat.c
index 8fafd2982e5..d6457ed64bb 100644
--- a/src/support/stat.c
+++ b/src/support/stat.c
@@ -65,6 +65,24 @@ static const char * const __stats_dsrc_desc[] = {
"cache: pages written from cache",
"cache: pages written requiring in-memory restoration",
"cache: unmodified pages evicted",
+ "cache_walk: Average difference between current eviction generation when the page was last considered",
+ "cache_walk: Average on-disk page image size seen",
+ "cache_walk: Clean pages currently in cache",
+ "cache_walk: Current eviction generation",
+ "cache_walk: Dirty pages currently in cache",
+ "cache_walk: Entries in the root page",
+ "cache_walk: Internal pages currently in cache",
+ "cache_walk: Leaf pages currently in cache",
+ "cache_walk: Maximum difference between current eviction generation when the page was last considered",
+ "cache_walk: Maximum page size seen",
+ "cache_walk: Minimum on-disk page image size seen",
+ "cache_walk: On-disk page image sizes smaller than a single allocation unit",
+ "cache_walk: Pages created in memory and never written",
+ "cache_walk: Pages currently queued for eviction",
+ "cache_walk: Pages that could not be queued for eviction",
+ "cache_walk: Refs skipped during cache traversal",
+ "cache_walk: Size of the root page",
+ "cache_walk: Total number of pages currently in cache",
"compression: compressed pages read",
"compression: compressed pages written",
"compression: page written failed to compress",
@@ -196,6 +214,24 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats)
stats->cache_write = 0;
stats->cache_write_restore = 0;
stats->cache_eviction_clean = 0;
+ /* not clearing cache_state_gen_avg_gap */
+ /* not clearing cache_state_avg_written_size */
+ /* not clearing cache_state_pages_clean */
+ /* not clearing cache_state_gen_current */
+ /* not clearing cache_state_pages_dirty */
+ /* not clearing cache_state_root_entries */
+ /* not clearing cache_state_pages_internal */
+ /* not clearing cache_state_pages_leaf */
+ /* not clearing cache_state_gen_max_gap */
+ /* not clearing cache_state_max_pagesize */
+ /* not clearing cache_state_min_written_size */
+ /* not clearing cache_state_smaller_alloc_size */
+ /* not clearing cache_state_memory */
+ /* not clearing cache_state_queued */
+ /* not clearing cache_state_not_queueable */
+ /* not clearing cache_state_refs_skipped */
+ /* not clearing cache_state_root_size */
+ /* not clearing cache_state_pages */
stats->compress_read = 0;
stats->compress_write = 0;
stats->compress_write_fail = 0;
@@ -325,6 +361,27 @@ __wt_stat_dsrc_aggregate_single(
to->cache_write += from->cache_write;
to->cache_write_restore += from->cache_write_restore;
to->cache_eviction_clean += from->cache_eviction_clean;
+ to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap;
+ to->cache_state_avg_written_size +=
+ from->cache_state_avg_written_size;
+ to->cache_state_pages_clean += from->cache_state_pages_clean;
+ to->cache_state_gen_current += from->cache_state_gen_current;
+ to->cache_state_pages_dirty += from->cache_state_pages_dirty;
+ to->cache_state_root_entries += from->cache_state_root_entries;
+ to->cache_state_pages_internal += from->cache_state_pages_internal;
+ to->cache_state_pages_leaf += from->cache_state_pages_leaf;
+ to->cache_state_gen_max_gap += from->cache_state_gen_max_gap;
+ to->cache_state_max_pagesize += from->cache_state_max_pagesize;
+ to->cache_state_min_written_size +=
+ from->cache_state_min_written_size;
+ to->cache_state_smaller_alloc_size +=
+ from->cache_state_smaller_alloc_size;
+ to->cache_state_memory += from->cache_state_memory;
+ to->cache_state_queued += from->cache_state_queued;
+ to->cache_state_not_queueable += from->cache_state_not_queueable;
+ to->cache_state_refs_skipped += from->cache_state_refs_skipped;
+ to->cache_state_root_size += from->cache_state_root_size;
+ to->cache_state_pages += from->cache_state_pages;
to->compress_read += from->compress_read;
to->compress_write += from->compress_write;
to->compress_write_fail += from->compress_write_fail;
@@ -467,6 +524,39 @@ __wt_stat_dsrc_aggregate(
to->cache_write += WT_STAT_READ(from, cache_write);
to->cache_write_restore += WT_STAT_READ(from, cache_write_restore);
to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean);
+ to->cache_state_gen_avg_gap +=
+ WT_STAT_READ(from, cache_state_gen_avg_gap);
+ to->cache_state_avg_written_size +=
+ WT_STAT_READ(from, cache_state_avg_written_size);
+ to->cache_state_pages_clean +=
+ WT_STAT_READ(from, cache_state_pages_clean);
+ to->cache_state_gen_current +=
+ WT_STAT_READ(from, cache_state_gen_current);
+ to->cache_state_pages_dirty +=
+ WT_STAT_READ(from, cache_state_pages_dirty);
+ to->cache_state_root_entries +=
+ WT_STAT_READ(from, cache_state_root_entries);
+ to->cache_state_pages_internal +=
+ WT_STAT_READ(from, cache_state_pages_internal);
+ to->cache_state_pages_leaf +=
+ WT_STAT_READ(from, cache_state_pages_leaf);
+ to->cache_state_gen_max_gap +=
+ WT_STAT_READ(from, cache_state_gen_max_gap);
+ to->cache_state_max_pagesize +=
+ WT_STAT_READ(from, cache_state_max_pagesize);
+ to->cache_state_min_written_size +=
+ WT_STAT_READ(from, cache_state_min_written_size);
+ to->cache_state_smaller_alloc_size +=
+ WT_STAT_READ(from, cache_state_smaller_alloc_size);
+ to->cache_state_memory += WT_STAT_READ(from, cache_state_memory);
+ to->cache_state_queued += WT_STAT_READ(from, cache_state_queued);
+ to->cache_state_not_queueable +=
+ WT_STAT_READ(from, cache_state_not_queueable);
+ to->cache_state_refs_skipped +=
+ WT_STAT_READ(from, cache_state_refs_skipped);
+ to->cache_state_root_size +=
+ WT_STAT_READ(from, cache_state_root_size);
+ to->cache_state_pages += WT_STAT_READ(from, cache_state_pages);
to->compress_read += WT_STAT_READ(from, compress_read);
to->compress_write += WT_STAT_READ(from, compress_write);
to->compress_write_fail += WT_STAT_READ(from, compress_write_fail);
diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py
index 3d2a83d1c3c..047d2c74499 100644
--- a/test/suite/test_stat02.py
+++ b/test/suite/test_stat02.py
@@ -165,7 +165,7 @@ class test_stat_cursor_conn_error(wttest.WiredTigerTestCase):
args = ['none', 'all', 'fast']
for i in list(itertools.permutations(args, 2)):
config = 'create,statistics=(' + i[0] + ',' + i[1] + ')'
- msg = '/only one statistics configuration value/'
+ msg = '/Only one of/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.wiredtiger_open('.', config), msg)
@@ -188,10 +188,76 @@ class test_stat_cursor_dsrc_error(wttest.WiredTigerTestCase):
args = ['all', 'fast']
for i in list(itertools.permutations(args, 2)):
config = 'statistics=(' + i[0] + ',' + i[1] + ')'
- msg = '/only one statistics configuration value/'
+ msg = '/Only one of/'
self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
lambda: self.session.open_cursor(
'statistics:' + self.uri, None, config), msg)
+# Test data-source cache walk statistics
+class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase):
+ uri = 'file:test_stat_cursor_dsrc_cache_walk'
+
+ conn_config = 'statistics=(none)'
+
+ def test_stat_cursor_dsrc_cache_walk(self):
+ simple_populate(self, self.uri, 'key_format=S', 100)
+ # Ensure that it's an error to get cache_walk stats if none is set
+ msg = '/doesn\'t match the database statistics/'
+ self.assertRaisesWithMessage(wiredtiger.WiredTigerError,
+ lambda: self.session.open_cursor(
+ 'statistics:' + self.uri, None, None), msg)
+
+ # Test configurations that are valid but should not collect
+ # cache walk information. Do these first since the cache walk
+ # statistics are mostly marked as not cleared - so once they are
+ # populated the values will always be returned
+ self.conn.reconfigure('statistics=(cache_walk,fast,clear)')
+ c = self.session.open_cursor(
+ 'statistics:' + self.uri, None, 'statistics=(fast)')
+ self.assertEqual(c[stat.dsrc.cache_state_root_size][2], 0)
+ c.close()
+
+ self.conn.reconfigure('statistics=(all,clear)')
+ c = self.session.open_cursor(
+ 'statistics:' + self.uri, None, 'statistics=(fast)')
+ self.assertEqual(c[stat.dsrc.cache_state_root_size][2], 0)
+ c.close()
+
+ self.conn.reconfigure('statistics=(cache_walk,fast,clear)')
+ c = self.session.open_cursor('statistics:' + self.uri, None, None)
+ self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0)
+ # Verify that cache_walk didn't imply tree_walk
+ self.assertEqual(c[stat.dsrc.btree_entries][2], 0)
+ c.close()
+
+ self.conn.reconfigure('statistics=(cache_walk,tree_walk,fast,clear)')
+ c = self.session.open_cursor('statistics:' + self.uri, None, None)
+ self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0)
+ # Verify that cache_walk didn't exclude tree_walk
+ self.assertGreater(c[stat.dsrc.btree_entries][2], 0)
+ c.close()
+
+ self.conn.reconfigure('statistics=(all,clear)')
+ c = self.session.open_cursor(
+ 'statistics:' + self.uri, None, 'statistics=(all)')
+ self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0)
+ self.assertGreater(c[stat.dsrc.btree_entries][2], 0)
+ c.close()
+
+ # Verify that cache and tree walk can operate independantly
+ self.conn.reconfigure('statistics=(all,clear)')
+ c = self.session.open_cursor(
+ 'statistics:' + self.uri, None, 'statistics=(cache_walk,fast)')
+ self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0)
+ self.assertEqual(c[stat.dsrc.btree_entries][2], 0)
+ c.close()
+
+ self.conn.reconfigure('statistics=(all,clear)')
+ c = self.session.open_cursor(
+ 'statistics:' + self.uri, None, 'statistics=(tree_walk,fast)')
+ # Don't check the cache walk stats for empty - they won't be cleared
+ self.assertGreater(c[stat.dsrc.btree_entries][2], 0)
+ c.close()
+
if __name__ == '__main__':
wttest.run()
diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py
index a7202788a95..635e710c469 100644
--- a/tools/wtstats/stat_data.py
+++ b/tools/wtstats/stat_data.py
@@ -91,6 +91,24 @@ no_scale_per_second_list = [
'btree: row-store leaf pages',
'cache: bytes currently in the cache',
'cache: overflow values cached in memory',
+ 'cache_walk: Average difference between current eviction generation when the page was last considered',
+ 'cache_walk: Average on-disk page image size seen',
+ 'cache_walk: Clean pages currently in cache',
+ 'cache_walk: Current eviction generation',
+ 'cache_walk: Dirty pages currently in cache',
+ 'cache_walk: Entries in the root page',
+ 'cache_walk: Internal pages currently in cache',
+ 'cache_walk: Leaf pages currently in cache',
+ 'cache_walk: Maximum difference between current eviction generation when the page was last considered',
+ 'cache_walk: Maximum page size seen',
+ 'cache_walk: Minimum on-disk page image size seen',
+ 'cache_walk: On-disk page image sizes smaller than a single allocation unit',
+ 'cache_walk: Pages created in memory and never written',
+ 'cache_walk: Pages currently queued for eviction',
+ 'cache_walk: Pages that could not be queued for eviction',
+ 'cache_walk: Refs skipped during cache traversal',
+ 'cache_walk: Size of the root page',
+ 'cache_walk: Total number of pages currently in cache',
'LSM: bloom filters in the LSM tree',
'LSM: chunks in the LSM tree',
'LSM: highest merge generation in the LSM tree',
@@ -162,6 +180,24 @@ no_clear_list = [
'transaction: transaction range of IDs currently pinned by named snapshots',
'btree: btree checkpoint generation',
'cache: bytes currently in the cache',
+ 'cache_walk: Average difference between current eviction generation when the page was last considered',
+ 'cache_walk: Average on-disk page image size seen',
+ 'cache_walk: Clean pages currently in cache',
+ 'cache_walk: Current eviction generation',
+ 'cache_walk: Dirty pages currently in cache',
+ 'cache_walk: Entries in the root page',
+ 'cache_walk: Internal pages currently in cache',
+ 'cache_walk: Leaf pages currently in cache',
+ 'cache_walk: Maximum difference between current eviction generation when the page was last considered',
+ 'cache_walk: Maximum page size seen',
+ 'cache_walk: Minimum on-disk page image size seen',
+ 'cache_walk: On-disk page image sizes smaller than a single allocation unit',
+ 'cache_walk: Pages created in memory and never written',
+ 'cache_walk: Pages currently queued for eviction',
+ 'cache_walk: Pages that could not be queued for eviction',
+ 'cache_walk: Refs skipped during cache traversal',
+ 'cache_walk: Size of the root page',
+ 'cache_walk: Total number of pages currently in cache',
'session: open cursor count',
]
prefix_list = [
@@ -177,9 +213,10 @@ prefix_list = [
'session',
'block-manager',
'thread-yield',
+ 'cache_walk',
'async',
'btree',
'thread-state',
'compression',
]
-groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session', 'thread-state'], 'evict': ['block-manager', 'cache', 'connection', 'thread-state'], 'memory': ['cache', 'connection', 'reconciliation']} \ No newline at end of file
+groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session', 'thread-state'], 'evict': ['block-manager', 'cache', 'cache_walk', 'connection', 'thread-state'], 'memory': ['cache', 'cache_walk', 'connection', 'reconciliation']} \ No newline at end of file
diff --git a/tools/wtstats/wtstats.py b/tools/wtstats/wtstats.py
index 3549031c30f..bf5557d12f4 100755
--- a/tools/wtstats/wtstats.py
+++ b/tools/wtstats/wtstats.py
@@ -115,6 +115,9 @@ def parse_wtstats_file(file, result):
# Parse file
for line in open(file, 'rU'):
month, day, time, v, title = line.strip('\n').split(" ", 4)
+ # The colon in the URI confuses parsing, strip it out.
+ if "cache_walk" in title:
+ title = title.replace("file:", "", 1)
result[title].append((month + " " + day + " " + time, v))