diff options
| -rw-r--r-- | dist/stat.py | 12 | ||||
| -rw-r--r-- | dist/stat_data.py | 2 | ||||
| -rw-r--r-- | src/conn/conn_api.c | 2 | ||||
| -rw-r--r-- | src/conn/conn_cache_pool.c | 2 | ||||
| -rw-r--r-- | src/conn/conn_dhandle.c | 3 | ||||
| -rw-r--r-- | src/conn/conn_log.c | 3 | ||||
| -rw-r--r-- | src/conn/conn_stat.c | 84 | ||||
| -rw-r--r-- | src/cursor/cur_backup.c | 10 | ||||
| -rw-r--r-- | src/cursor/cur_stat.c | 188 | ||||
| -rw-r--r-- | src/include/cursor.h | 14 | ||||
| -rw-r--r-- | src/include/extern.h | 28 | ||||
| -rw-r--r-- | src/include/stat.h | 4 | ||||
| -rw-r--r-- | src/lsm/lsm_merge.c | 4 | ||||
| -rw-r--r-- | src/lsm/lsm_stat.c | 77 | ||||
| -rw-r--r-- | src/lsm/lsm_tree.c | 8 | ||||
| -rw-r--r-- | src/schema/schema_stat.c | 100 | ||||
| -rw-r--r-- | src/session/session_api.c | 2 | ||||
| -rw-r--r-- | src/support/stat.c | 12 | ||||
| -rw-r--r-- | src/txn/txn_ckpt.c | 2 | ||||
| -rw-r--r-- | tools/statlog.py | 16 |
20 files changed, 326 insertions, 247 deletions
diff --git a/dist/stat.py b/dist/stat.py index 3c4be936aed..e6021cab18d 100644 --- a/dist/stat.py +++ b/dist/stat.py @@ -1,5 +1,5 @@ # Read the source files and output the statistics #defines plus the -# initialize and clear code. +# initialize and refresh code. import re, string, sys, textwrap from dist import compare_srcfile @@ -94,6 +94,9 @@ def print_func(name, list): void __wt_stat_init_''' + name + '''_stats(WT_''' + name.upper() + '''_STATS *stats) { +\t/* Clear, so can also be called for reinitialization. */ +\tmemset(stats, 0, sizeof(*stats)); + ''') for l in sorted(list): o = '\tstats->' + l.name + '.desc = "' + l.desc + '";\n' @@ -105,7 +108,7 @@ __wt_stat_init_''' + name + '''_stats(WT_''' + name.upper() + '''_STATS *stats) f.write(''' void -__wt_stat_clear_''' + name + '''_stats(void *stats_arg) +__wt_stat_refresh_''' + name + '''_stats(void *stats_arg) { \tWT_''' + name.upper() + '''_STATS *stats; @@ -123,7 +126,8 @@ __wt_stat_clear_''' + name + '''_stats(void *stats_arg) f.write(''' void -__wt_stat_aggregate_''' + name + '''_stats(void *child, void *parent) +__wt_stat_aggregate_''' + name + +'''_stats(const void *child, const void *parent) { \tWT_''' + name.upper() + '''_STATS *c, *p; @@ -141,7 +145,7 @@ __wt_stat_aggregate_''' + name + '''_stats(void *child, void *parent) f.write('\t' + o + '\n') f.write('}\n') -# Write the stat initialization and clear routines to the stat.c file. +# Write the stat initialization and refresh routines to the stat.c file. f = open(tmp_file, 'w') f.write('/* DO NOT EDIT: automatically built by dist/stat.py. */\n\n') f.write('#include "wt_internal.h"\n') diff --git a/dist/stat_data.py b/dist/stat_data.py index 03b07c6305f..8cd0c38bb77 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -7,7 +7,7 @@ # currently open'. # # Optional configuration flags: -# no_clear Value ignored by the statistics clear function +# no_clear Value ignored by the statistics refresh function # no_aggregate Ignore the value when aggregating statistics # max_aggregate Take the maximum value when aggregating statistics # no_scale Don't scale value per second in the logging tool script diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index a0bb2bcaecb..831ceb7bdf9 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -552,7 +552,7 @@ __conn_reconfigure(WT_CONNECTION *wt_conn, const char *config) __wt_config_gets(session, raw_cfg, "statistics", &cval)) == 0) { conn->statistics = cval.val == 0 ? 0 : 1; if (conn->statistics) - __wt_stat_clear_connection_stats(&conn->stats); + __wt_stat_refresh_connection_stats(&conn->stats); } WT_ERR_NOTFOUND_OK(ret); diff --git a/src/conn/conn_cache_pool.c b/src/conn/conn_cache_pool.c index 025ffc79d77..15aed596cca 100644 --- a/src/conn/conn_cache_pool.c +++ b/src/conn/conn_cache_pool.c @@ -442,7 +442,7 @@ __cache_pool_adjust(uint64_t highest, uint64_t bump_threshold) read_pressure = cache->cp_current_evict / highest; WT_VERBOSE_RET(session, shared_cache, - "\t%"PRIu64", %"PRIu64", %d", + "\t%" PRIu64 ", %" PRIu64 ", %" PRIu32, entry->cache_size, read_pressure, cache->cp_skip_count); /* Allow to stabilize after changes. */ diff --git a/src/conn/conn_dhandle.c b/src/conn/conn_dhandle.c index cdd060ae907..d5a66bf3747 100644 --- a/src/conn/conn_dhandle.c +++ b/src/conn/conn_dhandle.c @@ -441,6 +441,7 @@ __wt_conn_btree_get(WT_SESSION_IMPL *session, */ int __wt_conn_btree_apply(WT_SESSION_IMPL *session, + int apply_checkpoints, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]) { WT_CONNECTION_IMPL *conn; @@ -454,7 +455,7 @@ __wt_conn_btree_apply(WT_SESSION_IMPL *session, SLIST_FOREACH(dhandle, &conn->dhlh, l) if (F_ISSET(dhandle, WT_DHANDLE_OPEN) && WT_PREFIX_MATCH(dhandle->name, "file:") && - dhandle->checkpoint == NULL && + (apply_checkpoints || dhandle->checkpoint == NULL) && !WT_IS_METADATA(dhandle)) { /* * We need to pull the handle into the session handle diff --git a/src/conn/conn_log.c b/src/conn/conn_log.c index 3103063040b..1997d96b864 100644 --- a/src/conn/conn_log.c +++ b/src/conn/conn_log.c @@ -97,7 +97,8 @@ __log_archive_server(void *arg) lsn = log->ckpt_lsn; lsn.offset = 0; WT_VERBOSE_ERR(session, log, - "log_archive: ckpt LSN %d,%" PRIu64, lsn.file, lsn.offset); + "log_archive: ckpt LSN %" PRIu32 ",%" PRIu64, + lsn.file, lsn.offset); /* * Main archive code. Get the list of all log files and * remove any earlier than the checkpoint LSN. diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index 6a332d9fd78..5d135f88a30 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -103,7 +103,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, int *runp) /* * __statlog_dump -- - * Dump out the connection statistics. + * Dump out handle/connection statistics. */ static int __statlog_dump(WT_SESSION_IMPL *session, const char *name, int conn_stats) @@ -112,52 +112,55 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, int conn_stats) WT_CURSOR *cursor; WT_DECL_ITEM(tmp); WT_DECL_RET; - WT_SESSION *wt_session; - uint64_t value; - const char *config, *desc, *pdesc, *uri; + WT_STATS *stats; + u_int i; + uint64_t max; + const char *uri; + const char *cfg[] = { + WT_CONFIG_BASE(session, session_open_cursor), NULL, NULL }; conn = S2C(session); - /* Build the statistics cursor URI. */ + /* Build URI and configuration string. */ if (conn_stats) uri = "statistics:"; else { - WT_RET(__wt_scr_alloc( - session, strlen("statistics:") + strlen(name) + 5, &tmp)); - (void)strcpy(tmp->mem, "statistics:"); - (void)strcat(tmp->mem, name); - uri = tmp->mem; + WT_RET(__wt_scr_alloc(session, 0, &tmp)); + WT_ERR(__wt_buf_fmt(session, tmp, "statistics:%s", name)); + uri = tmp->data; } - - /* - * Open the statistics cursor; immediately free any temporary buffer, - * it makes error handling easier. - */ - wt_session = (WT_SESSION *)session; - config = S2C(session)->stat_clear ? + cfg[1] = S2C(session)->stat_clear ? "statistics_clear,statistics_fast" : "statistics_fast"; - ret = wt_session->open_cursor(wt_session, uri, NULL, config, &cursor); - __wt_scr_free(&tmp); /* + * Open the statistics cursor and dump the statistics. + * * If we don't find an underlying object, silently ignore it, the object - * may exist only intermittently. User-level APIs return ENOENT instead - * of WT_NOTFOUND for missing files, check both, as well as for EBUSY if - * the handle is exclusively locked at the moment. + * may exist only intermittently. */ - if (ret == EBUSY || ret == ENOENT || ret == WT_NOTFOUND) - return (0); - WT_RET(ret); - - while ((ret = cursor->next(cursor)) == 0 && - (ret = cursor->get_value(cursor, &desc, &pdesc, &value)) == 0) - WT_ERR_TEST((fprintf(conn->stat_fp, - "%s %" PRIu64 " %s %s\n", - conn->stat_stamp, value, name, desc) < 0), __wt_errno()); - WT_ERR_NOTFOUND_OK(ret); - -err: WT_TRET(cursor->close(cursor)); + switch (ret = __wt_curstat_open(session, uri, cfg, &cursor)) { + case 0: + max = conn_stats ? + sizeof(WT_CONNECTION_STATS) / sizeof(WT_STATS) : + sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); + for (i = 0, + stats = WT_CURSOR_STATS(cursor); i < max; ++i, ++stats) + WT_ERR_TEST((fprintf(conn->stat_fp, + "%s %" PRIu64 " %s %s\n", + conn->stat_stamp, + stats->v, name, stats->desc) < 0), __wt_errno()); + WT_ERR(cursor->close(cursor)); + break; + case EBUSY: + case ENOENT: + case WT_NOTFOUND: + ret = 0; + break; + default: + break; + } +err: __wt_scr_free(&tmp); return (ret); } @@ -183,14 +186,14 @@ __statlog_apply(WT_SESSION_IMPL *session, const char *cfg[]) } /* - * __wt_statlog_lsm_apply -- + * __statlog_lsm_apply -- * Review the list open LSM trees, and dump statistics on demand. * * XXX * This code should be removed when LSM objects are converted to data handles. */ static int -__wt_statlog_lsm_apply(WT_SESSION_IMPL *session) +__statlog_lsm_apply(WT_SESSION_IMPL *session) { #define WT_LSM_TREE_LIST_SLOTS 100 WT_LSM_TREE *lsm_tree, *list[WT_LSM_TREE_LIST_SLOTS]; @@ -322,11 +325,12 @@ __statlog_server(void *arg) * Lock the schema and walk the list of open handles, dumping * any that match the list of object sources. */ - if (conn->stat_sources != NULL) + if (conn->stat_sources != NULL) { WT_WITH_SCHEMA_LOCK(session, ret = __wt_conn_btree_apply( - session, __statlog_apply, NULL)); - WT_ERR(ret); + session, 0, __statlog_apply, NULL)); + WT_ERR(ret); + } /* * Walk the list of open LSM trees, dumping any that match the @@ -337,7 +341,7 @@ __statlog_server(void *arg) * data handles. */ if (conn->stat_sources != NULL) - WT_ERR(__wt_statlog_lsm_apply(session)); + WT_ERR(__statlog_lsm_apply(session)); /* Flush. */ WT_ERR(fflush(fp) == 0 ? 0 : __wt_errno()); diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c index 126f802061c..803f224abc2 100644 --- a/src/cursor/cur_backup.c +++ b/src/cursor/cur_backup.c @@ -11,6 +11,7 @@ static int __backup_all(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *); static int __backup_file_remove(WT_SESSION_IMPL *); +static int __backup_list_all_append(WT_SESSION_IMPL *, const char *[]); static int __backup_list_append( WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *); static int __backup_start( @@ -339,8 +340,7 @@ __backup_all(WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb) WT_ERR_NOTFOUND_OK(ret); /* Build a list of the file objects that need to be copied. */ - WT_ERR( - __wt_meta_btree_apply(session, __wt_backup_list_all_append, NULL)); + WT_ERR(__wt_meta_btree_apply(session, __backup_list_all_append, NULL)); err: if (cursor != NULL) WT_TRET(cursor->close(cursor)); @@ -449,12 +449,12 @@ __wt_backup_list_uri_append(WT_SESSION_IMPL *session, const char *name) } /* - * __wt_backup_list_all_append -- + * __backup_list_all_append -- * Append a new file name to the list, allocate space as necessary. * Called via the __wt_meta_btree_apply function. */ -int -__wt_backup_list_all_append(WT_SESSION_IMPL *session, const char *cfg[]) +static int +__backup_list_all_append(WT_SESSION_IMPL *session, const char *cfg[]) { WT_CURSOR_BACKUP *cb; diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 272afcafa13..5c07757e131 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -45,7 +45,7 @@ __curstat_get_key(WT_CURSOR *cursor, ...) cst = (WT_CURSOR_STAT *)cursor; va_start(ap, cursor); - CURSOR_API_CALL(cursor, session, get_key, cst->btree); + CURSOR_API_CALL(cursor, session, get_key, NULL); WT_CURSOR_NEEDKEY(cursor); @@ -85,7 +85,7 @@ __curstat_get_value(WT_CURSOR *cursor, ...) cst = (WT_CURSOR_STAT *)cursor; va_start(ap, cursor); - CURSOR_API_CALL(cursor, session, get_value, cst->btree); + CURSOR_API_CALL(cursor, session, get_value, NULL); WT_CURSOR_NEEDVALUE(cursor); @@ -132,7 +132,7 @@ __curstat_set_key(WT_CURSOR *cursor, ...) va_list ap; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, set_key, cst->btree); + CURSOR_API_CALL(cursor, session, set_key, NULL); F_CLR(cursor, WT_CURSTD_KEY_SET); va_start(ap, cursor); @@ -173,7 +173,7 @@ __curstat_next(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, next, cst->btree); + CURSOR_API_CALL(cursor, session, next, NULL); /* Move to the next item. */ if (cst->notpositioned) { @@ -205,7 +205,7 @@ __curstat_prev(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, prev, cst->btree); + CURSOR_API_CALL(cursor, session, prev, NULL); /* Move to the previous item. */ if (cst->notpositioned) { @@ -238,7 +238,7 @@ __curstat_reset(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, reset, cst->btree); + CURSOR_API_CALL(cursor, session, reset, NULL); cst->notpositioned = 1; F_CLR(cursor, WT_CURSTD_KEY_SET | WT_CURSTD_VALUE_SET); @@ -259,7 +259,7 @@ __curstat_search(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, search, cst->btree); + CURSOR_API_CALL(cursor, session, search, NULL); WT_CURSOR_NEEDKEY(cursor); F_CLR(cursor, WT_CURSTD_VALUE_SET | WT_CURSTD_VALUE_SET); @@ -287,18 +287,11 @@ __curstat_close(WT_CURSOR *cursor) WT_SESSION_IMPL *session; cst = (WT_CURSOR_STAT *)cursor; - CURSOR_API_CALL(cursor, session, close, cst->btree); - - if (cst->clear_func) - cst->clear_func(cst->stats_first); + CURSOR_API_CALL(cursor, session, close, NULL); __wt_buf_free(session, &cst->pv); - if (cst->btree != NULL) - WT_TRET(__wt_session_release_btree(session)); - - __wt_free(session, cst->stats); - WT_TRET(__wt_cursor_close(cursor)); + WT_ERR(__wt_cursor_close(cursor)); err: API_END(session); return (ret); @@ -312,63 +305,115 @@ static void __curstat_conn_init( WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst, uint32_t flags) { + WT_CONNECTION_IMPL *conn; + + conn = S2C(session); + + /* + * Fill in the connection statistics, and copy them to the cursor. + * Optionally clear the connection statistics. + */ __wt_conn_stat_init(session, flags); + cst->u.conn_stats = conn->stats; + if (LF_ISSET(WT_STATISTICS_CLEAR)) + __wt_stat_refresh_connection_stats(&conn->stats); - cst->btree = NULL; - cst->notpositioned = 1; - cst->stats_first = (WT_STATS *)&S2C(session)->stats; - cst->stats_count = sizeof(S2C(session)->stats) / sizeof(WT_STATS); - cst->clear_func = LF_ISSET(WT_STATISTICS_CLEAR) ? - __wt_stat_clear_connection_stats : NULL; + cst->stats_first = cst->stats = (WT_STATS *)&cst->u.conn_stats; + cst->stats_count = sizeof(WT_CONNECTION_STATS) / sizeof(WT_STATS); } /* - * __curstat_file_init -- - * Initialize the statistics for a file. + * When returning the statistics for a file URI, we review open handles, and + * aggregate checkpoint handle statistics with the file URI statistics. To + * make that work, we have to pass information to the function reviewing the + * handles, this structure is what we pass. + */ +struct __checkpoint_args { + const char *name; /* Data source handle name */ + WT_DSRC_STATS *stats; /* Stat structure being filled */ + int clear; /* WT_STATISTICS_CLEAR */ +}; + +/* + * __curstat_checkpoint -- + * Aggregate statistics from checkpoint handles. */ static int -__curstat_file_init(WT_SESSION_IMPL *session, - const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) +__curstat_checkpoint(WT_SESSION_IMPL *session, const char *cfg[]) { - WT_BTREE *btree; - - WT_RET(__wt_session_get_btree_ckpt(session, uri, cfg, 0)); - btree = S2BT(session); - WT_RET(__wt_btree_stat_init(session, flags)); + struct __checkpoint_args *args; + WT_DATA_HANDLE *dhandle; + + dhandle = session->dhandle; + args = (struct __checkpoint_args *)cfg[0]; + + /* Aggregate the flagged file's checkpoint handles. */ + if (dhandle->checkpoint != NULL && + strcmp(dhandle->name, args->name) == 0) { + __wt_stat_aggregate_dsrc_stats(&dhandle->stats, args->stats); + if (args->clear) + __wt_stat_refresh_dsrc_stats(&dhandle->stats); + } - cst->btree = btree; - cst->notpositioned = 1; - cst->stats_first = (WT_STATS *)&btree->dhandle->stats; - cst->stats_count = sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); - cst->clear_func = LF_ISSET(WT_STATISTICS_CLEAR) ? - __wt_stat_clear_dsrc_stats : NULL; return (0); } /* - * __curstat_lsm_init -- - * Initialize the statistics for a LSM tree. + * __curstat_file_init -- + * Initialize the statistics for a file. */ static int -__curstat_lsm_init(WT_SESSION_IMPL *session, - const char *uri, WT_CURSOR_STAT *cst, uint32_t flags) +__curstat_file_init(WT_SESSION_IMPL *session, + const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) { + struct __checkpoint_args args; + WT_DATA_HANDLE *dhandle, *saved_dhandle; WT_DECL_RET; - WT_LSM_TREE *lsm_tree; + const char *cfg_arg[] = { NULL, NULL }; - WT_WITH_SCHEMA_LOCK(session, - ret = __wt_lsm_tree_get(session, uri, 0, &lsm_tree)); - WT_RET(ret); + WT_RET(__wt_session_get_btree_ckpt(session, uri, cfg, 0)); + dhandle = session->dhandle; - ret = __wt_lsm_stat_init(session, lsm_tree, cst, flags); - __wt_lsm_tree_release(session, lsm_tree); + /* + * Fill in the data source statistics, and copy them to the cursor. + * Optionally clear the data source statistics. + */ + if ((ret = __wt_btree_stat_init(session, flags)) == 0) { + cst->u.dsrc_stats = dhandle->stats; + if (LF_ISSET(WT_STATISTICS_CLEAR)) + __wt_stat_refresh_dsrc_stats(&dhandle->stats); + + cst->stats_first = cst->stats = (WT_STATS *)&cst->u.dsrc_stats; + cst->stats_count = sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); + } + + /* Release the handle, we're done with it. */ + WT_TRET(__wt_session_release_btree(session)); WT_RET(ret); - cst->btree = NULL; - cst->notpositioned = 1; - cst->clear_func = LF_ISSET(WT_STATISTICS_CLEAR) ? - __wt_stat_clear_dsrc_stats : NULL; - return (0); + /* + * If no checkpoint was specified, review the open handles and aggregate + * the statistics from any checkpoint handles matching this file. + */ + if (dhandle->checkpoint == NULL) { + args.name = dhandle->name; + args.stats = &cst->u.dsrc_stats; + args.clear = LF_ISSET(WT_STATISTICS_CLEAR) ? 1 : 0; + cfg_arg[0] = (char *)&args; + + /* + * We're likely holding the schema lock inside the statistics + * logging thread, not to mention calling __wt_conn_btree_apply + * from there as well. Save/restore the handle. + */ + saved_dhandle = dhandle; + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_conn_btree_apply( + session, 1, __curstat_checkpoint, cfg_arg)); + session->dhandle = saved_dhandle; + } + + return (ret); } /* @@ -379,17 +424,37 @@ int __wt_curstat_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) { + const char *dsrc_uri; + + cst->notpositioned = 1; + if (strcmp(uri, "statistics:") == 0) { __curstat_conn_init(session, cst, flags); return (0); - } else if (WT_PREFIX_MATCH(uri, "statistics:file:")) - return (__curstat_file_init(session, - uri + strlen("statistics:"), cfg, cst, flags)); - else if (WT_PREFIX_MATCH(uri, "statistics:lsm:")) - return (__curstat_lsm_init(session, - uri + strlen("statistics:"), cst, flags)); - else - return (__wt_schema_stat_init(session, uri, cfg, cst, flags)); + } + + dsrc_uri = uri + strlen("statistics:"); + + if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) + return (__wt_curstat_colgroup_init( + session, dsrc_uri, cfg, cst, flags)); + + if (WT_PREFIX_MATCH(dsrc_uri, "file:")) + return (__curstat_file_init( + session, dsrc_uri, cfg, cst, flags)); + + if (WT_PREFIX_MATCH(dsrc_uri, "index:")) + return (__wt_curstat_index_init( + session, dsrc_uri, cfg, cst, flags)); + + if (WT_PREFIX_MATCH(dsrc_uri, "lsm:")) + return (__wt_curstat_lsm_init(session, dsrc_uri, cst, flags)); + + if (WT_PREFIX_MATCH(dsrc_uri, "table:")) + return (__wt_curstat_table_init( + session, dsrc_uri, cfg, cst, flags)); + + return (__wt_bad_object_type(session, uri)); } /* @@ -452,8 +517,7 @@ __wt_curstat_open(WT_SESSION_IMPL *session, WT_ERR(__wt_cursor_init(cursor, uri, NULL, cfg, cursorp)); if (0) { -err: __wt_free(session, cst->stats); - __wt_free(session, cst); +err: __wt_free(session, cst); } return (ret); diff --git a/src/include/cursor.h b/src/include/cursor.h index 885ddb132f3..fd97d0832b3 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -205,7 +205,7 @@ struct __wt_cursor_data_source { WT_COLLATOR *collator; /* Configured collator */ - WT_CURSOR *source; /* Application-owned cursor. */ + WT_CURSOR *source; /* Application-owned cursor */ }; struct __wt_cursor_dump { @@ -228,18 +228,20 @@ struct __wt_cursor_index { struct __wt_cursor_stat { WT_CURSOR iface; - WT_STATS *stats; /* Stats owned by the cursor. */ + WT_STATS *stats; /* Stats owned by the cursor */ WT_STATS *stats_first; /* First stats reference */ int stats_count; /* Count of stats elements */ + union { /* Copies of the statistics */ + WT_DSRC_STATS dsrc_stats; + WT_CONNECTION_STATS conn_stats; + } u; + int notpositioned; /* Cursor not positioned */ int key; /* Current stats key */ uint64_t v; /* Current stats value */ - WT_ITEM pv; /* Current stats value (string) */ - - void (*clear_func)(void *); /* Function to clear stats. */ - WT_BTREE *btree; /* Pinned btree handle. */ + WT_ITEM pv; /* Current stats value (string) */ }; /* diff --git a/src/include/extern.h b/src/include/extern.h index ec53a2bc7f7..8e4711010b8 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -565,6 +565,7 @@ extern int __wt_conn_btree_get(WT_SESSION_IMPL *session, const char *op_cfg[], uint32_t flags); extern int __wt_conn_btree_apply(WT_SESSION_IMPL *session, + int apply_checkpoints, int (*func)(WT_SESSION_IMPL *, const char *[]), const char *cfg[]); @@ -607,8 +608,6 @@ extern int __wt_curbackup_open(WT_SESSION_IMPL *session, WT_CURSOR **cursorp); extern int __wt_backup_list_uri_append(WT_SESSION_IMPL *session, const char *name); -extern int __wt_backup_list_all_append(WT_SESSION_IMPL *session, - const char *cfg[]); extern int __wt_curbulk_init(WT_CURSOR_BULK *cbulk, int bitmap); extern int __wt_curconfig_open(WT_SESSION_IMPL *session, const char *uri, @@ -742,8 +741,8 @@ extern int __wt_lsm_merge( WT_SESSION_IMPL *session, int aggressive); extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); -extern int __wt_lsm_stat_init(WT_SESSION_IMPL *session, - WT_LSM_TREE *lsm_tree, +extern int __wt_curstat_lsm_init(WT_SESSION_IMPL *session, + const char *uri, WT_CURSOR_STAT *cst, uint32_t flags); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session); @@ -768,7 +767,7 @@ extern int __wt_lsm_tree_get(WT_SESSION_IMPL *session, WT_LSM_TREE **treep); extern void __wt_lsm_tree_release(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); -extern int __wt_lsm_tree_switch( WT_SESSION_IMPL *session, +extern int __wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_tree_drop( WT_SESSION_IMPL *session, const char *name, @@ -1112,7 +1111,17 @@ extern int __wt_schema_rename(WT_SESSION_IMPL *session, const char *uri, const char *newuri, const char *cfg[]); -extern int __wt_schema_stat_init(WT_SESSION_IMPL *session, +extern int __wt_curstat_colgroup_init(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR_STAT *cst, + uint32_t flags); +extern int __wt_curstat_index_init(WT_SESSION_IMPL *session, + const char *uri, + const char *cfg[], + WT_CURSOR_STAT *cst, + uint32_t flags); +extern int __wt_curstat_table_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, @@ -1326,10 +1335,11 @@ extern void __wt_ext_scr_free(WT_EXTENSION_API *wt_api, extern void __wt_session_dump_all(WT_SESSION_IMPL *session); extern void __wt_session_dump(WT_SESSION_IMPL *session); extern void __wt_stat_init_dsrc_stats(WT_DSRC_STATS *stats); -extern void __wt_stat_clear_dsrc_stats(void *stats_arg); -extern void __wt_stat_aggregate_dsrc_stats(void *child, void *parent); +extern void __wt_stat_refresh_dsrc_stats(void *stats_arg); +extern void __wt_stat_aggregate_dsrc_stats(const void *child, + const void *parent); extern void __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats); -extern void __wt_stat_clear_connection_stats(void *stats_arg); +extern void __wt_stat_refresh_connection_stats(void *stats_arg); extern int __wt_txnid_cmp(const void *v1, const void *v2); extern void __wt_txn_release_evict_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); diff --git a/src/include/stat.h b/src/include/stat.h index 98aeb01c3a5..ae2896f2607 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -33,10 +33,6 @@ struct __wt_stats { if (S2C(session)->statistics) \ (stats)->fld.v += (value); \ } while (0) -#define WT_STAT_INCRKV(session, stats, key, value) do { \ - if (S2C(session)->statistics) \ - ((WT_STATS *)stats)[(key)].v += (value); \ -} while (0) #define WT_STAT_SET(session, stats, fld, value) do { \ if (S2C(session)->statistics) \ (stats)->fld.v = (uint64_t)(value); \ diff --git a/src/lsm/lsm_merge.c b/src/lsm/lsm_merge.c index 477ed4bd6e3..57c82070fe8 100644 --- a/src/lsm/lsm_merge.c +++ b/src/lsm/lsm_merge.c @@ -186,8 +186,8 @@ __wt_lsm_merge( dest_id = WT_ATOMIC_ADD(lsm_tree->last, 1); WT_VERBOSE_RET(session, lsm, - "Merging chunks %d-%d into %d (%" PRIu64 " records)" - ", generation %d\n", + "Merging chunks %u-%u into %u (%" PRIu64 " records)" + ", generation %" PRIu32 "\n", start_chunk, end_chunk, dest_id, record_count, generation); WT_RET(__wt_calloc_def(session, 1, &chunk)); diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c index bb12f3d3181..f738312b779 100644 --- a/src/lsm/lsm_stat.c +++ b/src/lsm/lsm_stat.c @@ -7,18 +7,42 @@ #include "wt_internal.h" +static int __lsm_stat_init( + WT_SESSION_IMPL *, WT_LSM_TREE *, WT_CURSOR_STAT *, uint32_t); + /* - * __wt_lsm_stat_init -- - * Initialize a LSM statistics structure. + * __wt_curstat_lsm_init -- + * Initialize the statistics for a LSM tree. */ int -__wt_lsm_stat_init(WT_SESSION_IMPL *session, +__wt_curstat_lsm_init(WT_SESSION_IMPL *session, + const char *uri, WT_CURSOR_STAT *cst, uint32_t flags) +{ + WT_DECL_RET; + WT_LSM_TREE *lsm_tree; + + WT_WITH_SCHEMA_LOCK(session, + ret = __wt_lsm_tree_get(session, uri, 0, &lsm_tree)); + WT_RET(ret); + + ret = __lsm_stat_init(session, lsm_tree, cst, flags); + + __wt_lsm_tree_release(session, lsm_tree); + return (ret); +} + +/* + * __lsm_stat_init -- + * Initialize a LSM statistics structure. + */ +static int +__lsm_stat_init(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, WT_CURSOR_STAT *cst, uint32_t flags) { WT_CURSOR *stat_cursor; WT_DECL_ITEM(uribuf); WT_DECL_RET; - WT_DSRC_STATS *child, *stats; + WT_DSRC_STATS *new, *stats; WT_LSM_CHUNK *chunk; u_int i; int locked; @@ -49,16 +73,13 @@ __wt_lsm_stat_init(WT_SESSION_IMPL *session, *p++ = "statistics_fast=true"; /* - * Allocate an aggregated statistics structure, or clear any already - * allocated one. + * Set the cursor to reference the data source statistics; we don't + * initialize it, instead we copy (rather than aggregate), the first + * chunk's statistics, which has the same effect. */ - if ((stats = (WT_DSRC_STATS *)cst->stats) == NULL) { - WT_ERR(__wt_calloc_def(session, 1, &stats)); - __wt_stat_init_dsrc_stats(stats); - cst->stats_first = cst->stats = (WT_STATS *)stats; - cst->stats_count = sizeof(*stats) / sizeof(WT_STATS); - } else - __wt_stat_clear_dsrc_stats(stats); + stats = &cst->u.dsrc_stats; + cst->stats_first = cst->stats = (WT_STATS *)stats; + cst->stats_count = sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); /* Hold the LSM lock so that we can safely walk through the chunks. */ WT_ERR(__wt_readlock(session, lsm_tree->rwlock)); @@ -95,11 +116,19 @@ __wt_lsm_stat_init(WT_SESSION_IMPL *session, * values from the chunk's information, then aggregate into the * top-level. */ - child = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); + new = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); WT_STAT_SET( - session, child, lsm_generation_max, chunk->generation); + session, new, lsm_generation_max, chunk->generation); - __wt_stat_aggregate_dsrc_stats(child, stats); + /* + * We want to aggregate the table's statistics. Get a base set + * of statistics from the first chunk, then aggregate statistics + * from each new chunk. + */ + if (i == 0) + *stats = *new; + else + __wt_stat_aggregate_dsrc_stats(new, stats); WT_ERR(stat_cursor->close(stat_cursor)); if (!F_ISSET(chunk, WT_LSM_CHUNK_BLOOM)) @@ -119,24 +148,24 @@ __wt_lsm_stat_init(WT_SESSION_IMPL *session, * values from the bloom filter's information, then aggregate * into the top-level. */ - child = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); - WT_STAT_SET(session, child, + new = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); + WT_STAT_SET(session, new, bloom_size, (chunk->count * lsm_tree->bloom_bit_count) / 8); - WT_STAT_SET(session, child, + WT_STAT_SET(session, new, bloom_page_evict, - WT_STAT(child, cache_eviction_clean) + - WT_STAT(child, cache_eviction_dirty)); + WT_STAT(new, cache_eviction_clean) + + WT_STAT(new, cache_eviction_dirty)); WT_STAT_SET(session, - child, bloom_page_read, WT_STAT(child, cache_read)); + new, bloom_page_read, WT_STAT(new, cache_read)); - __wt_stat_aggregate_dsrc_stats(child, stats); + __wt_stat_aggregate_dsrc_stats(new, stats); WT_ERR(stat_cursor->close(stat_cursor)); } /* Aggregate, and optionally clear, LSM-level specific information. */ __wt_stat_aggregate_dsrc_stats(&lsm_tree->stats, stats); if (LF_ISSET(WT_STATISTICS_CLEAR)) - __wt_stat_clear_dsrc_stats(&lsm_tree->stats); + __wt_stat_refresh_dsrc_stats(&lsm_tree->stats); err: if (locked) WT_TRET(__wt_rwunlock(session, lsm_tree->rwlock)); diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index b8a6865bc27..79ba533b014 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -611,8 +611,7 @@ __lsm_tree_throttle( * Switch to a new in-memory tree. */ int -__wt_lsm_tree_switch( - WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +__wt_lsm_tree_switch(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) { WT_DECL_RET; WT_LSM_CHUNK *chunk; @@ -626,8 +625,9 @@ __wt_lsm_tree_switch( WT_ERR(__wt_realloc_def(session, &lsm_tree->chunk_alloc, lsm_tree->nchunks + 1, &lsm_tree->chunk)); - WT_VERBOSE_ERR(session, lsm, "Tree switch to: %d, throttle %d", - new_id, (int)lsm_tree->throttle_sleep); + WT_VERBOSE_ERR(session, lsm, + "Tree switch to: %" PRIu32 ", throttle %ld", + new_id, lsm_tree->throttle_sleep); WT_ERR(__wt_calloc_def(session, 1, &chunk)); chunk->id = new_id; diff --git a/src/schema/schema_stat.c b/src/schema/schema_stat.c index 42bfb874703..94c94a02a3e 100644 --- a/src/schema/schema_stat.c +++ b/src/schema/schema_stat.c @@ -8,11 +8,11 @@ #include "wt_internal.h" /* - * __curstat_colgroup_init -- + * __wt_curstat_colgroup_init -- * Initialize the statistics for a column group. */ -static int -__curstat_colgroup_init(WT_SESSION_IMPL *session, +int +__wt_curstat_colgroup_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) { WT_COLGROUP *colgroup; @@ -30,11 +30,11 @@ err: __wt_scr_free(&buf); } /* - * __curstat_index_init -- + * __wt_curstat_index_init -- * Initialize the statistics for an index. */ -static int -__curstat_index_init(WT_SESSION_IMPL *session, +int +__wt_curstat_index_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) { WT_DECL_ITEM(buf); @@ -52,24 +52,20 @@ err: __wt_scr_free(&buf); } /* - * __curstat_table_init -- + * __wt_curstat_table_init -- * Initialize the statistics for a table. */ -static int -__curstat_table_init(WT_SESSION_IMPL *session, +int +__wt_curstat_table_init(WT_SESSION_IMPL *session, const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) { - WT_COLGROUP *colgroup; WT_CURSOR *stat_cursor; WT_DECL_ITEM(buf); WT_DECL_RET; - WT_DSRC_STATS *stats; - WT_INDEX *idx; + WT_DSRC_STATS *new, *stats; WT_TABLE *table; - const char *desc, *name, *pvalue; - uint64_t value; u_int i; - int stat_key; + const char *name; WT_UNUSED(flags); name = uri + strlen("table:"); @@ -77,53 +73,38 @@ __curstat_table_init(WT_SESSION_IMPL *session, WT_RET(__wt_scr_alloc(session, 0, &buf)); WT_ERR(__wt_schema_get_table(session, name, strlen(name), 0, &table)); - /* Clear the statistics we are about to recalculate. */ - if (cst->stats != NULL) { - __wt_stat_clear_dsrc_stats(cst->stats); - stats = (WT_DSRC_STATS *)cst->stats; - } else { - WT_ERR(__wt_calloc_def(session, 1, &stats)); - __wt_stat_init_dsrc_stats(stats); - cst->stats_first = cst->stats = (WT_STATS *)stats; - cst->stats_count = sizeof(*stats) / sizeof(WT_STATS); - } + /* + * Set the cursor to reference the data source statistics; we don't + * initialize it, instead we copy (rather than aggregate), the first + * column's statistics, which has the same effect. + */ + stats = &cst->u.dsrc_stats; + cst->stats_first = cst->stats = (WT_STATS *)stats; + cst->stats_count = sizeof(WT_DSRC_STATS) / sizeof(WT_STATS); /* Process the column groups. */ for (i = 0; i < WT_COLGROUPS(table); i++) { - colgroup = table->cgroups[i]; - WT_ERR(__wt_buf_fmt( - session, buf, "statistics:%s", colgroup->name)); + session, buf, "statistics:%s", table->cgroups[i]->name)); WT_ERR(__wt_curstat_open( session, buf->data, cfg, &stat_cursor)); - - while ((ret = stat_cursor->next(stat_cursor)) == 0) { - WT_ERR(stat_cursor->get_key(stat_cursor, &stat_key)); - WT_ERR(stat_cursor->get_value( - stat_cursor, &desc, &pvalue, &value)); - WT_STAT_INCRKV(session, stats, stat_key, value); - } - WT_ERR_NOTFOUND_OK(ret); + new = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); + if (i == 0) + *stats = *new; + else + __wt_stat_aggregate_dsrc_stats(new, stats); WT_ERR(stat_cursor->close(stat_cursor)); } /* Process the indices. */ WT_ERR(__wt_schema_open_indices(session, table)); for (i = 0; i < table->nindices; i++) { - idx = table->indices[i]; - WT_ERR(__wt_buf_fmt( - session, buf, "statistics:%s", idx->name)); + session, buf, "statistics:%s", table->indices[i]->name)); WT_ERR(__wt_curstat_open( session, buf->data, cfg, &stat_cursor)); - - while ((ret = stat_cursor->next(stat_cursor)) == 0) { - WT_ERR(stat_cursor->get_key(stat_cursor, &stat_key)); - WT_ERR(stat_cursor->get_value( - stat_cursor, &desc, &pvalue, &value)); - WT_STAT_INCRKV(session, stats, stat_key, value); - } - WT_ERR_NOTFOUND_OK(ret); + new = (WT_DSRC_STATS *)WT_CURSOR_STATS(stat_cursor); + __wt_stat_aggregate_dsrc_stats(new, stats); WT_ERR(stat_cursor->close(stat_cursor)); } @@ -131,28 +112,3 @@ err: __wt_scr_free(&buf); __wt_schema_release_table(session, table); return (ret); } - -/* - * __wt_schema_stat_init -- - * Configure a statistics cursor for a schema-level object. - */ -int -__wt_schema_stat_init(WT_SESSION_IMPL *session, - const char *uri, const char *cfg[], WT_CURSOR_STAT *cst, uint32_t flags) -{ - const char *dsrc_uri; - - dsrc_uri = uri + strlen("statistics:"); - - if (WT_PREFIX_MATCH(dsrc_uri, "colgroup:")) - return (__curstat_colgroup_init( - session, dsrc_uri, cfg, cst, flags)); - else if (WT_PREFIX_MATCH(dsrc_uri, "index:")) - return (__curstat_index_init( - session, dsrc_uri, cfg, cst, flags)); - else if (WT_PREFIX_MATCH(dsrc_uri, "table:")) - return (__curstat_table_init( - session, dsrc_uri, cfg, cst, flags)); - - return (__wt_bad_object_type(session, uri)); -} diff --git a/src/session/session_api.c b/src/session/session_api.c index de58823eab9..0590ed66f60 100644 --- a/src/session/session_api.c +++ b/src/session/session_api.c @@ -831,7 +831,7 @@ __wt_open_session(WT_CONNECTION_IMPL *conn, int internal, break; if (i == conn->session_size) WT_ERR_MSG(session, WT_ERROR, - "only configured to support %d thread contexts", + "only configured to support %" PRIu32 " thread contexts", conn->session_size); /* diff --git a/src/support/stat.c b/src/support/stat.c index 03a75122b51..3c53db49f58 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -5,6 +5,9 @@ void __wt_stat_init_dsrc_stats(WT_DSRC_STATS *stats) { + /* Clear, so can also be called for reinitialization. */ + memset(stats, 0, sizeof(*stats)); + stats->block_alloc.desc = "blocks allocated"; stats->block_allocsize.desc = "block manager file allocation unit size"; @@ -117,7 +120,7 @@ __wt_stat_init_dsrc_stats(WT_DSRC_STATS *stats) } void -__wt_stat_clear_dsrc_stats(void *stats_arg) +__wt_stat_refresh_dsrc_stats(void *stats_arg) { WT_DSRC_STATS *stats; @@ -209,7 +212,7 @@ __wt_stat_clear_dsrc_stats(void *stats_arg) } void -__wt_stat_aggregate_dsrc_stats(void *child, void *parent) +__wt_stat_aggregate_dsrc_stats(const void *child, const void *parent) { WT_DSRC_STATS *c, *p; @@ -298,6 +301,9 @@ __wt_stat_aggregate_dsrc_stats(void *child, void *parent) void __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats) { + /* Clear, so can also be called for reinitialization. */ + memset(stats, 0, sizeof(*stats)); + stats->block_byte_map_read.desc = "mapped bytes read by the block manager"; stats->block_byte_read.desc = "bytes read by the block manager"; @@ -398,7 +404,7 @@ __wt_stat_init_connection_stats(WT_CONNECTION_STATS *stats) } void -__wt_stat_clear_connection_stats(void *stats_arg) +__wt_stat_refresh_connection_stats(void *stats_arg) { WT_CONNECTION_STATS *stats; diff --git a/src/txn/txn_ckpt.c b/src/txn/txn_ckpt.c index c9b9b5f0a75..8c64290e63c 100644 --- a/src/txn/txn_ckpt.c +++ b/src/txn/txn_ckpt.c @@ -128,7 +128,7 @@ __checkpoint_apply(WT_SESSION_IMPL *session, const char *cfg[], } WT_ERR(ckpt_closed ? __wt_meta_btree_apply(session, op, cfg) : - __wt_conn_btree_apply(session, op, cfg)); + __wt_conn_btree_apply(session, 0, op, cfg)); } err: __wt_scr_free(&tmp); diff --git a/tools/statlog.py b/tools/statlog.py index 7706c560dc5..c1bea5aab55 100644 --- a/tools/statlog.py +++ b/tools/statlog.py @@ -73,8 +73,12 @@ no_scale_per_second_list = [ ] # no scale-per-second list section: END +reportno = 0 + # Plot a set of entries for a title. -def plot(title, values, num): +def plot(title, values): + global reportno + # Ignore entries where the value never changes. skip = True t0, v0 = values[0] @@ -83,10 +87,12 @@ def plot(title, values, num): skip = False break if skip: - print '\tskipping ' + title + print 'skipping: ' + title return - print 'building ' + title + print 'building: ' + title + reportno = reportno + 1 + num = "%03d" % reportno ylabel = 'Value' if title.split(' ', 1)[1] in no_scale_per_second_list: @@ -148,5 +154,5 @@ for line in fileinput.input(sys.argv[1:]): d[desc].append((month + " " + day + " " + time, v)) # Plot each entry in the dictionary. -for rno, items in enumerate(sorted(d.iteritems()), 1): - plot(items[0], items[1], "%03d" % rno) +for items in sorted(d.iteritems()): + plot(items[0], items[1]) |