diff options
author | Michael Cahill <mjc@wiredtiger.com> | 2012-10-08 03:33:42 -0700 |
---|---|---|
committer | Michael Cahill <mjc@wiredtiger.com> | 2012-10-08 03:33:42 -0700 |
commit | 2bc7c975ab1641e6d2382b35f9ba4c8ac5bcbef5 (patch) | |
tree | 493151a9835be8dea1e35876d3188f32dfd502a2 | |
parent | af357b2111bf7c8c11ff9156d6754dfc8dfbc5b4 (diff) | |
parent | 600ec6ff8f2fceb02a8bccb822dc2a977900300b (diff) | |
download | mongo-2bc7c975ab1641e6d2382b35f9ba4c8ac5bcbef5.tar.gz |
Merge pull request #354 from wiredtiger/lsm-stats
Add a new statistics cursor type "statistics:lsm".
-rw-r--r-- | dist/filelist | 1 | ||||
-rw-r--r-- | dist/stat.py | 18 | ||||
-rw-r--r-- | dist/stat_data.py | 10 | ||||
-rw-r--r-- | examples/c/ex_stat.c | 22 | ||||
-rw-r--r-- | examples/c/ex_test_perf.c | 124 | ||||
-rw-r--r-- | src/cursor/cur_stat.c | 25 | ||||
-rw-r--r-- | src/include/extern.h | 4 | ||||
-rw-r--r-- | src/include/lsm.h | 2 | ||||
-rw-r--r-- | src/include/stat.h | 9 | ||||
-rw-r--r-- | src/include/wiredtiger.in | 14 | ||||
-rw-r--r-- | src/include/wt_internal.h | 2 | ||||
-rw-r--r-- | src/lsm/lsm_cursor.c | 8 | ||||
-rw-r--r-- | src/lsm/lsm_stat.c | 24 | ||||
-rw-r--r-- | src/lsm/lsm_tree.c | 3 | ||||
-rw-r--r-- | src/support/stat.c | 28 |
15 files changed, 286 insertions, 8 deletions
diff --git a/dist/filelist b/dist/filelist index e2ed4911683..aa7e9f709ae 100644 --- a/dist/filelist +++ b/dist/filelist @@ -70,6 +70,7 @@ src/lsm/lsm_cursor.c src/lsm/lsm_dsrc.c src/lsm/lsm_merge.c src/lsm/lsm_meta.c +src/lsm/lsm_stat.c src/lsm/lsm_tree.c src/lsm/lsm_worker.c src/meta/meta_api.c diff --git a/dist/stat.py b/dist/stat.py index 0af0e65c04b..bbaaa08b06c 100644 --- a/dist/stat.py +++ b/dist/stat.py @@ -6,7 +6,7 @@ from dist import compare_srcfile from dist import source_paths_list # Read the source files. -from stat_data import btree_stats, connection_stats +from stat_data import btree_stats, connection_stats, lsm_stats # print_struct -- # Print the structures for the stat.h file. @@ -37,6 +37,7 @@ for line in open('../src/include/stat.h', 'r'): skip = 1 print_struct('BTREE', 'btree', btree_stats) print_struct('CONNECTION', 'connection', connection_stats) + print_struct('LSM', 'lsm', lsm_stats) f.close() compare_srcfile(tmp_file, '../src/include/stat.h') @@ -75,6 +76,20 @@ def print_define(): max(1, 6 - int((len('WT_STAT_') + len(l.name)) / 8)) + str(v) + '\n') f.write('/*! @} */\n') + f.write(''' +/*! + * @} + * @name Statistics for lsm objects + * @anchor statistics_lsm + * @{ + */ +''') + for v, l in enumerate(sorted(lsm_stats, key=attrgetter('desc'))): + f.write('/*! %s */\n' % '\n * '.join(textwrap.wrap(l.desc, 70))) + f.write('#define\tWT_STAT_' + l.name + "\t" * + max(1, 6 - int((len('WT_STAT_') + len(l.name)) / 8)) + + str(v) + '\n') + f.write('/*! @} */\n') # Update the #defines in the wiredtiger.in file. tmp_file = '__tmp' @@ -141,5 +156,6 @@ f.write('#include "wt_internal.h"\n') print_func('btree', btree_stats) print_func('connection', connection_stats) +print_func('lsm', lsm_stats) f.close() compare_srcfile(tmp_file, '../src/support/stat.c') diff --git a/dist/stat_data.py b/dist/stat_data.py index a6111ebddd6..99115d84a0c 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -93,3 +93,13 @@ btree_stats = [ Stat('rec_written', 'reconcile: pages written'), Stat('update_conflict', 'update conflicts'), ] + +########################################## +# LSM statistics +########################################## +lsm_stats = [ + Stat('bloom_hits', 'Number of successful bloom filter lookups'), + Stat('bloom_misses', 'Number of successful bloom filter false positives'), + Stat('bloom_skips', 'Number of unsuccessful bloom filter lookups'), +] + diff --git a/examples/c/ex_stat.c b/examples/c/ex_stat.c index 77d4581680d..9399f45f775 100644 --- a/examples/c/ex_stat.c +++ b/examples/c/ex_stat.c @@ -38,6 +38,7 @@ int print_cursor(WT_CURSOR *); int print_database_stats(WT_SESSION *); int print_file_stats(WT_SESSION *); +int print_lsm_stats(WT_SESSION *); int print_overflow_pages(WT_SESSION *); const char *home = "WT_TEST"; @@ -90,6 +91,25 @@ print_file_stats(WT_SESSION *session) } int +print_lsm_stats(WT_SESSION *session) +{ + WT_CURSOR *cursor; + int ret; + + /* Create an LSM tree, and add some content. */ + ret = session->create(session, + "lsm:access", "key_format=S,value_format=S"); + + /*! [statistics lsm function] */ + if ((ret = session->open_cursor(session, + "statistics:lsm:access", NULL, NULL, &cursor)) != 0) + return (ret); + + return (print_cursor(cursor)); + /*! [statistics lsm function] */ +} + +int print_overflow_pages(WT_SESSION *session) { /*! [statistics retrieve by key] */ @@ -127,6 +147,8 @@ main(void) ret = print_file_stats(session); + ret = print_lsm_stats(session); + ret = print_overflow_pages(session); return (conn->close(conn, NULL) == 0 ? ret : EXIT_FAILURE); diff --git a/examples/c/ex_test_perf.c b/examples/c/ex_test_perf.c index 13396d1da9b..3e701588b56 100644 --- a/examples/c/ex_test_perf.c +++ b/examples/c/ex_test_perf.c @@ -50,8 +50,10 @@ typedef struct { uint32_t key_sz; uint32_t report_interval; uint32_t read_time; + uint32_t elapsed_time; uint32_t read_threads; /* Number of read threads. */ uint32_t verbose; + uint32_t stat_thread; /* Whether to create a stat thread. */ WT_CONNECTION *conn; } CONFIG; @@ -60,6 +62,7 @@ int execute_reads(CONFIG *); int populate(CONFIG *); void print_config(CONFIG *); void *read_thread(void *); +void *stat_worker(void *); void usage(void); /* Default values - these are tiny, we want the basic run to be fast. */ @@ -75,8 +78,10 @@ CONFIG default_cfg = { 20, /* key_sz */ 2, /* report_interval */ 2, /* read_time */ + 0, /* elapsed_time */ 2, /* read_threads */ 0, /* verbose */ + 0, /* stat_thread */ NULL }; /* Small config values - these are small. */ @@ -93,8 +98,10 @@ CONFIG small_cfg = { 20, /* key_sz */ 10, /* report_interval */ 20, /* read_time */ + 0, /* elapsed_time */ 8, /* read_threads */ 0, /* verbose */ + 0, /* stat_thread */ NULL }; /* Default values - these are small, we want the basic run to be fast. */ @@ -111,8 +118,10 @@ CONFIG med_cfg = { 20, /* key_sz */ 20, /* report_interval */ 100, /* read_time */ + 0, /* elapsed_time */ 16, /* read_threads */ 0, /* verbose */ + 0, /* stat_thread */ NULL }; /* Default values - these are small, we want the basic run to be fast. */ @@ -129,8 +138,10 @@ CONFIG large_cfg = { 20, /* key_sz */ 20, /* report_interval */ 600, /* read_time */ + 0, /* elapsed_time */ 16, /* read_threads */ 0, /* verbose */ + 0, /* stat_thread */ NULL }; @@ -178,6 +189,90 @@ read_thread(void *arg) return (arg); } +void * +stat_worker(void *arg) +{ + CONFIG *cfg; + WT_CONNECTION *conn; + WT_SESSION *session; + WT_CURSOR *cursor; + FILE *logf; + const char *desc, *pvalue; + char *fname, *lsm_uri; + int offset, ret; + uint64_t value; + + cfg = (CONFIG *)arg; + conn = cfg->conn; + fname = lsm_uri = NULL; + logf = NULL; + + if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0) { + fprintf(stderr, + "open_session failed in read thread: %d\n", ret); + return (NULL); + } + + if (strncmp(cfg->uri, "lsm:", strlen("lsm:")) == 0) { + lsm_uri = calloc(strlen(cfg->uri) + strlen("statistics:"), 1); + if (lsm_uri == NULL) { + fprintf(stderr, "No memory in stat thread.\n"); + goto err; + } + sprintf(lsm_uri, "statistics:%s", cfg->uri); + } + /* Open the file for logging statistics information. */ + if ((fname = calloc(strlen(cfg->home) + + strlen(cfg->uri) + strlen(".stat") + 1, 1)) == NULL) { + fprintf(stderr, "No memory in stat thread\n"); + goto err; + } + for (offset = 0; + cfg->uri[offset] != 0 && cfg->uri[offset] != ':'; + offset++) {} + if (cfg->uri[offset] == 0) + offset = 0; + else + ++offset; + sprintf(fname, "%s/%s.stat", cfg->home, cfg->uri + offset); + if ((logf = fopen(fname, "w")) == NULL) { + fprintf(stderr, "Statistics failed to open log file.\n"); + goto err; + } + + while (running) { + sleep(cfg->report_interval); + /* Generic header. */ + fprintf(logf, "=========================================\n"); + fprintf(logf, "reads completed: %" PRIu64", elapsed time: ~%d\n", + nops, cfg->elapsed_time); + /* Report LSM tree stats, if using LSM. */ + if (lsm_uri != NULL) { + if ((ret = session->open_cursor(session, lsm_uri, + NULL, NULL, &cursor)) != 0) { + fprintf(stderr, + "open_cursor LSM statistics: %d\n", ret); + goto err; + } + while ( + (ret = cursor->next(cursor)) == 0 && + (ret = cursor->get_value( + cursor, &desc, &pvalue, &value)) == 0) + fprintf(logf, "stat:lsm:%s=%s\n", desc, pvalue); + cursor->close(cursor); + } + fflush(logf); + } +err: session->close(session, NULL); + if (lsm_uri != NULL) + free(lsm_uri); + if (fname != NULL) + free(fname); + if (logf != NULL) + fclose(logf); + return (arg); +} + int populate(CONFIG *cfg) { WT_CONNECTION *conn; @@ -249,9 +344,10 @@ int main(int argc, char **argv) CONFIG cfg; WT_CONNECTION *conn; const char *user_cconfig, *user_tconfig; - const char *opts = "C:R:T:d:eh:i:k:r:s:u:v:SML"; + const char *opts = "C:R:T:d:eh:i:k:lr:s:u:v:SML"; char *cc_buf, *tc_buf; - int ch, ret; + int ch, ret, stat_created; + pthread_t stat; uint64_t req_len; /* Setup the default configuration values. */ @@ -259,6 +355,7 @@ int main(int argc, char **argv) cc_buf = tc_buf = NULL; user_cconfig = user_tconfig = NULL; conn = NULL; + stat_created = 0; /* * First parse different config structures - other options override @@ -299,6 +396,9 @@ int main(int argc, char **argv) case 'k': cfg.key_sz = atoi(optarg); break; + case 'l': + cfg.stat_thread = 1; + break; case 'r': cfg.read_time = atoi(optarg); break; @@ -382,6 +482,15 @@ int main(int argc, char **argv) if (cfg.create != 0 && (ret = populate(&cfg)) != 0) goto err; + if (cfg.stat_thread) { + if ((ret = pthread_create( + &stat, NULL, stat_worker, &cfg)) != 0) { + fprintf(stderr, "Error creating statistics thread.\n"); + goto err; + } + stat_created = 1; + } + if (cfg.read_time != 0 && cfg.read_threads != 0) if ((ret = execute_reads(&cfg)) != 0) goto err; @@ -391,7 +500,9 @@ int main(int argc, char **argv) printf("Executed %" PRIu64 " read operations\n", nops); /* Cleanup. */ -err: if (conn != NULL && (ret = conn->close(conn, NULL)) != 0) +err: if (stat_created != 0 && (ret = pthread_join(stat, NULL)) != 0) + fprintf(stderr, "Error joining stat thread: %d.\n", ret); + if (conn != NULL && (ret = conn->close(conn, NULL)) != 0) fprintf(stderr, "Error connecting to %s: %s\n", cfg.home, wiredtiger_strerror(ret)); if (cc_buf != NULL) @@ -406,7 +517,7 @@ int execute_reads(CONFIG *cfg) { pthread_t *read_threads; int ret; - uint32_t i, slept; + uint32_t i; uint64_t last_ops; if (cfg->verbose > 0) @@ -430,8 +541,9 @@ int execute_reads(CONFIG *cfg) if (cfg->report_interval > cfg->read_time) cfg->report_interval = cfg->read_time; - for (slept = 0, last_ops = 0; slept < cfg->read_time; - slept += cfg->report_interval) { + for (cfg->elapsed_time = 0, last_ops = 0; + cfg->elapsed_time < cfg->read_time; + cfg->elapsed_time += cfg->report_interval) { sleep(cfg->report_interval); if (cfg->verbose > 0) { printf("%" PRIu64 " ops in %d secs\n", diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index f24e8bbe515..55095eea484 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -337,6 +337,28 @@ __curstat_file_init(WT_SESSION_IMPL *session, } /* + * __curstat_lsm_init -- + * Initialize the statistics for a LSM tree. + */ +static int +__curstat_lsm_init(WT_SESSION_IMPL *session, + const char *uri, WT_CURSOR_STAT *cst, int statistics_clear) +{ + WT_LSM_TREE *lsm_tree; + + WT_RET(__wt_lsm_tree_get(session, uri, &lsm_tree)); + + WT_RET(__wt_lsm_stat_init(session, lsm_tree)); + + cst->btree = NULL; + cst->notpositioned = 1; + cst->stats_first = (WT_STATS *)lsm_tree->stats; + cst->stats_count = sizeof(WT_LSM_STATS) / sizeof(WT_STATS); + cst->clear_func = statistics_clear ? __wt_stat_clear_lsm_stats : NULL; + return (0); +} + +/* * __wt_curstat_open -- * WT_SESSION->open_cursor method for the statistics cursor type. */ @@ -401,6 +423,9 @@ __wt_curstat_open(WT_SESSION_IMPL *session, else if (WT_PREFIX_MATCH(uri, "statistics:file:")) WT_ERR(__curstat_file_init(session, uri + strlen("statistics:"), cst, statistics_clear)); + else if (WT_PREFIX_MATCH(uri, "statistics:lsm:")) + WT_ERR(__curstat_lsm_init(session, + uri + strlen("statistics:"), cst, statistics_clear)); else WT_ERR(__wt_bad_object_type(session, uri)); diff --git a/src/include/extern.h b/src/include/extern.h index 86f16de8c32..a3aeeca65ff 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -689,6 +689,7 @@ extern int __wt_lsm_merge_update_tree(WT_SESSION_IMPL *session, extern int __wt_lsm_merge(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_meta_read(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_meta_write(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); +extern int __wt_lsm_stat_init(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree); extern int __wt_lsm_tree_close_all(WT_SESSION_IMPL *session); extern int __wt_lsm_tree_bloom_name( WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree, @@ -1177,6 +1178,9 @@ extern void __wt_stat_clear_btree_stats(WT_STATS *stats_arg); extern int __wt_stat_alloc_connection_stats(WT_SESSION_IMPL *session, WT_CONNECTION_STATS **statsp); extern void __wt_stat_clear_connection_stats(WT_STATS *stats_arg); +extern int __wt_stat_alloc_lsm_stats(WT_SESSION_IMPL *session, + WT_LSM_STATS **statsp); +extern void __wt_stat_clear_lsm_stats(WT_STATS *stats_arg); extern int __wt_txnid_cmp(const void *v1, const void *v2); extern void __wt_txn_release_snapshot(WT_SESSION_IMPL *session); extern void __wt_txn_get_snapshot(WT_SESSION_IMPL *session, wt_txnid_t max_id); diff --git a/src/include/lsm.h b/src/include/lsm.h index 0dc7759fde7..d54ed008020 100644 --- a/src/include/lsm.h +++ b/src/include/lsm.h @@ -59,6 +59,8 @@ struct __wt_lsm_tree { WT_RWLOCK *rwlock; TAILQ_ENTRY(__wt_lsm_tree) q; + WT_LSM_STATS *stats; /* LSM statistics */ + WT_SPINLOCK lock; uint64_t dsk_gen; uint32_t *memsizep; diff --git a/src/include/stat.h b/src/include/stat.h index c7a7ed13983..5ae315f0a99 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -46,6 +46,15 @@ struct __wt_stats { /* Statistics section: BEGIN */ /* + * Statistics entries for LSM handle. + */ +struct __wt_lsm_stats { + WT_STATS bloom_misses; + WT_STATS bloom_hits; + WT_STATS bloom_skips; +}; + +/* * Statistics entries for BTREE handle. */ struct __wt_btree_stats { diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index a15e7a5001d..f223b3fe46b 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -1778,6 +1778,20 @@ extern int wiredtiger_extension_init(WT_SESSION *session, /*! write generation conflicts */ #define WT_STAT_file_write_conflicts 43 /*! @} */ + +/*! + * @} + * @name Statistics for lsm objects + * @anchor statistics_lsm + * @{ + */ +/*! Number of successful bloom filter false positives */ +#define WT_STAT_bloom_misses 0 +/*! Number of successful bloom filter lookups */ +#define WT_STAT_bloom_hits 1 +/*! Number of unsuccessful bloom filter lookups */ +#define WT_STAT_bloom_skips 2 +/*! @} */ /* * Statistics section: END * DO NOT EDIT: automatically built by dist/api_stat.py. diff --git a/src/include/wt_internal.h b/src/include/wt_internal.h index 890ba369c7c..bcb5754bcbc 100644 --- a/src/include/wt_internal.h +++ b/src/include/wt_internal.h @@ -133,6 +133,8 @@ struct __wt_lsm_chunk; typedef struct __wt_lsm_chunk WT_LSM_CHUNK; struct __wt_lsm_data_source; typedef struct __wt_lsm_data_source WT_LSM_DATA_SOURCE; +struct __wt_lsm_stats; + typedef struct __wt_lsm_stats WT_LSM_STATS; struct __wt_lsm_tree; typedef struct __wt_lsm_tree WT_LSM_TREE; struct __wt_lsm_worker_cookie; diff --git a/src/lsm/lsm_cursor.c b/src/lsm/lsm_cursor.c index 9b9274102f6..bbe3ded5603 100644 --- a/src/lsm/lsm_cursor.c +++ b/src/lsm/lsm_cursor.c @@ -513,8 +513,11 @@ __clsm_search(WT_CURSOR *cursor) /* If there is a Bloom filter, see if we can skip the read. */ if ((bloom = clsm->blooms[i]) != NULL) { ret = __wt_bloom_get(bloom, &cursor->key); - if (ret == WT_NOTFOUND) + if (ret == WT_NOTFOUND) { + WT_STAT_INCR(clsm->lsm_tree->stats, bloom_skips); continue; + } else if (ret == 0) + WT_STAT_INCR(clsm->lsm_tree->stats, bloom_hits); WT_RET(ret); } c->set_key(c, &cursor->key); @@ -527,6 +530,9 @@ __clsm_search(WT_CURSOR *cursor) goto done; } else if (ret != WT_NOTFOUND) goto err; + else if (bloom != NULL) { + WT_STAT_INCR(clsm->lsm_tree->stats, bloom_misses); + } } ret = WT_NOTFOUND; diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c new file mode 100644 index 00000000000..61de4d6bf9f --- /dev/null +++ b/src/lsm/lsm_stat.c @@ -0,0 +1,24 @@ +/*- + * Copyright (c) 2008-2012 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __wt_lsm_stat_init -- + * Initialize a LSM statistics structure. + */ +int +__wt_lsm_stat_init(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) +{ + WT_UNUSED(session); + WT_UNUSED(lsm_tree); + /* + * TODO: Make a copy of the stat fields so the stat cursor gets a + * consistent view? If so should the copy belong to the stat cursor? + */ + return (0); +} diff --git a/src/lsm/lsm_tree.c b/src/lsm/lsm_tree.c index 5b0c82563d7..5c616e9357f 100644 --- a/src/lsm/lsm_tree.c +++ b/src/lsm/lsm_tree.c @@ -46,6 +46,7 @@ __lsm_tree_discard(WT_SESSION_IMPL *session, WT_LSM_TREE *lsm_tree) __wt_free(session, chunk); } __wt_free(session, lsm_tree->old_chunks); + __wt_free(session, lsm_tree->stats); __wt_free(session, lsm_tree); } @@ -331,6 +332,8 @@ __lsm_tree_open( __wt_spin_init(session, &lsm_tree->lock); WT_ERR(__wt_strdup(session, uri, &lsm_tree->name)); lsm_tree->filename = lsm_tree->name + strlen("lsm:"); + WT_ERR(__wt_stat_alloc_lsm_stats(session, &lsm_tree->stats)); + WT_ERR(__wt_lsm_meta_read(session, lsm_tree)); /* diff --git a/src/support/stat.c b/src/support/stat.c index 6a4c35a0c4e..426e0ef7104 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -182,3 +182,31 @@ __wt_stat_clear_connection_stats(WT_STATS *stats_arg) stats->txn_fail_cache.v = 0; stats->txn_rollback.v = 0; } + +int +__wt_stat_alloc_lsm_stats(WT_SESSION_IMPL *session, WT_LSM_STATS **statsp) +{ + WT_LSM_STATS *stats; + + WT_RET(__wt_calloc_def(session, 1, &stats)); + + stats->bloom_hits.desc = "Number of successful bloom filter lookups"; + stats->bloom_misses.desc = + "Number of successful bloom filter false positives"; + stats->bloom_skips.desc = + "Number of unsuccessful bloom filter lookups"; + + *statsp = stats; + return (0); +} + +void +__wt_stat_clear_lsm_stats(WT_STATS *stats_arg) +{ + WT_LSM_STATS *stats; + + stats = (WT_LSM_STATS *)stats_arg; + stats->bloom_hits.v = 0; + stats->bloom_misses.v = 0; + stats->bloom_skips.v = 0; +} |