diff options
Diffstat (limited to 'src/third_party/wiredtiger/bench')
11 files changed, 177 insertions, 93 deletions
diff --git a/src/third_party/wiredtiger/bench/wtperf/config.c b/src/third_party/wiredtiger/bench/wtperf/config.c index 0dc38287155..a2960902ec1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/config.c +++ b/src/third_party/wiredtiger/bench/wtperf/config.c @@ -94,31 +94,42 @@ config_unescape(char *orig) } /* - * config_assign -- - * Assign the src config to the dest, any storage allocated in dest is - * freed as a result. + * config_copy -- + * CONFIG structure initialization, based on a source configuration. */ int -config_assign(CONFIG *dest, const CONFIG *src) +config_copy(CONFIG *dest, const CONFIG *src) { CONFIG_QUEUE_ENTRY *conf_line, *tmp_line; size_t i; char *newstr, **pstr; - config_free(dest); memcpy(dest, src, sizeof(CONFIG)); + if (src->home != NULL) + dest->home = dstrdup(src->home); + if (src->monitor_dir != NULL) + dest->monitor_dir = dstrdup(src->monitor_dir); + if (src->partial_config != NULL) + dest->partial_config = dstrdup(src->partial_config); + if (src->reopen_config != NULL) + dest->reopen_config = dstrdup(src->reopen_config); + if (src->base_uri != NULL) + dest->base_uri = dstrdup(src->base_uri); + if (src->uris != NULL) { dest->uris = dcalloc(src->table_count, sizeof(char *)); for (i = 0; i < src->table_count; i++) dest->uris[i] = dstrdup(src->uris[i]); } + + if (src->async_config != NULL) + dest->async_config = dstrdup(src->async_config); + dest->ckptthreads = NULL; dest->popthreads = NULL; dest->workers = NULL; - if (src->base_uri != NULL) - dest->base_uri = dstrdup(src->base_uri); if (src->workload != NULL) { dest->workload = dcalloc(WORKLOAD_MAX, sizeof(WORKLOAD)); memcpy(dest->workload, @@ -159,6 +170,28 @@ config_free(CONFIG *cfg) size_t i; char **pstr; + free(cfg->home); + free(cfg->monitor_dir); + free(cfg->partial_config); + free(cfg->reopen_config); + free(cfg->base_uri); + + if (cfg->uris != NULL) { + for (i = 0; i < cfg->table_count; i++) + free(cfg->uris[i]); + free(cfg->uris); + } + + free(cfg->async_config); + + free(cfg->ckptthreads); + free(cfg->popthreads); + + free(cfg->workers); + free(cfg->workload); + + cleanup_truncate_config(cfg); + while (!TAILQ_EMPTY(&cfg->config_head)) { config_line = TAILQ_FIRST(&cfg->config_head); TAILQ_REMOVE(&cfg->config_head, config_line, c); @@ -174,20 +207,6 @@ config_free(CONFIG *cfg) free(*pstr); *pstr = NULL; } - if (cfg->uris != NULL) { - for (i = 0; i < cfg->table_count; i++) - free(cfg->uris[i]); - free(cfg->uris); - } - - cleanup_truncate_config(cfg); - free(cfg->base_uri); - free(cfg->ckptthreads); - free(cfg->partial_config); - free(cfg->popthreads); - free(cfg->reopen_config); - free(cfg->workers); - free(cfg->workload); } /* @@ -390,9 +409,9 @@ config_threads(CONFIG *cfg, const char *config, size_t len) return (0); err: if (group != NULL) - (void)group->close(group); + testutil_check(group->close(group)); if (scan != NULL) - (void)scan->close(scan); + testutil_check(scan->close(scan)); fprintf(stderr, "invalid thread configuration or scan error: %.*s\n", diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf index 4ca956d2afe..06745bf7cca 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-50r50u.wtperf @@ -1,19 +1,22 @@ -# wtperf options file: simulate riak and its test3 configuration -# The configuration for the connection and table are from riak and the -# specification of the data (count, size, threads) is from basho_bench. +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. # This test assumes that its correlating populate already completed and exists. # -#conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,statistics=(fast,clear),statistics_log=(wait=60)" -conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,log=(enabled),statistics=(fast),statistics_log=(wait=30)" -checkpoint_interval=60 +# Set cache to half of memory of AWS perf instance. Enable logging and +# checkpoints. Collect wiredtiger stats for ftdc. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" create=false compression="snappy" sess_config="isolation=snapshot" +table_count=2 key_sz=40 -value_sz=1000 +value_sz=120 max_latency=2000 pareto=20 report_interval=10 run_time=7200 sample_interval=10 +sample_rate=1 threads=((count=10,reads=1),(count=10,updates=1)) +warmup=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf index 50488f72ead..77edbfb4941 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-80r20u.wtperf @@ -1,18 +1,21 @@ -# wtperf options file: simulate riak and its test2 configuration -# The configuration for the connection and table are from riak and the -# specification of the data (count, size, threads) is from basho_bench. +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. # This test assumes that its correlating populate already completed and exists. # -#conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,statistics=(fast,clear),statistics_log=(wait=60)" -conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,log=(enabled),statistics=(fast),statistics_log=(wait=30)" -checkpoint_interval=60 +# Set cache to half of memory of AWS perf instance. Enable logging and +# checkpoints. Collect wiredtiger stats for ftdc. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" create=false compression="snappy" sess_config="isolation=snapshot +table_count=2 key_sz=40 -value_sz=1000 +value_sz=120 max_latency=2000 report_interval=10 run_time=7200 sample_interval=10 +sample_rate=1 threads=((count=20,reads=4,updates=1)) +warmup=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf index 9f34b8a8b1f..f9aed094aa1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-populate.wtperf @@ -1,20 +1,26 @@ -# wtperf options file: simulate riak and its test1 configuration -# The configuration for the connection and table are from riak and the -# specification of the data (count, size, threads) is from basho_bench. -# This task is the populate phase for its family of tests. +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. +# This test assumes that its correlating populate already completed and exists. # -#conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,statistics=(fast,clear),statistics_log=(wait=60)" -conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,log=(enabled),statistics=(fast),statistics_log=(wait=30)" -checkpoint_interval=60 +# Set cache to half of memory. This value is computed from the AWS instance +# where Jenkins perf runs which has 32G. Enable logging and checkpoints. +# Collect wiredtiger stats for ftdc. +# +# This generates about 80 Gb of uncompressed data. But it should compress +# well and be small on disk. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" compact=true compression="snappy" sess_config="isolation=snapshot" -table_config="internal_page_max=128K,type=file,leaf_page_max=16K" +table_config="internal_page_max=16K,type=file,leaf_page_max=16K,memory_page_max=10M,split_pct=90" +table_count=2 icount=500000000 key_sz=40 -value_sz=1000 +value_sz=120 max_latency=2000 populate_threads=20 report_interval=10 random_value=true sample_interval=10 +sample_rate=1 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf index 965306262a3..2c9540ff589 100644 --- a/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf +++ b/src/third_party/wiredtiger/bench/wtperf/runners/500m-btree-rdonly.wtperf @@ -1,18 +1,21 @@ -# wtperf options file: simulate riak and its test4 configuration -# The configuration for the connection and table are from riak and the -# specification of the data (count, size, threads) is from basho_bench. +# wtperf options file: simulate MongoDB. +# The configuration for the connection and table are from mongoDB. +# We use multiple tables to simulate collections and indexes. # This test assumes that its correlating populate already completed and exists. # -#conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,statistics=(fast,clear),statistics_log=(wait=60)" -conn_config="cache_size=21G,checkpoint_sync=false,mmap=false,session_max=1024,log=(enabled),statistics=(fast),statistics_log=(wait=30)" -checkpoint_interval=60 +# Set cache to half of memory of AWS perf instance. Enable logging and +# checkpoints. Collect wiredtiger stats for ftdc. +conn_config="cache_size=16G,checkpoint=(wait=60,log_size=2GB),session_max=20000,log=(enabled),statistics=(fast),statistics_log=(wait=30,json),eviction=(threads_max=4)" create=false compression="snappy" sess_config="isolation=snapshot" +table_count=2 key_sz=40 -value_sz=1000 +value_sz=120 max_latency=2000 report_interval=10 run_time=7200 sample_interval=10 +sample_rate=1 threads=((count=20,reads=1)) +warmup=120 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf new file mode 100644 index 00000000000..6c517c6ddae --- /dev/null +++ b/src/third_party/wiredtiger/bench/wtperf/runners/checkpoint-stress.wtperf @@ -0,0 +1,20 @@ +# A stress configuration to create long running checkpoints while doing a lot +# of updates. +conn_config="cache_size=10GB,log=(enabled=false)" +table_config="leaf_page_max=32k,internal_page_max=16k,allocation_size=4k,split_pct=90,type=file" +# Enough data to fill the cache. 100 million 1k records results in two ~6GB +# tables +icount=100000000 +create=true +compression="snappy" +populate_threads=1 +checkpoint_interval=60 +checkpoint_threads=1 +report_interval=5 +# Run for a longer duration to ensure checkpoints are completing. +run_time=600 +# MongoDB always has multiple tables, and checkpoints behave differently when +# there is more than a single table. +table_count=2 +threads=((count=6,updates=1)) +value_sz=1000 diff --git a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh index 7a1ad44f39c..9968edc468a 100755 --- a/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh +++ b/src/third_party/wiredtiger/bench/wtperf/runners/wtperf_run.sh @@ -22,11 +22,16 @@ runmax=$2 # have 3 or 4 args. wtarg="" wtarg2="" +create=1 if test "$#" -gt "2"; then wtarg=$3 if test "$#" -eq "4"; then wtarg2=$4 fi + if test "$wtarg" == "NOCREATE"; then + create=0 + wtarg=$wtarg2 + fi fi home=./WT_TEST @@ -86,9 +91,11 @@ getmin=0 getmax=1 run=1 while test "$run" -le "$runmax"; do - rm -rf $home - mkdir $home - LD_PRELOAD=/usr/lib64/libjemalloc.so.1 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ./wtperf -O $wttest $wtarg $wtarg2 + if test "$create" -eq "1"; then + rm -rf $home + mkdir $home + fi + LD_PRELOAD=/usr/local/lib/libtcmalloc.so LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib ./wtperf -O $wttest $wtarg $wtarg2 if test "$?" -ne "0"; then exit 1 fi diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.c b/src/third_party/wiredtiger/bench/wtperf/wtperf.c index 58271106d61..d3d54fff6e3 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.c @@ -29,9 +29,11 @@ #include "wtperf.h" /* Default values. */ +#define DEFAULT_HOME "WT_TEST" +#define DEFAULT_MONITOR_DIR "WT_TEST" static const CONFIG default_cfg = { - "WT_TEST", /* home */ - "WT_TEST", /* monitor dir */ + NULL, /* home */ + NULL, /* monitor dir */ NULL, /* partial logging */ NULL, /* reopen config */ NULL, /* base_uri */ @@ -1538,8 +1540,7 @@ execute_populate(CONFIG *cfg) print_ops_sec = 0; } else { print_secs = (double)msecs / (double)MSEC_PER_SEC; - print_ops_sec = - (uint64_t)((cfg->icount / msecs) / MSEC_PER_SEC); + print_ops_sec = (uint64_t)(cfg->icount / print_secs); } lprintf(cfg, 0, 1, "Load time: %.2f\n" "load ops/sec: %" PRIu64, @@ -1944,13 +1945,12 @@ start_all_runs(CONFIG *cfg) { CONFIG *next_cfg, **configs; pthread_t *threads; - size_t cmd_len, home_len, i; + size_t home_len, i; int ret, t_ret; - char *cmd_buf, *new_home; + char *new_home; ret = 0; configs = NULL; - cmd_buf = NULL; if (cfg->database_count == 1) return (start_run(cfg)); @@ -1962,24 +1962,25 @@ start_all_runs(CONFIG *cfg) threads = dcalloc(cfg->database_count, sizeof(pthread_t)); home_len = strlen(cfg->home); - cmd_len = (home_len * 2) + 30; /* Add some slop. */ - cmd_buf = dcalloc(cmd_len, 1); for (i = 0; i < cfg->database_count; i++) { next_cfg = dcalloc(1, sizeof(CONFIG)); configs[i] = next_cfg; - if ((ret = config_assign(next_cfg, cfg)) != 0) + if ((ret = config_copy(next_cfg, cfg)) != 0) goto err; /* Setup a unique home directory for each database. */ new_home = dmalloc(home_len + 5); snprintf(new_home, home_len + 5, "%s/D%02d", cfg->home, (int)i); + free(next_cfg->home); next_cfg->home = new_home; /* If the monitor dir is default, update it too. */ - if (strcmp(cfg->monitor_dir, cfg->home) == 0) - next_cfg->monitor_dir = new_home; + if (strcmp(cfg->monitor_dir, cfg->home) == 0) { + free(next_cfg->monitor_dir); + next_cfg->monitor_dir = dstrdup(new_home); + } - /* If creating the sub-database, recreate it's home */ + /* If creating the sub-database, recreate its home */ if (cfg->create != 0) recreate_dir(next_cfg->home); @@ -1991,22 +1992,19 @@ start_all_runs(CONFIG *cfg) } /* Wait for threads to finish. */ - for (i = 0; i < cfg->database_count; i++) { + for (i = 0; i < cfg->database_count; i++) if ((t_ret = pthread_join(threads[i], NULL)) != 0) { lprintf(cfg, ret, 0, "Error joining thread"); if (ret == 0) ret = t_ret; } - } err: for (i = 0; i < cfg->database_count && configs[i] != NULL; i++) { - free((char *)configs[i]->home); config_free(configs[i]); free(configs[i]); } free(configs); free(threads); - free(cmd_buf); return (ret); } @@ -2199,10 +2197,10 @@ main(int argc, char *argv[]) /* Setup the default configuration values. */ cfg = &_cfg; memset(cfg, 0, sizeof(*cfg)); - if (config_assign(cfg, &default_cfg)) + if (config_copy(cfg, &default_cfg)) goto err; - - TAILQ_INIT(&cfg->config_head); + cfg->home = dstrdup(DEFAULT_HOME); + cfg->monitor_dir = dstrdup(DEFAULT_MONITOR_DIR); /* Do a basic validation of options, and home is needed before open. */ while ((ch = __wt_getopt("wtperf", argc, argv, opts)) != EOF) @@ -2219,10 +2217,12 @@ main(int argc, char *argv[]) } break; case 'h': - cfg->home = __wt_optarg; + free(cfg->home); + cfg->home = dstrdup(__wt_optarg); break; case 'm': - cfg->monitor_dir = __wt_optarg; + free(cfg->monitor_dir); + cfg->monitor_dir = dstrdup(__wt_optarg); monitor_set = true; break; case 'O': @@ -2248,8 +2248,10 @@ main(int argc, char *argv[]) * If the user did not specify a monitor directory then set the * monitor directory to the home dir. */ - if (!monitor_set) - cfg->monitor_dir = cfg->home; + if (!monitor_set) { + free(cfg->monitor_dir); + cfg->monitor_dir = dstrdup(cfg->home); + } /* Parse configuration settings from configuration file. */ if (config_opts != NULL && config_opt_file(cfg, config_opts) != 0) @@ -2558,7 +2560,7 @@ drop_all_tables(CONFIG *cfg) "Error opening a session on %s", cfg->home); return (ret); } - (void)__wt_epoch(NULL, &start); + testutil_check(__wt_epoch(NULL, &start)); for (i = 0; i < cfg->table_count; i++) { if ((ret = session->drop( session, cfg->uris[i], NULL)) != 0) { @@ -2567,7 +2569,7 @@ drop_all_tables(CONFIG *cfg) goto err; } } - (void)__wt_epoch(NULL, &stop); + testutil_check(__wt_epoch(NULL, &stop)); msecs = WT_TIMEDIFF_MS(stop, start); lprintf(cfg, 0, 1, "Executed %" PRIu32 " drop operations average time %" PRIu64 "ms", diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf.h b/src/third_party/wiredtiger/bench/wtperf/wtperf.h index 27c3832d316..e0467f4f3a1 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf.h +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf.h @@ -116,8 +116,8 @@ typedef struct { * an initialization in wtperf.c in the default_cfg. */ struct __config { /* Configuration structure */ - const char *home; /* WiredTiger home */ - const char *monitor_dir; /* Monitor output dir */ + char *home; /* WiredTiger home */ + char *monitor_dir; /* Monitor output dir */ char *partial_config; /* Config string for partial logging */ char *reopen_config; /* Config string for conn reopen */ char *base_uri; /* Object URI */ @@ -259,9 +259,9 @@ struct __config_thread { /* Per-thread structure */ }; void cleanup_truncate_config(CONFIG *); -int config_assign(CONFIG *, const CONFIG *); int config_compress(CONFIG *); void config_free(CONFIG *); +int config_copy(CONFIG *, const CONFIG *); int config_opt_file(CONFIG *, const char *); int config_opt_line(CONFIG *, const char *); int config_opt_str(CONFIG *, const char *, const char *); diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i index f6c96febc85..f1f26bd0d01 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_opt.i @@ -194,6 +194,8 @@ DEF_OPT_AS_CONFIG_STRING(transaction_config, "", "transaction configuration string, relevant when populate_opts_per_txn " "is nonzero") DEF_OPT_AS_STRING(table_name, "test", "table name") +DEF_OPT_AS_BOOL(truncate_single_ops, 0, + "Implement truncate via cursor remove instead of session API") DEF_OPT_AS_UINT32(value_sz_max, 1000, "maximum value size when delta updates are present. Default disabled") DEF_OPT_AS_UINT32(value_sz_min, 1, diff --git a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c index 11b09c60d5d..e6ebc83c681 100644 --- a/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c +++ b/src/third_party/wiredtiger/bench/wtperf/wtperf_truncate.c @@ -128,6 +128,7 @@ run_truncate(CONFIG *cfg, CONFIG_THREAD *thread, TRUNCATE_CONFIG *trunc_cfg; TRUNCATE_QUEUE_ENTRY *truncate_item; + char *next_key; int ret, t_ret; uint64_t used_stone_gap; @@ -184,15 +185,33 @@ run_truncate(CONFIG *cfg, CONFIG_THREAD *thread, truncate_item = TAILQ_FIRST(&cfg->stone_head); trunc_cfg->num_stones--; TAILQ_REMOVE(&cfg->stone_head, truncate_item, q); - cursor->set_key(cursor,truncate_item->key); - if ((ret = cursor->search(cursor)) != 0) { - lprintf(cfg, ret, 0, "Truncate search: failed"); - goto err; - } - if ((ret = session->truncate(session, NULL, NULL, cursor, NULL)) != 0) { - lprintf(cfg, ret, 0, "Truncate: failed"); - goto err; + /* + * Truncate the content via a single truncate call or a cursor walk + * depending on the configuration. + */ + if (cfg->truncate_single_ops) { + while ((ret = cursor->next(cursor)) == 0) { + testutil_check(cursor->get_key(cursor, &next_key)); + if (strcmp(next_key, truncate_item->key) == 0) + break; + if ((ret = cursor->remove(cursor)) != 0) { + lprintf(cfg, ret, 0, "Truncate remove: failed"); + goto err; + } + } + } else { + cursor->set_key(cursor,truncate_item->key); + if ((ret = cursor->search(cursor)) != 0) { + lprintf(cfg, ret, 0, "Truncate search: failed"); + goto err; + } + + if ((ret = session->truncate( + session, NULL, NULL, cursor, NULL)) != 0) { + lprintf(cfg, ret, 0, "Truncate: failed"); + goto err; + } } *truncatedp = 1; |