diff options
author | Alex Gorrod <alexander.gorrod@mongodb.com> | 2016-01-22 16:40:39 +1100 |
---|---|---|
committer | Alex Gorrod <alexander.gorrod@mongodb.com> | 2016-01-22 16:40:39 +1100 |
commit | cd1aef20bfb5c4ee32177f59631099e7da577cc5 (patch) | |
tree | 7b8ddc9b3ebd859eae12a85775439972de4a4bfe | |
parent | 17d4afc247f48b9f4436de038bb939a8445efc93 (diff) | |
parent | 494022b3de6710136e72e9d292d58f1f33f57650 (diff) | |
download | mongo-cd1aef20bfb5c4ee32177f59631099e7da577cc5.tar.gz |
Merge pull request #2452 from wiredtiger/WT-2342-wtperf-background-schema
WT-2342 Add background create/drop to wtperf.
-rw-r--r-- | bench/wtperf/Makefile.am | 4 | ||||
-rw-r--r-- | bench/wtperf/idle_table_cycle.c | 192 | ||||
-rw-r--r-- | bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf | 19 | ||||
-rw-r--r-- | bench/wtperf/wtperf.c | 20 | ||||
-rw-r--r-- | bench/wtperf/wtperf.h | 4 | ||||
-rw-r--r-- | bench/wtperf/wtperf_opt.i | 4 | ||||
-rw-r--r-- | src/docs/wtperf.dox | 4 |
7 files changed, 245 insertions, 2 deletions
diff --git a/bench/wtperf/Makefile.am b/bench/wtperf/Makefile.am index c8b98a3c37b..cc1f84b5406 100644 --- a/bench/wtperf/Makefile.am +++ b/bench/wtperf/Makefile.am @@ -5,8 +5,8 @@ LDADD = $(top_builddir)/libwiredtiger.la -lm noinst_PROGRAMS = wtperf wtperf_LDFLAGS = -static wtperf_SOURCES =\ - config.c misc.c track.c wtperf.c wtperf.h \ - wtperf_opt.i wtperf_throttle.c wtperf_truncate.c + config.c idle_table_cycle.c misc.c track.c wtperf.c \ + wtperf.h wtperf_opt.i wtperf_throttle.c wtperf_truncate.c TESTS = smoke.sh AM_TESTS_ENVIRONMENT = rm -rf WT_TEST ; mkdir WT_TEST ; diff --git a/bench/wtperf/idle_table_cycle.c b/bench/wtperf/idle_table_cycle.c new file mode 100644 index 00000000000..d80508a843e --- /dev/null +++ b/bench/wtperf/idle_table_cycle.c @@ -0,0 +1,192 @@ +/*- + * Public Domain 2014-2016 MongoDB, Inc. + * Public Domain 2008-2014 WiredTiger, Inc. + * + * This is free and unencumbered software released into the public domain. + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this software, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this software dedicate any and all copyright interest in the + * software to the public domain. We make this dedication for the benefit + * of the public at large and to the detriment of our heirs and + * successors. We intend this dedication to be an overt act of + * relinquishment in perpetuity of all present and future rights to this + * software under copyright law. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "wtperf.h" + +static int +check_timing(CONFIG *cfg, + const char *name, struct timespec start, struct timespec *stop) +{ + uint64_t last_interval; + int ret; + + if ((ret = __wt_epoch(NULL, stop)) != 0) { + lprintf(cfg, ret, 0, + "Get time failed in cycle_idle_tables."); + cfg->error = ret; + return (ret); + } + + last_interval = (uint64_t)(WT_TIMEDIFF_SEC(*stop, start)); + + if (last_interval > cfg->idle_table_cycle) { + lprintf(cfg, ret, 0, + "Cycling idle table failed because %s took %" PRIu64 + " seconds which is longer than configured acceptable" + " maximum of %" PRIu32 ".", + name, last_interval, cfg->idle_table_cycle); + cfg->error = ETIMEDOUT; + return (ETIMEDOUT); + } + return (0); +} +/* + * Regularly create, open a cursor and drop a table. + * Measure how long each step takes, and flag an error if it exceeds the + * configured maximum. + */ +static void * +cycle_idle_tables(void *arg) +{ + struct timespec start, stop; + CONFIG *cfg; + WT_SESSION *session; + WT_CURSOR *cursor; + char uri[512]; + int cycle_count, ret; + + cfg = (CONFIG *)arg; + cycle_count = 0; + + if ((ret = cfg->conn->open_session( + cfg->conn, NULL, cfg->sess_config, &session)) != 0) { + lprintf(cfg, ret, 0, + "Error opening a session on %s", cfg->home); + return (NULL); + } + + for (cycle_count = 0; cfg->idle_cycle_run; ++cycle_count) { + snprintf(uri, 512, "%s_cycle%07d", cfg->uris[0], cycle_count); + /* Don't busy cycle in this loop. */ + __wt_sleep(1, 0); + + /* Setup a start timer. */ + if ((ret = __wt_epoch(NULL, &start)) != 0) { + lprintf(cfg, ret, 0, + "Get time failed in cycle_idle_tables."); + cfg->error = ret; + return (NULL); + } + + /* Create a table. */ + if ((ret = session->create( + session, uri, cfg->table_config)) != 0) { + if (ret == EBUSY) + continue; + lprintf(cfg, ret, 0, + "Table create failed in cycle_idle_tables."); + cfg->error = ret; + return (NULL); + } + if (check_timing(cfg, "create", start, &stop) != 0) + return (NULL); + start = stop; + + /* Open and close cursor. */ + if ((ret = session->open_cursor( + session, uri, NULL, NULL, &cursor)) != 0) { + lprintf(cfg, ret, 0, + "Cursor open failed in cycle_idle_tables."); + cfg->error = ret; + return (NULL); + } + if ((ret = cursor->close(cursor)) != 0) { + lprintf(cfg, ret, 0, + "Cursor close failed in cycle_idle_tables."); + cfg->error = ret; + return (NULL); + } + if (check_timing(cfg, "cursor", start, &stop) != 0) + return (NULL); + start = stop; + + /* + * Drop the table. Keep retrying on EBUSY failure - it is an + * expected return when checkpoints are happening. + */ + while ((ret = session->drop(session, uri, "force")) == EBUSY) + __wt_sleep(1, 0); + + if (ret != 0 && ret != EBUSY) { + lprintf(cfg, ret, 0, + "Table drop failed in cycle_idle_tables."); + cfg->error = ret; + return (NULL); + } + if (check_timing(cfg, "drop", start, &stop) != 0) + return (NULL); + } + + return (NULL); +} + +/* + * Start a thread the creates and drops tables regularly. + * TODO: Currently accepts a pthread_t as a parameter, since it is not + * possible to portably staticially initialize it in the global configuration + * structure. Should reshuffle the configuration structure so explicit static + * initializers aren't necessary. + */ +int +start_idle_table_cycle(CONFIG *cfg, pthread_t *idle_table_cycle_thread) +{ + pthread_t thread_id; + int ret; + + if (cfg->idle_table_cycle == 0) + return (0); + + cfg->idle_cycle_run = true; + if ((ret = pthread_create( + &thread_id, NULL, cycle_idle_tables, cfg)) != 0) { + lprintf( + cfg, ret, 0, "Error creating idle table cycle thread."); + cfg->idle_cycle_run = false; + return (ret); + } + *idle_table_cycle_thread = thread_id; + + return (0); +} + +int +stop_idle_table_cycle(CONFIG *cfg, pthread_t idle_table_cycle_thread) +{ + int ret; + + if (cfg->idle_table_cycle == 0 || !cfg->idle_cycle_run) + return (0); + + cfg->idle_cycle_run = false; + if ((ret = pthread_join(idle_table_cycle_thread, NULL)) != 0) { + lprintf( + cfg, ret, 0, "Error joining idle table cycle thread."); + return (ret); + } + return (0); +} diff --git a/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf b/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf new file mode 100644 index 00000000000..b69ead7f8b1 --- /dev/null +++ b/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf @@ -0,0 +1,19 @@ +# A stress configuration, to create long running checkpoints and see how +# they interfere with schema level operations (table create, drop). +# Setup a cache size large enough that checkpoints can take a long time. +conn_config="cache_size=8GB,log=(enabled=false),checkpoint=(wait=30)" +table_config="leaf_page_max=4k,internal_page_max=16k,type=file" +icount=10000000 +table_count_idle=100 +# Turn on create/drop of idle tables, and error if a single operation takes +# more than 5 seconds. +idle_table_cycle=5 +populate_threads=5 +checkpoint_threads=0 +report_interval=5 +# 100 million +random_range=100000000 +run_time=1000 +# Setup a workload that dirties a lot of the cache +threads=((count=2,reads=1),(count=2,inserts=1),(count=2,updates=1)) +value_sz=500 diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c index e5526880f2a..b2e68198e9a 100644 --- a/bench/wtperf/wtperf.c +++ b/bench/wtperf/wtperf.c @@ -57,6 +57,7 @@ static const CONFIG default_cfg = { 0, /* thread error */ 0, /* notify threads to stop */ 0, /* in warmup phase */ + false, /* Signal for idle cycle thread */ 0, /* total seconds running */ 0, /* has truncate */ {NULL, NULL}, /* the truncate queue */ @@ -1359,6 +1360,7 @@ execute_populate(CONFIG *cfg) struct timespec start, stop; CONFIG_THREAD *popth; WT_ASYNC_OP *asyncop; + pthread_t idle_table_cycle_thread; size_t i; uint64_t last_ops, msecs, print_ops_sec; uint32_t interval, tables; @@ -1371,6 +1373,10 @@ execute_populate(CONFIG *cfg) " populate thread(s) for %" PRIu32 " items", cfg->populate_threads, cfg->icount); + /* Start cycling idle tables if configured. */ + if ((ret = start_idle_table_cycle(cfg, &idle_table_cycle_thread)) != 0) + return (ret); + cfg->insert_key = 0; cfg->popthreads = dcalloc(cfg->populate_threads, sizeof(CONFIG_THREAD)); @@ -1498,6 +1504,11 @@ execute_populate(CONFIG *cfg) (uint64_t)(WT_TIMEDIFF_SEC(stop, start))); assert(tables == 0); } + + /* Stop cycling idle tables. */ + if ((ret = stop_idle_table_cycle(cfg, idle_table_cycle_thread)) != 0) + return (ret); + return (0); } @@ -1547,6 +1558,7 @@ execute_workload(CONFIG *cfg) { CONFIG_THREAD *threads; WORKLOAD *workp; + pthread_t idle_table_cycle_thread; uint64_t last_ckpts, last_inserts, last_reads, last_truncates; uint64_t last_updates; uint32_t interval, run_ops, run_time; @@ -1562,6 +1574,10 @@ execute_workload(CONFIG *cfg) last_updates = 0; ret = 0; + /* Start cycling idle tables. */ + if ((ret = start_idle_table_cycle(cfg, &idle_table_cycle_thread)) != 0) + return (ret); + if (cfg->warmup != 0) cfg->in_warmup = 1; @@ -1657,6 +1673,10 @@ execute_workload(CONFIG *cfg) /* Notify the worker threads they are done. */ err: cfg->stop = 1; + /* Stop cycling idle tables. */ + if ((ret = stop_idle_table_cycle(cfg, idle_table_cycle_thread)) != 0) + return (ret); + if ((t_ret = stop_threads( cfg, (u_int)cfg->workers_cnt, cfg->workers)) != 0 && ret == 0) ret = t_ret; diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h index 90922e1fc21..7dbe1822a26 100644 --- a/bench/wtperf/wtperf.h +++ b/bench/wtperf/wtperf.h @@ -187,6 +187,8 @@ struct __config { /* Configuration structure */ volatile int stop; /* notify threads to stop */ volatile int in_warmup; /* Running warmup phase */ + volatile bool idle_cycle_run; /* Signal for idle cycle thread */ + volatile uint32_t totalsec; /* total seconds running */ u_int has_truncate; /* if there is a truncate workload */ @@ -303,6 +305,8 @@ int run_truncate( int setup_log_file(CONFIG *); int setup_throttle(CONFIG_THREAD*); int setup_truncate(CONFIG *, CONFIG_THREAD *, WT_SESSION *); +int start_idle_table_cycle(CONFIG *, pthread_t *); +int stop_idle_table_cycle(CONFIG *, pthread_t); uint64_t sum_ckpt_ops(CONFIG *); uint64_t sum_insert_ops(CONFIG *); uint64_t sum_pop_ops(CONFIG *); diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i index c93cbf50314..6dc2a6d5569 100644 --- a/bench/wtperf/wtperf_opt.i +++ b/bench/wtperf/wtperf_opt.i @@ -110,6 +110,10 @@ DEF_OPT_AS_UINT32(drop_tables, 0, DEF_OPT_AS_UINT32(icount, 5000, "number of records to initially populate. If multiple tables are " "configured the count is spread evenly across all tables.") +DEF_OPT_AS_UINT32(idle_table_cycle, 0, + "Enable regular create and drop of idle tables, value is the maximum " + "number of seconds a create or drop is allowed before flagging an error. " + "Default 0 which means disabled.") DEF_OPT_AS_BOOL(index, 0, "Whether to create an index on the value field.") DEF_OPT_AS_BOOL(insert_rmw, 0, diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox index 339bf740265..64e25978dd8 100644 --- a/src/docs/wtperf.dox +++ b/src/docs/wtperf.dox @@ -173,6 +173,10 @@ taken to do the drop. @par icount (unsigned int, default=5000) number of records to initially populate. If multiple tables are configured the count is spread evenly across all tables. +@par idle_table_cycle (unsigned int, default=0) +Enable regular create and drop of idle tables, value is the maximum +number of seconds a create or drop is allowed before flagging an +error. Default 0 which means disabled. @par index (boolean, default=false) Whether to create an index on the value field. @par insert_rmw (boolean, default=false) |