WT-2342 Add background create/drop to wtperf.

Works on idle tables. Gives us a way to ensure that schema operations
don't get blocked during high throughput workloads.

7 files changed, 230 insertions, 2 deletions

diff --git a/bench/wtperf/Makefile.am b/bench/wtperf/Makefile.am
index c8b98a3c37b..cc1f84b5406 100644
--- a/bench/wtperf/Makefile.am
+++ b/bench/wtperf/Makefile.am
@@ -5,8 +5,8 @@ LDADD = $(top_builddir)/libwiredtiger.la -lm
 noinst_PROGRAMS = wtperf
 wtperf_LDFLAGS = -static
 wtperf_SOURCES =\
-	config.c misc.c track.c wtperf.c wtperf.h \
-	wtperf_opt.i wtperf_throttle.c wtperf_truncate.c
+	config.c idle_table_cycle.c misc.c track.c wtperf.c \
+	wtperf.h wtperf_opt.i wtperf_throttle.c wtperf_truncate.c
 
 TESTS = smoke.sh
 AM_TESTS_ENVIRONMENT = rm -rf WT_TEST ; mkdir WT_TEST ;
diff --git a/bench/wtperf/idle_table_cycle.c b/bench/wtperf/idle_table_cycle.c
new file mode 100644
index 00000000000..3337d396fe3
--- /dev/null
+++ b/bench/wtperf/idle_table_cycle.c
@@ -0,0 +1,177 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "wtperf.h"
+
+static int
+check_timing(CONFIG *cfg,
+    const char *name, struct timespec start, struct timespec *stop)
+{
+	uint64_t last_interval;
+	int ret;
+
+	if ((ret = __wt_epoch(NULL, stop)) != 0) {
+		lprintf(cfg, ret, 0,
+		    "Get time failed in cycle_idle_tables.");
+		cfg->error = ret;
+		return (ret);
+	}
+
+	last_interval = (uint64_t)(WT_TIMEDIFF_SEC(*stop, start));
+
+	if (last_interval > cfg->idle_table_cycle) {
+		lprintf(cfg, ret, 0,
+		    "Cycling idle table failed because %s took %" PRIu64
+		    " seconds which is longer than configured acceptable"
+		    " maximum of %" PRIu32 ".",
+		    name, last_interval, cfg->idle_table_cycle);
+		cfg->error = ETIMEDOUT;
+		return (ETIMEDOUT);
+	}
+	return (0);
+}
+/*
+ * Regularly create, open a cursor and drop a table.
+ * Measure how long each step takes, and flag an error if it exceeds the
+ * configured maximum.
+ */
+static void *
+cycle_idle_tables(void *arg)
+{
+	struct timespec start, stop;
+	CONFIG *cfg;
+	WT_SESSION *session;
+	WT_CURSOR *cursor;
+	char uri[512];
+	int cycle_count, ret;
+
+	cfg = (CONFIG *)arg;
+	cycle_count = 0;
+
+	if ((ret = cfg->conn->open_session(
+	    cfg->conn, NULL, cfg->sess_config, &session)) != 0) {
+		lprintf(cfg, ret, 0,
+		    "Error opening a session on %s", cfg->home);
+		return (NULL);
+	}
+
+	for (cycle_count = 0; cfg->idle_cycle_run == 1; ++cycle_count) {
+		snprintf(uri, 512, "%s_cycle%07d", cfg->uris[0], cycle_count);
+		/* Don't busy cycle in this loop. */
+		__wt_sleep(1, 0);
+
+		/* Setup a start timer. */
+		if ((ret = __wt_epoch(NULL, &start)) != 0) {
+			lprintf(cfg, ret, 0,
+			     "Get time failed in cycle_idle_tables.");
+			cfg->error = ret;
+			return (NULL);
+		}
+
+		/* Create a table. */
+		if ((ret = session->create(
+		    session, uri, cfg->table_config)) != 0) {
+			lprintf(cfg, ret, 0,
+			     "Table create failed in cycle_idle_tables.");
+			cfg->error = ret;
+			return (NULL);
+		}
+		if (check_timing(cfg, "create", start, &stop) != 0)
+			return (NULL);
+		start = stop;
+
+		/* Open and close cursor. */
+		if ((ret = session->open_cursor(
+		    session, uri, NULL, NULL, &cursor)) != 0) {
+			lprintf(cfg, ret, 0,
+			     "Cursor open failed in cycle_idle_tables.");
+			cfg->error = ret;
+			return (NULL);
+		}
+		if ((ret = cursor->close(cursor)) != 0) {
+			lprintf(cfg, ret, 0,
+			     "Cursor close failed in cycle_idle_tables.");
+			cfg->error = ret;
+			return (NULL);
+		}
+		if (check_timing(cfg, "cursor", start, &stop) != 0)
+			return (NULL);
+		start = stop;
+
+		/* Drop the table. */
+		if ((ret = session->drop(session, uri, NULL)) != 0) {
+			lprintf(cfg, ret, 0,
+			     "Table drop failed in cycle_idle_tables.");
+			cfg->error = ret;
+			return (NULL);
+		}
+		if (check_timing(cfg, "drop", start, &stop) != 0)
+			return (NULL);
+	}
+
+	return (NULL);
+}
+
+int
+start_idle_table_cycle(CONFIG *cfg)
+{
+	pthread_t thread_id;
+	int ret;
+
+	if (cfg->idle_table_cycle == 0)
+		return (0);
+
+	cfg->idle_cycle_run = 1;
+	if ((ret = pthread_create(
+	    &thread_id, NULL, cycle_idle_tables, cfg)) != 0) {
+		lprintf(
+		    cfg, ret, 0, "Error creating idle table cycle thread.");
+		cfg->idle_cycle_run = 0;
+		return (ret);
+	}
+	cfg->idle_table_cycle_thread = thread_id;
+
+	return (0);
+}
+
+int
+stop_idle_table_cycle(CONFIG *cfg)
+{
+	int ret;
+
+	if (cfg->idle_table_cycle == 0 || cfg->idle_cycle_run == 0)
+		return (0);
+
+	cfg->idle_cycle_run = 0;
+	if ((ret = pthread_join(cfg->idle_table_cycle_thread, NULL)) != 0) {
+		lprintf(
+		    cfg, ret, 0, "Error joining idle table cycle thread.");
+		return (ret);
+	}
+	return (0);
+}
diff --git a/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf b/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf
new file mode 100644
index 00000000000..b69ead7f8b1
--- /dev/null
+++ b/bench/wtperf/runners/checkpoint_stress_schema_ops.wtperf
@@ -0,0 +1,19 @@
+# A stress configuration, to create long running checkpoints and see how
+# they interfere with schema level operations (table create, drop).
+# Setup a cache size large enough that checkpoints can take a long time.
+conn_config="cache_size=8GB,log=(enabled=false),checkpoint=(wait=30)"
+table_config="leaf_page_max=4k,internal_page_max=16k,type=file"
+icount=10000000
+table_count_idle=100
+# Turn on create/drop of idle tables, and error if a single operation takes
+# more than 5 seconds.
+idle_table_cycle=5
+populate_threads=5
+checkpoint_threads=0
+report_interval=5
+# 100 million
+random_range=100000000
+run_time=1000
+# Setup a workload that dirties a lot of the cache
+threads=((count=2,reads=1),(count=2,inserts=1),(count=2,updates=1))
+value_sz=500
diff --git a/bench/wtperf/wtperf.c b/bench/wtperf/wtperf.c
index e5526880f2a..cef3138b865 100644
--- a/bench/wtperf/wtperf.c
+++ b/bench/wtperf/wtperf.c
@@ -57,6 +57,8 @@ static const CONFIG default_cfg = {
 	0,				/* thread error */
 	0,				/* notify threads to stop */
 	0,				/* in warmup phase */
+	NULL,				/* Thread ID of idle cycle thread */
+	0,				/* Signal idle cycle thread */
 	0,				/* total seconds running */
 	0,				/* has truncate */
 	{NULL, NULL},			/* the truncate queue */
@@ -1371,6 +1373,10 @@ execute_populate(CONFIG *cfg)
 	    " populate thread(s) for %" PRIu32 " items",
 	    cfg->populate_threads, cfg->icount);
 
+	/* Start cycling idle tables if configured. */
+	if ((ret = start_idle_table_cycle(cfg)) != 0)
+		return (ret);
+
 	cfg->insert_key = 0;
 
 	cfg->popthreads = dcalloc(cfg->populate_threads, sizeof(CONFIG_THREAD));
@@ -1498,6 +1504,11 @@ execute_populate(CONFIG *cfg)
 		    (uint64_t)(WT_TIMEDIFF_SEC(stop, start)));
 		assert(tables == 0);
 	}
+
+	/* Stop cycling idle tables. */
+	if ((ret = stop_idle_table_cycle(cfg)) != 0)
+		return (ret);
+
 	return (0);
 }
 
@@ -1562,6 +1573,10 @@ execute_workload(CONFIG *cfg)
 	last_updates = 0;
 	ret = 0;
 
+	/* Start cycling idle tables. */
+	if ((ret = start_idle_table_cycle(cfg)) != 0)
+		return (ret);
+
 	if (cfg->warmup != 0)
 		cfg->in_warmup = 1;
 
@@ -1657,6 +1672,10 @@ execute_workload(CONFIG *cfg)
 	/* Notify the worker threads they are done. */
 err:	cfg->stop = 1;
 
+	/* Stop cycling idle tables. */
+	if ((ret = stop_idle_table_cycle(cfg)) != 0)
+		return (ret);
+
 	if ((t_ret = stop_threads(
 	    cfg, (u_int)cfg->workers_cnt, cfg->workers)) != 0 && ret == 0)
 		ret = t_ret;
diff --git a/bench/wtperf/wtperf.h b/bench/wtperf/wtperf.h
index 90922e1fc21..3a03b68b116 100644
--- a/bench/wtperf/wtperf.h
+++ b/bench/wtperf/wtperf.h
@@ -187,6 +187,9 @@ struct __config {			/* Configuration structure */
 	volatile int stop;		/* notify threads to stop */
 	volatile int in_warmup;		/* Running warmup phase */
 
+	pthread_t idle_table_cycle_thread; /* Thread ID of idle cycle thread */
+	volatile int idle_cycle_run;	/* Signal idle cycle thread */
+
 	volatile uint32_t totalsec;	/* total seconds running */
 
 	u_int		 has_truncate;  /* if there is a truncate workload */
@@ -303,6 +306,8 @@ int	 run_truncate(
 int	 setup_log_file(CONFIG *);
 int	 setup_throttle(CONFIG_THREAD*);
 int	 setup_truncate(CONFIG *, CONFIG_THREAD *, WT_SESSION *);
+int	 start_idle_table_cycle(CONFIG *);
+int	 stop_idle_table_cycle(CONFIG *);
 uint64_t sum_ckpt_ops(CONFIG *);
 uint64_t sum_insert_ops(CONFIG *);
 uint64_t sum_pop_ops(CONFIG *);
diff --git a/bench/wtperf/wtperf_opt.i b/bench/wtperf/wtperf_opt.i
index c93cbf50314..6dc2a6d5569 100644
--- a/bench/wtperf/wtperf_opt.i
+++ b/bench/wtperf/wtperf_opt.i
@@ -110,6 +110,10 @@ DEF_OPT_AS_UINT32(drop_tables, 0,
 DEF_OPT_AS_UINT32(icount, 5000,
     "number of records to initially populate. If multiple tables are "
     "configured the count is spread evenly across all tables.")
+DEF_OPT_AS_UINT32(idle_table_cycle, 0,
+    "Enable regular create and drop of idle tables, value is the maximum "
+    "number of seconds a create or drop is allowed before flagging an error. "
+    "Default 0 which means disabled.")
 DEF_OPT_AS_BOOL(index, 0,
     "Whether to create an index on the value field.")
 DEF_OPT_AS_BOOL(insert_rmw, 0,
diff --git a/src/docs/wtperf.dox b/src/docs/wtperf.dox
index 339bf740265..64e25978dd8 100644
--- a/src/docs/wtperf.dox
+++ b/src/docs/wtperf.dox
@@ -173,6 +173,10 @@ taken to do the drop.
 @par icount (unsigned int, default=5000)
 number of records to initially populate. If multiple tables are
 configured the count is spread evenly across all tables.
+@par idle_table_cycle (unsigned int, default=0)
+Enable regular create and drop of idle tables, value is the maximum
+number of seconds a create or drop is allowed before flagging an
+error. Default 0 which means disabled.
 @par index (boolean, default=false)
 Whether to create an index on the value field.
 @par insert_rmw (boolean, default=false)