WT-2399 Add initial implementation for cursor traversal test application.

Based loosely on the test/checkpoint implementation. Hopefully we can
refactor, so code is shared between the two via test_util.i.

5 files changed, 869 insertions, 0 deletions

diff --git a/test/cursor_order/Makefile.am b/test/cursor_order/Makefile.am
new file mode 100644
index 00000000000..c0c0ed639bf
--- /dev/null
+++ b/test/cursor_order/Makefile.am
@@ -0,0 +1,13 @@
+AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
+    -I$(top_srcdir)/test/utility
+
+noinst_PROGRAMS = cursor_order
+cursor_order_LDADD = $(top_builddir)/libwiredtiger.la
+
+cursor_order_SOURCES = cursor_order_file.c cursor_order_ops.c cursor_order.c
+cursor_order_LDFLAGS = -static
+
+TESTS = $(noinst_PROGRAMS)
+
+clean-local:
+	rm -rf WiredTiger* wt.* *.core __stats
diff --git a/test/cursor_order/cursor_order.c b/test/cursor_order/cursor_order.c
new file mode 100644
index 00000000000..0155339d71e
--- /dev/null
+++ b/test/cursor_order/cursor_order.c
@@ -0,0 +1,303 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "cursor_order.h"
+
+static char home[512];				/* Program working dir */
+static char *progname;				/* Program name */
+static FILE *logfp;				/* Log file */
+
+static int  handle_error(WT_EVENT_HANDLER *, WT_SESSION *, int, const char *);
+static int  handle_message(WT_EVENT_HANDLER *, WT_SESSION *, const char *);
+static void onint(int);
+static void shutdown(void);
+static int  usage(void);
+static void wt_connect(SHARED_CONFIG *, char *);
+static void wt_shutdown(SHARED_CONFIG *);
+
+extern int __wt_optind;
+extern char *__wt_optarg;
+
+int
+main(int argc, char *argv[])
+{
+	SHARED_CONFIG _cfg, *cfg;
+	int ch, cnt, runs;
+	char *config_open, *working_dir;
+
+	if ((progname = strrchr(argv[0], DIR_DELIM)) == NULL)
+		progname = argv[0];
+	else
+		++progname;
+
+	cfg = &_cfg;
+	config_open = NULL;
+	working_dir = NULL;
+	runs = 1;
+
+	/*
+	 * Explicitly initialize the shared configuration object before
+	 * parsing command line options.
+	 */
+	cfg->append_inserters = 1;
+	cfg->conn = NULL;
+	cfg->ftype = ROW;
+	cfg->max_nops = 1000000;
+	cfg->multiple_files = false;
+	cfg->nkeys = 1000;
+	cfg->reverse_scanners = 5;
+	cfg->reverse_scan_ops = 10;
+	cfg->thread_finish = false;
+	cfg->vary_nops = false;
+
+	while ((ch = __wt_getopt(
+	    progname, argc, argv, "C:Fk:h:l:n:R:r:t:vw:W:")) != EOF)
+		switch (ch) {
+		case 'C':			/* wiredtiger_open config */
+			config_open = __wt_optarg;
+			break;
+		case 'F':			/* multiple files */
+			cfg->multiple_files = true;
+			break;
+		case 'h':
+			working_dir = __wt_optarg;
+			break;
+		case 'k':			/* rows */
+			cfg->nkeys = (u_int64_t)atol(__wt_optarg);
+			break;
+		case 'l':			/* log */
+			if ((logfp = fopen(__wt_optarg, "w")) == NULL) {
+				fprintf(stderr,
+				    "%s: %s\n", __wt_optarg, strerror(errno));
+				return (EXIT_FAILURE);
+			}
+			break;
+		case 'n':			/* operations */
+			cfg->max_nops = (u_int64_t)atol(__wt_optarg);
+			break;
+		case 'R':
+			cfg->reverse_scanners = (u_int64_t)atol(__wt_optarg);
+			break;
+		case 'r':			/* runs */
+			runs = atoi(__wt_optarg);
+			break;
+		case 't':
+			switch (__wt_optarg[0]) {
+			case 'f':
+				cfg->ftype = FIX;
+				break;
+			case 'r':
+				cfg->ftype = ROW;
+				break;
+			case 'v':
+				cfg->ftype = VAR;
+				break;
+			default:
+				return (usage());
+			}
+			break;
+		case 'v':			/* vary operation count */
+			cfg->vary_nops = true;
+			break;
+		case 'w':
+			cfg->reverse_scan_ops = (u_int64_t)atol(__wt_optarg);
+			break;
+		case 'W':
+			cfg->append_inserters = (u_int64_t)atol(__wt_optarg);
+			break;
+		default:
+			return (usage());
+		}
+
+	argc -= __wt_optind;
+	argv += __wt_optind;
+	if (argc != 0)
+		return (usage());
+
+	testutil_work_dir_from_path(home, 512, working_dir);
+
+	if (cfg->vary_nops && !cfg->multiple_files) {
+		fprintf(stderr,
+		    "Variable op counts only supported with multiple tables\n");
+		return (usage());
+	}
+
+	/* Clean up on signal. */
+	(void)signal(SIGINT, onint);
+
+	printf("%s: process %" PRIu64 "\n", progname, (uint64_t)getpid());
+	for (cnt = 1; runs == 0 || cnt <= runs; ++cnt) {
+		printf("    %d: %u reverse scanners, %u writers\n", cnt,
+		    (int)cfg->reverse_scanners, (int)cfg->append_inserters);
+
+		shutdown();			/* Clean up previous runs */
+
+		wt_connect(cfg, config_open);	/* WiredTiger connection */
+
+		if (ops_start(cfg))
+			return (EXIT_FAILURE);
+
+		wt_shutdown(cfg);		/* WiredTiger shut down */
+	}
+	return (0);
+}
+
+/*
+ * wt_connect --
+ *	Configure the WiredTiger connection.
+ */
+static void
+wt_connect(SHARED_CONFIG *cfg, char *config_open)
+{
+	static WT_EVENT_HANDLER event_handler = {
+		handle_error,
+		handle_message,
+		NULL,
+		NULL	/* Close handler. */
+	};
+	int ret;
+	char config[512];
+	size_t print_count;
+
+	testutil_clean_work_dir(home);
+	testutil_make_work_dir(home);
+
+	print_count = (size_t)snprintf(config, sizeof(config),
+	    "create,statistics=(all),error_prefix=\"%s\",%s%s",
+	    progname,
+	    config_open == NULL ? "" : ",",
+	    config_open == NULL ? "" : config_open);
+
+	if (print_count >= sizeof(config))
+		testutil_die(EINVAL, "Config string too long");
+
+	if ((ret = wiredtiger_open(
+	    home, &event_handler, config, &cfg->conn)) != 0)
+		testutil_die(ret, "wiredtiger_open");
+}
+
+/*
+ * wt_shutdown --
+ *	Flush the file to disk and shut down the WiredTiger connection.
+ */
+static void
+wt_shutdown(SHARED_CONFIG *cfg)
+{
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	int ret;
+
+	conn = cfg->conn;
+
+	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
+		testutil_die(ret, "conn.session");
+
+	if ((ret = session->checkpoint(session, NULL)) != 0)
+		testutil_die(ret, "session.checkpoint");
+
+	if ((ret = conn->close(conn, NULL)) != 0)
+		testutil_die(ret, "conn.close");
+}
+
+/*
+ * shutdown --
+ *	Clean up from previous runs.
+ */
+static void
+shutdown(void)
+{
+	testutil_clean_work_dir(home);
+}
+
+static int
+handle_error(WT_EVENT_HANDLER *handler,
+    WT_SESSION *session, int error, const char *errmsg)
+{
+	(void)(handler);
+	(void)(session);
+	(void)(error);
+
+	return (fprintf(stderr, "%s\n", errmsg) < 0 ? -1 : 0);
+}
+
+static int
+handle_message(WT_EVENT_HANDLER *handler,
+    WT_SESSION *session, const char *message)
+{
+	(void)(handler);
+	(void)(session);
+
+	if (logfp != NULL)
+		return (fprintf(logfp, "%s\n", message) < 0 ? -1 : 0);
+
+	return (printf("%s\n", message) < 0 ? -1 : 0);
+}
+
+/*
+ * onint --
+ *	Interrupt signal handler.
+ */
+static void
+onint(int signo)
+{
+	(void)(signo);
+
+	shutdown();
+
+	fprintf(stderr, "\n");
+	exit(EXIT_FAILURE);
+}
+
+/*
+ * usage --
+ *	Display usage statement and exit failure.
+ */
+static int
+usage(void)
+{
+	fprintf(stderr,
+	    "usage: %s "
+	    "[-FLv] [-C wiredtiger-config] [-k keys] [-l log]\n\t"
+	    "[-n ops] [-R reverse_scanners] [-r runs] [-t f|r|v] "
+	    "[-W append_inserters]\n",
+	    progname);
+	fprintf(stderr, "%s",
+	    "\t-C specify wiredtiger_open configuration arguments\n"
+	    "\t-F create a file per thread\n"
+	    "\t-k set number of keys to load\n"
+	    "\t-L log print per operation\n"
+	    "\t-l specify a log file\n"
+	    "\t-n set number of operations each thread does\n"
+	    "\t-R set number of reverse scanner threads\n"
+	    "\t-r set number of runs (0 for continuous)\n"
+	    "\t-t set a file type (fix | row | var)\n"
+	    "\t-v do a different number of operations on different tables\n"
+	    "\t-w set number of items to walk in a reverse scan\n"
+	    "\t-W set number of threads doing append inserts\n");
+	return (EXIT_FAILURE);
+}
diff --git a/test/cursor_order/cursor_order.h b/test/cursor_order/cursor_order.h
new file mode 100644
index 00000000000..9a3ae51ed91
--- /dev/null
+++ b/test/cursor_order/cursor_order.h
@@ -0,0 +1,63 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <sys/types.h>
+#include <sys/time.h>
+
+#include <errno.h>
+#include <inttypes.h>
+#include <pthread.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "test_util.i"
+
+#define	FNAME		"file:cursor_order.%03d"	/* File name */
+
+typedef enum { FIX, ROW, VAR } __ftype;		/* File type */
+typedef struct {
+	uint64_t append_inserters;		/* Number of append threads */
+	WT_CONNECTION *conn;			/* WiredTiger connection */
+	__ftype ftype;
+	uint64_t key_range;			/* Current key range */
+	uint64_t max_nops;			/* Operations per thread */
+	bool      multiple_files;		/* File per thread */
+	uint64_t nkeys;			/* Keys to load */
+	uint64_t reverse_scanners;		/* Number of scan threads */
+	uint64_t reverse_scan_ops;		/* Keys to visit per scan */
+	bool      thread_finish;		/* Signal to finish run. */
+	bool      vary_nops;			/* Operations per thread */
+
+} SHARED_CONFIG;
+
+void load(SHARED_CONFIG *, const char *);
+int  ops_start(SHARED_CONFIG *);
+void verify(SHARED_CONFIG *, const char *);
diff --git a/test/cursor_order/cursor_order_file.c b/test/cursor_order/cursor_order_file.c
new file mode 100644
index 00000000000..81c76a3a63e
--- /dev/null
+++ b/test/cursor_order/cursor_order_file.c
@@ -0,0 +1,129 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "cursor_order.h"
+
+static void
+file_create(SHARED_CONFIG *cfg, const char *name)
+{
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	int ret;
+	char *p, *end, config[128];
+
+	conn = cfg->conn;
+
+	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
+		testutil_die(ret, "conn.session");
+
+	p = config;
+	end = config + sizeof(config);
+	p += snprintf(p, (size_t)(end - p),
+	    "key_format=%s,"
+	    "internal_page_max=%d,"
+	    "split_deepen_min_child=200,"
+	    "leaf_page_max=%d,",
+	    cfg->ftype == ROW ? "S" : "r", 16 * 1024, 128 * 1024);
+	if (cfg->ftype == FIX)
+		(void)snprintf(p, (size_t)(end - p), ",value_format=3t");
+
+	if ((ret = session->create(session, name, config)) != 0)
+		if (ret != EEXIST)
+			testutil_die(ret, "session.create");
+
+	if ((ret = session->close(session, NULL)) != 0)
+		testutil_die(ret, "session.close");
+}
+
+void
+load(SHARED_CONFIG *cfg, const char *name)
+{
+	WT_CONNECTION *conn;
+	WT_CURSOR *cursor;
+	WT_ITEM *value, _value;
+	WT_SESSION *session;
+	char keybuf[64], valuebuf[64];
+	int64_t keyno;
+	int ret;
+
+	conn = cfg->conn;
+
+	file_create(cfg, name);
+
+	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
+		testutil_die(ret, "conn.session");
+
+	if ((ret =
+	    session->open_cursor(session, name, NULL, "bulk", &cursor)) != 0)
+		testutil_die(ret, "cursor.open");
+
+	value = &_value;
+	for (keyno = 1; keyno <= (int64_t)cfg->nkeys; ++keyno) {
+		if (cfg->ftype == ROW) {
+			snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno);
+			cursor->set_key(cursor, &keybuf);
+		} else
+			cursor->set_key(cursor, (uint32_t)keyno);
+		value->data = valuebuf;
+		if (cfg->ftype == FIX)
+			cursor->set_value(cursor, 0x01);
+		else {
+			value->size = (uint32_t)snprintf(
+			    valuebuf, sizeof(valuebuf), "%37u", (u_int)keyno);
+			cursor->set_value(cursor, value);
+		}
+		if ((ret = cursor->insert(cursor)) != 0)
+			testutil_die(ret, "cursor.insert");
+	}
+	/* Setup the starting key range for the workload phase. */
+	cfg->key_range = cfg->nkeys;
+	cursor->close(cursor);
+	session->checkpoint(session, NULL);
+
+	if ((ret = session->close(session, NULL)) != 0)
+		testutil_die(ret, "session.close");
+}
+
+void
+verify(SHARED_CONFIG *cfg, const char *name)
+{
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	int ret;
+
+	conn = cfg->conn;
+
+	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
+		testutil_die(ret, "conn.session");
+
+	if ((ret = session->verify(session, name, NULL)) != 0)
+		testutil_die(ret, "session.create");
+
+	if ((ret = session->close(session, NULL)) != 0)
+		testutil_die(ret, "session.close");
+}
diff --git a/test/cursor_order/cursor_order_ops.c b/test/cursor_order/cursor_order_ops.c
new file mode 100644
index 00000000000..c60923d9cc0
--- /dev/null
+++ b/test/cursor_order/cursor_order_ops.c
@@ -0,0 +1,361 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "cursor_order.h"
+
+static void  print_stats(SHARED_CONFIG *);
+static void *reverse_scan(void *);
+static void *append_insert(void *);
+
+typedef struct {
+	char *name;				/* object name */
+	uint64_t nops;				/* Thread op count */
+
+	WT_RAND_STATE rnd;			/* RNG */
+
+	int append_insert;			/* cursor.insert */
+	int reverse_scans;			/* cursor.prev sequences */
+	SHARED_CONFIG *cfg;
+} INFO;
+
+static INFO *run_info;
+
+int
+ops_start(SHARED_CONFIG *cfg)
+{
+	struct timeval start, stop;
+	double seconds;
+	pthread_t *tids;
+	uint64_t i, name_index, offset, total_nops;
+	int ret;
+	void *thread_ret;
+
+	tids = NULL;	/* Keep GCC 4.1 happy. */
+	total_nops = 0;
+
+	/* Create per-thread structures. */
+	if ((run_info = calloc(
+	    (size_t)(cfg->reverse_scanners + cfg->append_inserters),
+	    sizeof(*run_info))) == NULL)
+		testutil_die(errno, "calloc");
+
+	if ((tids = calloc(
+	    (size_t)(cfg->reverse_scanners + cfg->append_inserters),
+	    sizeof(*tids))) == NULL)
+		testutil_die(errno, "calloc");
+
+	/* Create the files and load the initial records. */
+	for (i = 0; i < cfg->append_inserters; ++i) {
+		run_info[i].cfg = cfg;
+		if (i == 0 || cfg->multiple_files) {
+			if ((run_info[i].name = malloc(64)) == NULL)
+				testutil_die(errno, "malloc");
+			snprintf(run_info[i].name, 64, FNAME, (int)i);
+
+			/* Vary by orders of magnitude */
+			if (cfg->vary_nops)
+				run_info[i].nops =
+				    WT_MAX(1000, cfg->max_nops >> i);
+			load(cfg, run_info[i].name);
+		} else
+			run_info[i].name = run_info[0].name;
+
+		/* Setup op count if not varying ops. */
+		if (run_info[i].nops == 0)
+			run_info[i].nops = cfg->max_nops;
+		total_nops += run_info[i].nops;
+	}
+
+	/* Setup the reverse scanner configurations */
+	for (i = 0; i < cfg->reverse_scanners; ++i) {
+		offset = i + cfg->append_inserters;
+		run_info[offset].cfg = cfg;
+		if (cfg->multiple_files) {
+			if ((run_info[offset].name = malloc(64)) == NULL)
+				testutil_die(errno, "malloc");
+			/* Have reverse scans read from tables with writes. */
+			name_index = i % cfg->append_inserters;
+			snprintf(
+			    run_info[offset].name, 64, FNAME, (int)name_index);
+
+			/* Vary by orders of magnitude */
+			if (cfg->vary_nops)
+				run_info[offset].nops =
+				    WT_MAX(1000, cfg->max_nops >> name_index);
+		} else
+			run_info[offset].name = run_info[0].name;
+
+		/* Setup op count if not varying ops. */
+		if (run_info[offset].nops == 0)
+			run_info[offset].nops = cfg->max_nops;
+		total_nops += run_info[offset].nops;
+	}
+
+	(void)gettimeofday(&start, NULL);
+
+	/* Create threads. */
+	for (i = 0; i < cfg->reverse_scanners; ++i)
+		if ((ret = pthread_create(
+		    &tids[i], NULL, reverse_scan, (void *)(uintptr_t)i)) != 0)
+			testutil_die(ret, "pthread_create");
+	for (; i < cfg->reverse_scanners + cfg->append_inserters; ++i) {
+		if ((ret = pthread_create(
+		    &tids[i], NULL, append_insert, (void *)(uintptr_t)i)) != 0)
+			testutil_die(ret, "pthread_create");
+	}
+
+	/* Wait for the threads. */
+	for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i)
+		(void)pthread_join(tids[i], &thread_ret);
+
+	(void)gettimeofday(&stop, NULL);
+	seconds = (stop.tv_sec - start.tv_sec) +
+	    (stop.tv_usec - start.tv_usec) * 1e-6;
+	fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n",
+	    seconds, (int)(((cfg->reverse_scanners + cfg->append_inserters) *
+	    total_nops) / seconds));
+
+	/* Verify the files. */
+	for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) {
+		verify(cfg, run_info[i].name);
+		if (!cfg->multiple_files)
+			break;
+	}
+
+	/* Output run statistics. */
+	print_stats(cfg);
+
+	/* Free allocated memory. */
+	for (i = 0; i < cfg->reverse_scanners + cfg->append_inserters; ++i) {
+		free(run_info[i].name);
+		if (!cfg->multiple_files)
+			break;
+	}
+
+	free(run_info);
+	free(tids);
+
+	return (0);
+}
+
+/*
+ * reverse_scan_op --
+ *	Walk a cursor back from the end of the file.
+ */
+static inline void
+reverse_scan_op(
+    SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s)
+{
+	uint64_t i;
+	int ret;
+	char *strkey;
+	uint64_t initial_key_range;
+	uint64_t prev_key, this_key;
+
+	WT_UNUSED(session);
+	WT_UNUSED(s);
+
+	/* Reset the cursor */
+	cursor->reset(cursor);
+
+	/* Save the key range. */
+	initial_key_range = cfg->key_range - cfg->append_inserters;
+
+	for (i = 0; i < cfg->reverse_scan_ops; i++) {
+		if ((ret = cursor->prev(cursor)) != 0) {
+			if (ret == WT_NOTFOUND)
+				break;
+			testutil_die(ret, "cursor.prev");
+		}
+
+		if (cfg->ftype == ROW) {
+			cursor->get_key(cursor, &strkey);
+			this_key = (uint64_t)atol(strkey);
+		} else
+			cursor->get_key(cursor, (uint64_t*)&this_key);
+
+		if (i == 0 && this_key < initial_key_range)
+			testutil_die(ret,
+			    "cursor scan start range wrong first prev %" PRIu64
+			    " initial range: %" PRIu64,
+			    this_key, initial_key_range);
+		if (i != 0 && this_key >= prev_key)
+			testutil_die(ret,
+			    "cursor scan out of order this: %" PRIu64
+			    " prev: %" PRIu64,
+			    this_key, prev_key);
+		prev_key = this_key;
+	}
+}
+
+/*
+ * reverse_scan --
+ *	Reader thread start function.
+ */
+static void *
+reverse_scan(void *arg)
+{
+	INFO *s;
+	SHARED_CONFIG *cfg;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	uint64_t i;
+	int id, ret;
+	char tid[128];
+
+	id = (int)(uintptr_t)arg;
+	s = &run_info[id];
+	cfg = s->cfg;
+	__wt_thread_id(tid, sizeof(tid));
+	__wt_random_init(&s->rnd);
+
+	printf(" reverse scan thread %2d starting: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	__wt_yield();		/* Get all the threads created. */
+
+	if ((ret = cfg->conn->open_session(
+	    cfg->conn, NULL, "isolation=snapshot", &session)) != 0)
+		testutil_die(ret, "conn.open_session");
+	if ((ret = session->open_cursor(
+	    session, s->name, NULL, NULL, &cursor)) != 0)
+		testutil_die(ret, "session.open_cursor");
+	for (i = 0; i < s->nops && !cfg->thread_finish;
+	    ++i, ++s->reverse_scans, __wt_yield())
+		reverse_scan_op(cfg, session, cursor, s);
+	if ((ret = session->close(session, NULL)) != 0)
+		testutil_die(ret, "session.close");
+
+	printf(" reverse scan thread %2d stopping: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	/* Notify all other threads to finish once the first thread is done */
+	cfg->thread_finish = true;
+
+	return (NULL);
+}
+
+/*
+ * append_insert_op --
+ *	Write operation.
+ */
+static inline void
+append_insert_op(
+    SHARED_CONFIG *cfg, WT_SESSION *session, WT_CURSOR *cursor, INFO *s)
+{
+	WT_ITEM *value, _value;
+	int64_t keyno;
+	int ret;
+	char keybuf[64], valuebuf[64];
+
+	WT_UNUSED(session);
+
+	value = &_value;
+
+	keyno = __wt_atomic_add64(&cfg->key_range, 1);
+	if (cfg->ftype == ROW) {
+		snprintf(keybuf, sizeof(keybuf), "%016u", (u_int)keyno);
+		cursor->set_key(cursor, &keybuf);
+	} else
+		cursor->set_key(cursor, (uint32_t)keyno);
+
+	++s->append_insert;
+	value->data = valuebuf;
+	if (cfg->ftype == FIX)
+		cursor->set_value(cursor, 0x10);
+	else {
+		value->size = (uint32_t)snprintf(
+		    valuebuf, sizeof(valuebuf), "XXX %37u", (u_int)keyno);
+		cursor->set_value(cursor, value);
+	}
+	if ((ret = cursor->insert(cursor)) != 0)
+		testutil_die(ret, "cursor.insert");
+}
+
+/*
+ * append_insert --
+ *	Writer thread start function.
+ */
+static void *
+append_insert(void *arg)
+{
+	INFO *s;
+	SHARED_CONFIG *cfg;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	uint64_t i;
+	int id, ret;
+	char tid[128];
+
+	id = (int)(uintptr_t)arg;
+	s = &run_info[id];
+	cfg = s->cfg;
+	__wt_thread_id(tid, sizeof(tid));
+	__wt_random_init(&s->rnd);
+
+	printf("write thread %2d starting: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	__wt_yield();		/* Get all the threads created. */
+
+	if ((ret = cfg->conn->open_session(
+	    cfg->conn, NULL, "isolation=snapshot", &session)) != 0)
+		testutil_die(ret, "conn.open_session");
+	if ((ret = session->open_cursor(
+	    session, s->name, NULL, NULL, &cursor)) != 0)
+		testutil_die(ret, "session.open_cursor");
+	for (i = 0; i < s->nops && !cfg->thread_finish; ++i, __wt_yield())
+		append_insert_op(cfg, session, cursor, s);
+	if ((ret = session->close(session, NULL)) != 0)
+		testutil_die(ret, "session.close");
+
+	printf("write thread %2d stopping: tid: %s, file: %s\n",
+	    id, tid, s->name);
+
+	/* Notify all other threads to finish once the first thread is done */
+	cfg->thread_finish = true;
+
+	return (NULL);
+}
+
+/*
+ * print_stats --
+ *	Display reverse scan/writer thread stats.
+ */
+static void
+print_stats(SHARED_CONFIG *cfg)
+{
+	INFO *s;
+	uint64_t id, total_threads;
+
+	total_threads = cfg->reverse_scanners + cfg->append_inserters;
+	s = run_info;
+	for (id = 0; id < total_threads; ++id, ++s)
+		printf("%3d: reverse scans %6d, append inserts %6d\n",
+		    (int)id, (int)s->reverse_scans, (int)s->append_insert);
+}