WT-2554 Add initial C test framework. (#2655)

* WT-2554 Add initial C test framework.

With a few sample applications. Need to do further work to cleanup
the same applications and share as much code/style as possible.

* WT-2554 Fixup build errors.

* Fix compiler warning.

* Cleanup

* Ensure each test runs in a different directory.

* Update C test suite names and add copyright.

* Move test code into subdirectories.

* Replace test_util.i with a real utility library now that we have one.

Nothing needs to be an inline function.

Mostly cherry picked from 7c18420.

* Use WiredTiger getopt in C test suite for platform portability.

* Add a header comment to each test case.

* Add a C test suite entry auto generator

* Style, KNF

* Implement review feedback.

* Build test/utility library on Windows

* Add comment to script.  Fix a printf.

41 files changed, 1164 insertions, 136 deletions

diff --git a/SConstruct b/SConstruct
index 425a531fda2..c724d94da7c 100644
--- a/SConstruct
+++ b/SConstruct
@@ -393,10 +393,16 @@ env.Append(BUILDERS={'SmokeTest' : Builder(action = builder_smoke_test)})
 
 #Build the tests and setup the "scons test" target
 
+testutil = env.Library('testutil',
+            [
+                'test/utility/misc.c',
+                'test/utility/parse_opts.c'
+            ])
+
 #Don't test bloom on Windows, its broken
 t = env.Program("t_bloom",
     "test/bloom/test_bloom.c",
-    LIBS=[wtlib] + wtlibs)
+    LIBS=[wtlib, testutil] + wtlibs)
 #env.Alias("check", env.SmokeTest(t))
 Default(t)
 
@@ -419,7 +425,7 @@ t = env.Program("t_fops",
     ["test/fops/file.c",
     "test/fops/fops.c",
     "test/fops/t.c"],
-    LIBS=[wtlib, shim] + wtlibs)
+    LIBS=[wtlib, shim, testutil] + wtlibs)
 env.Append(CPPPATH=["test/utility"])
 env.Alias("check", env.SmokeTest(t))
 Default(t)
diff --git a/build_posix/Make.subdirs b/build_posix/Make.subdirs
index 4e1f829c0c5..64749378ed1 100644
--- a/build_posix/Make.subdirs
+++ b/build_posix/Make.subdirs
@@ -25,8 +25,10 @@ examples/java JAVA
 lang/python PYTHON
 
 # Make the tests
+test/utility
 test/bloom
 test/checkpoint
+test/csuite
 test/cursor_order
 test/fops
 test/format
diff --git a/dist/s_c_test_create b/dist/s_c_test_create
new file mode 100755
index 00000000000..fd0fa809d99
--- /dev/null
+++ b/dist/s_c_test_create
@@ -0,0 +1,105 @@
+#! /bin/sh
+
+#
+# Usage: s_c_test_create test_name
+#
+# Create a new test case in the C test suite.
+# This will create the infrastructure for a new C test case.  The given
+# test name is a new directory in the C suite directory and the Makefile
+# components and C program template are created.
+#
+# Any 'make check' variations of this test should be added to the smoke.sh
+# script in the main C suite directory.
+#
+tmp=__a
+trap 'rm -f $tmp; exit 0' 0 1 2 3 13 15
+
+if [ "x$1" = "x" ]; then
+	echo "Usage: $0 test_name"
+	exit 1
+fi
+CSUITE_DIRECTORY=../test/csuite
+MAKEFILE_NAME=$CSUITE_DIRECTORY/Makefile.am
+
+TEST_NAME=$1
+
+exists=`grep $TEST_NAME $MAKEFILE_NAME`
+
+if [ "x$exists" != "x" ]; then
+	echo "Test with requested name already exists. Try another name."
+	exit 1
+fi
+
+# Create a subdirectory and stub for the new test
+mkdir $CSUITE_DIRECTORY/$TEST_NAME
+
+(cat <<EOF
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * JIRA ticket reference: 
+ * Test case description: 
+ * Failure mode: 
+ */
+
+void (*custom_die)(void) = NULL;
+
+int
+main(int argc, char *argv[])
+{
+	TEST_OPTS *opts, _opts;
+
+	opts = &_opts;
+	memset(opts, 0, sizeof(*opts));
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	/*
+	 * Insert test implementation here.
+	 */
+
+	testutil_cleanup(opts);
+
+	return (0);
+}
+EOF
+) > $CSUITE_DIRECTORY/$TEST_NAME/main.c
+
+
+# Now update the C test suite makefile to include the new test case
+
+NEW_MAKE_SECT="test_${TEST_NAME}_SOURCES = ${TEST_NAME}\/main.c\\nnoinst_PROGRAMS = test_${TEST_NAME}\\n\\n"
+
+cat $CSUITE_DIRECTORY/Makefile.am | awk \
+    "/^# Script add new line here/ && !modif { printf(\"$NEW_MAKE_SECT\"); modif=1 } {print}" > $tmp
+
+mv $tmp $CSUITE_DIRECTORY/Makefile.am
+
+exit 0
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 8debd9c3360..0cf4eb703eb 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -880,6 +880,7 @@ os
 osfhandle
 ovfl
 ownp
+pR
 packv
 pagesize
 parens
@@ -925,6 +926,7 @@ pushms
 putK
 putV
 pv
+pvA
 pwrite
 py
 qdown
diff --git a/test/bloom/Makefile.am b/test/bloom/Makefile.am
index 0592cec7e42..81a21f59882 100644
--- a/test/bloom/Makefile.am
+++ b/test/bloom/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
 t_SOURCES = test_bloom.c
-t_LDADD = $(top_builddir)/libwiredtiger.la
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/bloom/test_bloom.c b/test/bloom/test_bloom.c
index 6955813dc68..e9980cd53cb 100644
--- a/test/bloom/test_bloom.c
+++ b/test/bloom/test_bloom.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 static struct {
 	char *progname;				/* Program name */
diff --git a/test/checkpoint/Makefile.am b/test/checkpoint/Makefile.am
index cf879d046bf..2b5ba800c9c 100644
--- a/test/checkpoint/Makefile.am
+++ b/test/checkpoint/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
-t_LDADD = $(top_builddir)/libwiredtiger.la
 t_SOURCES = checkpointer.c workers.c test_checkpoint.c
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 TESTS = smoke.sh
diff --git a/test/checkpoint/test_checkpoint.h b/test/checkpoint/test_checkpoint.h
index 10e21289dd3..0d0d02447d5 100644
--- a/test/checkpoint/test_checkpoint.h
+++ b/test/checkpoint/test_checkpoint.h
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #include <signal.h>
 
diff --git a/test/csuite/Makefile.am b/test/csuite/Makefile.am
new file mode 100644
index 00000000000..6058a05431b
--- /dev/null
+++ b/test/csuite/Makefile.am
@@ -0,0 +1,21 @@
+AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
+    -I$(top_srcdir)/test/utility
+LDADD = $(top_builddir)/test/utility/libtest_util.la \
+    $(top_builddir)/libwiredtiger.la
+AM_LDFLAGS = -static
+
+test_wt1965_col_efficiency_SOURCES = wt1965_col_efficiency/main.c
+noinst_PROGRAMS = test_wt1965_col_efficiency
+
+test_wt2246_col_append_SOURCES = wt2246_col_append/main.c
+noinst_PROGRAMS += test_wt2246_col_append
+
+test_wt2535_insert_race_SOURCES = wt2535_insert_race/main.c
+noinst_PROGRAMS += test_wt2535_insert_race
+
+# Run this during a "make check" smoke test.
+TESTS = $(noinst_PROGRAMS)
+LOG_COMPILER = $(TEST_WRAPPER)
+
+clean-local:
+	rm -rf WT_TEST.* *.core
diff --git a/test/csuite/wt1965_col_efficiency/main.c b/test/csuite/wt1965_col_efficiency/main.c
new file mode 100644
index 00000000000..a7ea174460d
--- /dev/null
+++ b/test/csuite/wt1965_col_efficiency/main.c
@@ -0,0 +1,184 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * JIRA ticket reference: WT-1965
+ * Test case description: The reported issue was that column store tables
+ * exhibit high CPU usage when populated with sparse record IDs.
+ * Failure mode: It isn't simple to make this test case failure explicit since
+ * it is demonstrating an inefficiency rather than a correctness bug.
+ */
+
+void (*custom_die)(void) = NULL;
+
+#define	BUF_SIZE 256
+/* If changing field count also need to change set_value and get_value calls */
+#define	NR_FIELDS 8
+#define	NR_OBJECTS 100
+#define	NR_THREADS 4
+
+static uint64_t g_ts;
+
+/*
+ * Each thread inserts a set of keys into the record store database. The keys
+ * are generated in such a way that there are large gaps in the key range.
+ */
+static void
+*thread_func(void *arg)
+{
+	TEST_OPTS *opts;
+	WT_CURSOR *cursor, *idx_cursor;
+	WT_SESSION *session;
+	uint64_t *obj_data, thr_idx, ts = g_ts;
+	int i, o, r;
+
+	opts = (TEST_OPTS *)arg;
+	thr_idx = __wt_atomic_fetch_addv64(&opts->next_threadid, 1);
+	obj_data = calloc(
+	    (NR_OBJECTS/NR_THREADS + 1) * NR_FIELDS, sizeof(*obj_data));
+
+	testutil_check(opts->conn->open_session(
+	    opts->conn, NULL, NULL, &session));
+
+	testutil_check(session->open_cursor(
+	    session, opts->uri, NULL, NULL, &cursor));
+	testutil_check(session->open_cursor(
+	    session, "table:index", NULL, NULL, &idx_cursor));
+
+	for (r = 1; r < 10; ++r) {
+		for (o = thr_idx, i = 0;
+		    o < NR_OBJECTS; o += NR_THREADS, i += NR_FIELDS) {
+
+			testutil_check(
+			    session->begin_transaction(session, "sync=false"));
+
+			cursor->set_key(cursor, ((uint64_t)o) << 40 | r);
+			cursor->set_value(cursor, ts,
+			    obj_data[i+0], obj_data[i+1], obj_data[i+2],
+			    obj_data[i+3], obj_data[i+4], obj_data[i+5],
+			    obj_data[i+6], obj_data[i+7]);
+			testutil_check(cursor->insert(cursor));
+
+			idx_cursor->set_key(
+			    idx_cursor, ((uint64_t)o) << 40 | ts);
+			idx_cursor->set_value(idx_cursor, r);
+			testutil_check(idx_cursor->insert(idx_cursor));
+
+			testutil_check(
+			    session->commit_transaction(session, NULL));
+
+			/* change object fields */
+			++obj_data[i + ((o + r) % NR_FIELDS)];
+			++obj_data[i + ((o + r + 1) % NR_FIELDS)];
+
+			++g_ts;
+			/* 5K updates/sec */
+			usleep(1000000ULL * NR_THREADS / 5000);
+		}
+	}
+
+	testutil_check(session->close(session, NULL));
+	free(obj_data);
+	return (NULL);
+}
+
+int
+main(int argc, char *argv[])
+{
+	TEST_OPTS *opts, _opts;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	pthread_t thr[NR_THREADS];
+	size_t t;
+	uint64_t f[NR_FIELDS], r, ts;
+	int i, ret;
+	char table_format[256];
+
+	opts = &_opts;
+	memset(opts, 0, sizeof(*opts));
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	testutil_check(wiredtiger_open(opts->home, NULL,
+	    "create,cache_size=1G,checkpoint=(wait=30),"
+	    "eviction_trigger=80,eviction_target=64,eviction_dirty_target=65,"
+	    "log=(enabled,file_max=10M),"
+	    "transaction_sync=(enabled=true,method=none)", &opts->conn));
+	testutil_check(opts->conn->open_session(
+	    opts->conn, NULL, NULL, &session));
+
+	sprintf(table_format, "key_format=r,value_format=");
+	for (i = 0; i < NR_FIELDS; i++)
+		strcat(table_format, "Q");
+
+	/* recno -> timestamp + NR_FIELDS * Q */
+	testutil_check(session->create(
+	    session, opts->uri, table_format));
+	/* timestamp -> recno */
+	testutil_check(session->create(session,
+	    "table:index", "key_format=Q,value_format=Q"));
+
+	testutil_check(session->close(session, NULL));
+
+	for (t = 0; t < NR_THREADS; ++t)
+		testutil_check(pthread_create(
+		    &thr[t], NULL, thread_func, (void *)opts));
+
+	for (t = 0; t < NR_THREADS; ++t)
+		pthread_join(thr[t], NULL);
+
+	testutil_check(opts->conn->open_session(
+	    opts->conn, NULL, NULL, &session));
+
+	/* recno -> timestamp + NR_FIELDS * Q */
+	testutil_check(session->create(session, opts->uri, table_format));
+
+	testutil_check(session->open_cursor(
+	    session, opts->uri, NULL, NULL, &cursor));
+
+	while ((ret = cursor->next(cursor)) == 0) {
+		testutil_check(cursor->get_key(cursor, &r));
+		testutil_check(cursor->get_value(cursor, &ts,
+		    &f[0], &f[1], &f[2], &f[3], &f[4], &f[5], &f[6], &f[7]));
+
+		if (!opts->verbose)
+			continue;
+
+		printf("(%" PRIu64 ",%llu)\t\t%" PRIu64,
+		    (r >> 40), r & ((1ULL << 40) - 1), ts);
+
+		for (i = 0; i < NR_FIELDS; i++)
+			printf("\t%" PRIu64, f[i]);
+		printf("\n");
+	}
+
+	testutil_cleanup(opts);
+
+	return (0);
+}
diff --git a/test/csuite/wt2246_col_append/main.c b/test/csuite/wt2246_col_append/main.c
new file mode 100644
index 00000000000..12e040114b3
--- /dev/null
+++ b/test/csuite/wt2246_col_append/main.c
@@ -0,0 +1,157 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * JIRA ticket reference: WT-2246
+ * Test case description: The column-store search routine used to search the
+ * target leaf page even when the cursor is configured with append and we're
+ * allocating a record number. That was inefficient, this test case
+ * demonstrates the inefficiency.
+ * Failure mode: It isn't simple to make this test case failure explicit since
+ * it is demonstrating an inefficiency rather than a correctness bug.
+ */
+
+/* Don't move into shared function there is a cross platform solution */
+#include <signal.h>
+
+#define	MILLION		1000000
+
+void (*custom_die)(void) = NULL;
+
+/* Needs to be global for signal handling. */
+TEST_OPTS *opts;
+
+static void
+page_init(uint64_t n)
+{
+	WT_CONNECTION *conn;
+	WT_CURSOR *cursor;
+	WT_SESSION *session;
+	uint64_t recno, vrecno;
+	char buf[64];
+
+	conn = opts->conn;
+
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(
+	    session->open_cursor(session, opts->uri, NULL, "append", &cursor));
+
+	vrecno = 0;
+	buf[0] = '\2';
+	for (recno = 1;; ++recno) {
+		if (opts->table_type == TABLE_FIX)
+			cursor->set_value(cursor, buf[0]);
+		else {
+			if (recno % 3 == 0)
+				++vrecno;
+			snprintf(buf,
+			    sizeof(buf), "%" PRIu64 " VALUE ------", vrecno);
+			cursor->set_value(cursor, buf);
+		}
+		testutil_check(cursor->insert(cursor));
+		testutil_check(cursor->get_key(cursor, &opts->max_inserted_id));
+		if (opts->max_inserted_id >= n)
+			break;
+	}
+}
+
+/*
+ * TODO: Platform specific?
+ */
+static void
+onsig(int signo)
+{
+	WT_UNUSED(signo);
+	opts->running = false;
+}
+
+#define	N_APPEND_THREADS	6
+
+int
+main(int argc, char *argv[])
+{
+	TEST_OPTS _opts;
+	WT_SESSION *session;
+	clock_t ce, cs;
+	pthread_t idlist[100];
+	uint64_t i, id;
+	char buf[100];
+
+	opts = &_opts;
+	memset(opts, 0, sizeof(*opts));
+	opts->table_type = TABLE_ROW;
+	opts->n_append_threads = N_APPEND_THREADS;
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	snprintf(buf, sizeof(buf), 
+	    "create,"
+	    "cache_size=%s,"
+	    "eviction=(threads_max=5),"
+	    "statistics=(fast)",
+	    opts->table_type == TABLE_FIX ? "500MB" : "2GB");
+	testutil_check(wiredtiger_open(opts->home, NULL, buf, &opts->conn));
+	testutil_check(
+	    opts->conn->open_session(opts->conn, NULL, NULL, &session));
+	snprintf(buf, sizeof(buf),
+	    "key_format=r,value_format=%s,"
+	    "allocation_size=4K,leaf_page_max=64K",
+	    opts->table_type == TABLE_FIX ? "8t" : "S");
+	testutil_check(session->create(session, opts->uri, buf));
+	testutil_check(session->close(session, NULL));
+
+	page_init(5000);
+
+	/* Force to disk and re-open. */
+	testutil_check(opts->conn->close(opts->conn, NULL));
+	testutil_check(wiredtiger_open(opts->home, NULL, NULL, &opts->conn));
+
+	(void)signal(SIGINT, onsig);
+
+	cs = clock();
+	id = 0;
+	for (i = 0; i < opts->n_append_threads; ++i, ++id) {
+		printf("append: %" PRIu64 "\n", id);
+		testutil_check(pthread_create(
+		    &idlist[id], NULL, thread_append, (void *)opts));
+	}
+
+	for (i = 0; i < id; ++i)
+		testutil_check(pthread_join(idlist[i], NULL));
+
+	ce = clock();
+	printf("%" PRIu64 "M: %.2lf\n",
+	    opts->max_inserted_id / MILLION,
+	    (ce - cs) / (double)CLOCKS_PER_SEC);
+
+	testutil_cleanup(opts);
+	/* NOTREACHED */
+
+	return (0);
+}
diff --git a/test/csuite/wt2535_insert_race/main.c b/test/csuite/wt2535_insert_race/main.c
new file mode 100644
index 00000000000..c16d4d68952
--- /dev/null
+++ b/test/csuite/wt2535_insert_race/main.c
@@ -0,0 +1,159 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+/*
+ * JIRA ticket reference: WT-2535
+ * Test case description: This is a test case that looks for lost updates to
+ * a single record. That is multiple threads each do the same number of read
+ * modify write operations on a single record. At the end verify that the
+ * data contains the expected value.
+ * Failure mode: Check that the data is correct at the end of the run.
+ */
+
+void (*custom_die)(void) = NULL;
+
+void *thread_insert_race(void *);
+
+int
+main(int argc, char *argv[])
+{
+	TEST_OPTS *opts, _opts;
+	WT_CURSOR *c;
+	WT_SESSION *session;
+	clock_t ce, cs;
+	pthread_t id[100];
+	int i;
+	uint64_t current_value;
+
+	opts = &_opts;
+	memset(opts, 0, sizeof(*opts));
+	opts->nthreads = 10;
+	opts->nrecords = 1000;
+	opts->table_type = TABLE_ROW;
+	testutil_check(testutil_parse_opts(argc, argv, opts));
+	testutil_make_work_dir(opts->home);
+
+	testutil_check(wiredtiger_open(opts->home, NULL,
+	    "create,"
+	    "cache_size=2G,"
+	    "eviction=(threads_max=5),"
+	    "statistics=(fast)", &opts->conn));
+	testutil_check(
+	    opts->conn->open_session(opts->conn, NULL, NULL, &session));
+	testutil_check(session->create(session, opts->uri,
+	    "key_format=Q,value_format=Q,"
+	    "leaf_page_max=32k,"));
+
+	/* Create the single record. */
+	testutil_check(
+	    session->open_cursor(session, opts->uri, NULL, NULL, &c));
+	c->set_key(c, 1);
+	c->set_value(c, 0);
+	testutil_check(c->insert(c));
+	testutil_check(c->close(c));
+	cs = clock();
+	for (i = 0; i < (int)opts->nthreads; ++i) {
+		testutil_check(pthread_create(
+		    &id[i], NULL, thread_insert_race, (void *)opts));
+	}
+	while (--i >= 0)
+		testutil_check(pthread_join(id[i], NULL));
+	testutil_check(
+	    session->open_cursor(session, opts->uri, NULL, NULL, &c));
+	c->set_key(c, 1);
+	testutil_check(c->search(c));
+	c->get_value(c, &current_value);
+	if (current_value != opts->nthreads * opts->nrecords) {
+		fprintf(stderr,
+		    "ERROR: didn't get expected number of changes\n");
+		fprintf(stderr, "got: %d, expected: %d\n",
+		    (int)current_value, (int)(opts->nthreads * opts->nrecords));
+		return (EXIT_FAILURE);
+	}
+	testutil_check(session->close(session, NULL));
+	ce = clock();
+	printf("%" PRIu64 ": %.2lf\n",
+	    opts->nrecords, (ce - cs) / (double)CLOCKS_PER_SEC);
+
+	testutil_cleanup(opts);
+	return (EXIT_SUCCESS);
+}
+
+/*
+ * Append to a table in a "racy" fashion - that is attempt to insert the
+ * same record another thread is likely to also be inserting.
+ */
+void *
+thread_insert_race(void *arg)
+{
+	TEST_OPTS *opts;
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	WT_CURSOR *cursor;
+	int ret;
+	uint64_t i, value;
+
+	opts = (TEST_OPTS *)arg;
+	conn = opts->conn;
+
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(session->open_cursor(
+	    session, opts->uri, NULL, NULL, &cursor));
+
+	printf("Running insert thread\n");
+	for (i = 0; i < opts->nrecords; ++i) {
+		testutil_check(
+		    session->begin_transaction(session, "isolation=snapshot"));
+		cursor->set_key(cursor, 1);
+		cursor->search(cursor);
+		cursor->get_value(cursor, &value);
+		cursor->set_key(cursor, 1);
+		cursor->set_value(cursor, value + 1);
+		if ((ret = cursor->update(cursor)) != 0) {
+			if (ret == WT_ROLLBACK) {
+				testutil_check(session->rollback_transaction(
+				    session, NULL));
+				i--;
+				continue;
+			}
+			printf("Error in update: %d\n", ret);
+		}
+		testutil_check(session->commit_transaction(session, NULL));
+		if (i % 10000 == 0) {
+			printf("insert: %d\r", (int)i);
+			fflush(stdout);
+		}
+	}
+	if (i > 10000)
+		printf("\n");
+
+	opts->running = false;
+
+	return (NULL);
+}
diff --git a/test/cursor_order/Makefile.am b/test/cursor_order/Makefile.am
index 8afb8f122d8..c98cf1fa047 100644
--- a/test/cursor_order/Makefile.am
+++ b/test/cursor_order/Makefile.am
@@ -1,10 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = cursor_order
-cursor_order_LDADD = $(top_builddir)/libwiredtiger.la
-
 cursor_order_SOURCES = cursor_order_file.c cursor_order_ops.c cursor_order.c
+
+cursor_order_LDADD = $(top_builddir)/test/utility/libtest_util.la
+cursor_order_LDADD +=$(top_builddir)/libwiredtiger.la
 cursor_order_LDFLAGS = -static
 
 TESTS = $(noinst_PROGRAMS)
diff --git a/test/cursor_order/cursor_order.h b/test/cursor_order/cursor_order.h
index dd49fce124b..98a7d03c6f3 100644
--- a/test/cursor_order/cursor_order.h
+++ b/test/cursor_order/cursor_order.h
@@ -28,7 +28,7 @@
 
 #include <signal.h>
 
-#include "test_util.i"
+#include "test_util.h"
 
 #define	FNAME		"file:cursor_order.%03d"	/* File name */
 
diff --git a/test/fops/Makefile.am b/test/fops/Makefile.am
index a4fa7175f1b..f8a76de82bc 100644
--- a/test/fops/Makefile.am
+++ b/test/fops/Makefile.am
@@ -1,10 +1,13 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
-t_LDADD = $(top_builddir)/libwiredtiger.la
 
 t_SOURCES = thread.h file.c fops.c t.c
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/fops/thread.h b/test/fops/thread.h
index 630c2061285..89b7984a166 100644
--- a/test/fops/thread.h
+++ b/test/fops/thread.h
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #include <signal.h>
 
diff --git a/test/format/Makefile.am b/test/format/Makefile.am
index 8a2e2b49e4b..5d946e5b63d 100644
--- a/test/format/Makefile.am
+++ b/test/format/Makefile.am
@@ -1,21 +1,24 @@
-AM_CPPFLAGS = -I$(top_builddir) \
-    -I$(top_srcdir)/src/include -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 if HAVE_BERKELEY_DB
-AM_CPPFLAGS += -DHAVE_BERKELEY_DB \
-	-DBERKELEY_DB_PATH=\"$(BERKELEY_DB_PATH)\" -I$(BERKELEY_DB_PATH)/include
+AM_CPPFLAGS +=-DHAVE_BERKELEY_DB
+AM_CPPFLAGS +=-DBERKELEY_DB_PATH=\"$(BERKELEY_DB_PATH)\"
+AM_CPPFLAGS +=-I$(BERKELEY_DB_PATH)/include
 endif
 
 noinst_PROGRAMS = t
 noinst_SCRIPTS = s_dumpcmp
 t_SOURCES =\
-	config.h format.h backup.c bulk.c compact.c config.c lrt.c ops.c \
-	rebalance.c salvage.c t.c util.c wts.c
+	backup.c bulk.c compact.c config.c lrt.c ops.c rebalance.c \
+	salvage.c t.c util.c wts.c
 
 if HAVE_BERKELEY_DB
 t_SOURCES += bdb.c
 endif
 
-t_LDADD = $(top_builddir)/libwiredtiger.la
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 if HAVE_BERKELEY_DB
 t_LDADD += -L$(BERKELEY_DB_PATH)/lib -ldb
 endif
diff --git a/test/format/format.h b/test/format/format.h
index beaabe7e83c..8fd9b113311 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #ifdef BDB
 #include <assert.h>
diff --git a/test/huge/Makefile.am b/test/huge/Makefile.am
index 151d3a40dd4..894bff5eace 100644
--- a/test/huge/Makefile.am
+++ b/test/huge/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
 t_SOURCES = huge.c
-t_LDADD = $(top_builddir)/libwiredtiger.la
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/huge/huge.c b/test/huge/huge.c
index e7bfd08882f..1e104a705f2 100644
--- a/test/huge/huge.c
+++ b/test/huge/huge.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 static char home[512];				/* Program working dir */
 static const char *progname;			/* Program name */
diff --git a/test/manydbs/Makefile.am b/test/manydbs/Makefile.am
index d347868aa4f..2bc47ad7f2e 100644
--- a/test/manydbs/Makefile.am
+++ b/test/manydbs/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
 t_SOURCES = manydbs.c
-t_LDADD = $(top_builddir)/libwiredtiger.la
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/manydbs/manydbs.c b/test/manydbs/manydbs.c
index 4ab455f3620..d9639198c34 100644
--- a/test/manydbs/manydbs.c
+++ b/test/manydbs/manydbs.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #define	HOME_SIZE	512
 #define	HOME_BASE	"WT_TEST"
diff --git a/test/packing/Makefile.am b/test/packing/Makefile.am
index 0e7c8cc8b2e..c9128100cc3 100644
--- a/test/packing/Makefile.am
+++ b/test/packing/Makefile.am
@@ -1,8 +1,11 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = intpack-test intpack-test2 intpack-test3 packing-test
-LDADD = $(top_builddir)/libwiredtiger.la
+
+LDADD = $(top_builddir)/test/utility/libtest_util.la
+LDADD +=$(top_builddir)/libwiredtiger.la
 LDFLAGS = -static
 
 TESTS = smoke.sh
diff --git a/test/packing/intpack-test.c b/test/packing/intpack-test.c
index 6412ed296aa..76851b38e35 100644
--- a/test/packing/intpack-test.c
+++ b/test/packing/intpack-test.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 void (*custom_die)(void) = NULL;
 
diff --git a/test/packing/intpack-test2.c b/test/packing/intpack-test2.c
index e9443ad7ed1..a7d31329069 100644
--- a/test/packing/intpack-test2.c
+++ b/test/packing/intpack-test2.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 void (*custom_die)(void) = NULL;
 
diff --git a/test/packing/intpack-test3.c b/test/packing/intpack-test3.c
index 328b45d1bf7..aac0178578f 100644
--- a/test/packing/intpack-test3.c
+++ b/test/packing/intpack-test3.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 void (*custom_die)(void) = NULL;
 
@@ -42,6 +42,7 @@ test_value(int64_t val)
 	uint64_t uinput, uoutput;
 	size_t used_len;
 
+	soutput = 0;	/* -Werror=maybe-uninitialized */
 	sinput = val;
 	soutput = 0;	/* Make GCC happy. */
 	p = buf;
diff --git a/test/packing/packing-test.c b/test/packing/packing-test.c
index 706eeb0935c..f251c17eb67 100644
--- a/test/packing/packing-test.c
+++ b/test/packing/packing-test.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 void (*custom_die)(void) = NULL;
 
diff --git a/test/readonly/Makefile.am b/test/readonly/Makefile.am
index 8028e2ab845..84092e76f02 100644
--- a/test/readonly/Makefile.am
+++ b/test/readonly/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
 t_SOURCES = readonly.c
-t_LDADD = $(top_builddir)/libwiredtiger.la
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/readonly/readonly.c b/test/readonly/readonly.c
index a35e7ee23fc..402b99d7d29 100644
--- a/test/readonly/readonly.c
+++ b/test/readonly/readonly.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #include <sys/wait.h>
 
diff --git a/test/recovery/Makefile.am b/test/recovery/Makefile.am
index 6865d5edf3e..19fc48dce47 100644
--- a/test/recovery/Makefile.am
+++ b/test/recovery/Makefile.am
@@ -1,13 +1,16 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = random-abort truncated-log
 random_abort_SOURCES = random-abort.c
-random_abort_LDADD = $(top_builddir)/libwiredtiger.la
+random_abort_LDADD = $(top_builddir)/test/utility/libtest_util.la
+random_abort_LDADD +=$(top_builddir)/libwiredtiger.la
 random_abort_LDFLAGS = -static
 
 truncated_log_SOURCES = truncated-log.c
-truncated_log_LDADD = $(top_builddir)/libwiredtiger.la
+truncated_log_LDADD = $(top_builddir)/test/utility/libtest_util.la
+truncated_log_LDADD +=$(top_builddir)/libwiredtiger.la
 truncated_log_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/recovery/random-abort.c b/test/recovery/random-abort.c
index cd7d1b08708..33597245966 100644
--- a/test/recovery/random-abort.c
+++ b/test/recovery/random-abort.c
@@ -26,19 +26,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include "test_util.h"
+
 #include <sys/wait.h>
-#include <errno.h>
 #include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#ifndef _WIN32
-#include <unistd.h>
-#endif
-
-#include <wiredtiger.h>
-
-#include "test_util.i"
 
 static char home[512];			/* Program working dir */
 static const char *progname;		/* Program name */
diff --git a/test/recovery/truncated-log.c b/test/recovery/truncated-log.c
index e099873e5b9..3b99ea2c932 100644
--- a/test/recovery/truncated-log.c
+++ b/test/recovery/truncated-log.c
@@ -26,23 +26,15 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
+#include "test_util.h"
+
 #include <sys/wait.h>
-#include <errno.h>
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#ifndef _WIN32
-#include <unistd.h>
-#else
+
+#ifdef _WIN32
 /* snprintf is not supported on <= VS2013 */
 #define	snprintf _snprintf
 #endif
 
-#include <wiredtiger.h>
-
-#include "test_util.i"
-
 static char home[512];			/* Program working dir */
 static const char *progname;		/* Program name */
 static const char * const uri = "table:main";
diff --git a/test/salvage/Makefile.am b/test/salvage/Makefile.am
index 0fd46aefcb1..a3c49b9c41a 100644
--- a/test/salvage/Makefile.am
+++ b/test/salvage/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
 t_SOURCES = salvage.c
-t_LDADD = $(top_builddir)/libwiredtiger.la
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 # Run this during a "make check" smoke test.
diff --git a/test/salvage/salvage.c b/test/salvage/salvage.c
index f264be99e2b..497902e07b8 100644
--- a/test/salvage/salvage.c
+++ b/test/salvage/salvage.c
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #include <assert.h>
 
diff --git a/test/thread/Makefile.am b/test/thread/Makefile.am
index ead783185f8..58b715d4a80 100644
--- a/test/thread/Makefile.am
+++ b/test/thread/Makefile.am
@@ -1,9 +1,12 @@
-AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include \
-    -I$(top_srcdir)/test/utility
+AM_CPPFLAGS = -I$(top_builddir)
+AM_CPPFLAGS +=-I$(top_srcdir)/src/include
+AM_CPPFLAGS +=-I$(top_srcdir)/test/utility
 
 noinst_PROGRAMS = t
-t_LDADD = $(top_builddir)/libwiredtiger.la
-t_SOURCES = thread.h file.c rw.c stats.c t.c
+t_SOURCES = file.c rw.c stats.c t.c
+
+t_LDADD = $(top_builddir)/test/utility/libtest_util.la
+t_LDADD +=$(top_builddir)/libwiredtiger.la
 t_LDFLAGS = -static
 
 TESTS = smoke.sh
diff --git a/test/thread/thread.h b/test/thread/thread.h
index d5f0f42ea35..edcb919ec32 100644
--- a/test/thread/thread.h
+++ b/test/thread/thread.h
@@ -26,7 +26,7 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include "test_util.i"
+#include "test_util.h"
 
 #include <signal.h>
 
diff --git a/test/utility/Makefile.am b/test/utility/Makefile.am
new file mode 100644
index 00000000000..a2923eb41a8
--- /dev/null
+++ b/test/utility/Makefile.am
@@ -0,0 +1,4 @@
+AM_CPPFLAGS = -I$(top_builddir) -I$(top_srcdir)/src/include
+
+libtest_util_la_SOURCES = misc.c parse_opts.c thread.c
+noinst_LTLIBRARIES = libtest_util.la
diff --git a/test/utility/test_util.i b/test/utility/misc.c
index 833eddd87aa..74e0bff0aa9 100644
--- a/test/utility/test_util.i
+++ b/test/utility/misc.c
@@ -25,37 +25,13 @@
  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  * OTHER DEALINGS IN THE SOFTWARE.
  */
-#include "wt_internal.h"			/* For __wt_XXX */
-
-#ifdef _WIN32
-#include "windows_shim.h"
-#endif
-
-#ifdef _WIN32
-	#define DIR_DELIM '\\'
-	#define RM_COMMAND "rd /s /q "
-#else
-	#define	DIR_DELIM '/'
-	#define RM_COMMAND "rm -rf "
-#endif
-
-#define	DEFAULT_DIR "WT_TEST"
-#define	MKDIR_COMMAND "mkdir "
-
-/* Allow tests to add their own death handling. */
-extern void (*custom_die)(void);
-
-static void	 testutil_die(int, const char *, ...)
-#if defined(__GNUC__)
-__attribute__((__noreturn__))
-#endif
-;
+#include "test_util.h"
 
 /*
  * die --
  *	Report an error and quit.
  */
-static void
+void
 testutil_die(int e, const char *fmt, ...)
 {
 	va_list ap;
@@ -77,32 +53,11 @@ testutil_die(int e, const char *fmt, ...)
 }
 
 /*
- * testutil_check --
- *	Complain and quit if a function call fails.
- */
-#define	testutil_check(call) do {					\
-	int __r;							\
-	if ((__r = (call)) != 0)					\
-		testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\
-} while (0)
-
-/*
- * testutil_checkfmt --
- *	Complain and quit if a function call fails, with additional arguments.
- */
-#define	testutil_checkfmt(call, fmt, ...) do {				\
-	int __r;							\
-	if ((__r = (call)) != 0)					\
-		testutil_die(__r, "%s/%d: %s: " fmt,			\
-		    __func__, __LINE__, #call, __VA_ARGS__);		\
-} while (0)
-
-/*
  * testutil_work_dir_from_path --
  *	Takes a buffer, its size and the intended work directory.
  *	Creates the full intended work directory in buffer.
  */
-static inline void
+void
 testutil_work_dir_from_path(char *buffer, size_t len, const char *dir)
 {
 	/* If no directory is provided, use the default. */
@@ -120,7 +75,7 @@ testutil_work_dir_from_path(char *buffer, size_t len, const char *dir)
  * testutil_clean_work_dir --
  *	Remove the work directory.
  */
-static inline void
+void
 testutil_clean_work_dir(char *dir)
 {
 	size_t len;
@@ -143,7 +98,7 @@ testutil_clean_work_dir(char *dir)
  * testutil_make_work_dir --
  *	Delete the existing work directory, then create a new one.
  */
-static inline void
+void
 testutil_make_work_dir(char *dir)
 {
 	size_t len;
@@ -165,10 +120,32 @@ testutil_make_work_dir(char *dir)
 }
 
 /*
+ * testutil_cleanup --
+ *	Delete the existing work directory and free the options structure.
+ */
+void
+testutil_cleanup(TEST_OPTS *opts)
+{
+	if (opts->conn != NULL)
+		testutil_check(opts->conn->close(opts->conn, NULL));
+
+	if (!opts->preserve)
+		testutil_clean_work_dir(opts->home);
+
+	if (opts->conn_config != NULL)
+		free(opts->conn_config);
+	if (opts->table_config != NULL)
+		free(opts->table_config);
+	if (opts->uri != NULL)
+		free(opts->uri);
+	free(opts->home);
+}
+
+/*
  * dcalloc --
  *	Call calloc, dying on failure.
  */
-static inline void *
+void *
 dcalloc(size_t number, size_t size)
 {
 	void *p;
@@ -182,7 +159,7 @@ dcalloc(size_t number, size_t size)
  * dmalloc --
  *	Call malloc, dying on failure.
  */
-static inline void *
+void *
 dmalloc(size_t len)
 {
 	void *p;
@@ -196,7 +173,7 @@ dmalloc(size_t len)
  * drealloc --
  *	Call realloc, dying on failure.
  */
-static inline void *
+void *
 drealloc(void *p, size_t len)
 {
 	void *t;
@@ -209,7 +186,7 @@ drealloc(void *p, size_t len)
  * dstrdup --
  *	Call strdup, dying on failure.
  */
-static inline void *
+void *
 dstrdup(const void *str)
 {
 	char *p;
diff --git a/test/utility/parse_opts.c b/test/utility/parse_opts.c
new file mode 100644
index 00000000000..0e9040e6fef
--- /dev/null
+++ b/test/utility/parse_opts.c
@@ -0,0 +1,133 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "test_util.h"
+
+extern int   __wt_opterr;		/* if error message should be printed */
+extern int   __wt_optind;		/* index into parent argv vector */
+extern int   __wt_optopt;		/* character checked for validity */
+extern int   __wt_optreset;		/* reset getopt */
+extern char *__wt_optarg;		/* argument associated with option */
+
+/*
+ * testutil_parse_opts --
+ *    Parse command line options for a test case.
+ */
+int
+testutil_parse_opts(int argc, char *argv[], TEST_OPTS *opts)
+{
+	int ch;
+	size_t len;
+
+	opts->preserve = false;
+	opts->running = true;
+	opts->verbose = false;
+
+	if ((opts->progname = strrchr(argv[0], '/')) == NULL)
+		opts->progname = argv[0];
+	else
+		++opts->progname;
+
+	while ((ch =
+	    __wt_getopt(opts->progname,
+		argc, argv, "A:h:n:o:pR:T:t:vW:")) != EOF)
+		switch (ch) {
+		case 'A': /* Number of append threads */
+			opts->n_append_threads = (uint64_t)atoll(__wt_optarg);
+			break;
+		case 'h': /* Home directory */
+			opts->home = __wt_optarg;
+			break;
+		case 'n': /* Number of records */
+			opts->nrecords = (uint64_t)atoll(__wt_optarg);
+			break;
+		case 'o': /* Number of operations */
+			opts->nops = (uint64_t)atoll(__wt_optarg);
+			break;
+		case 'p': /* Preserve directory contents */
+			opts->preserve = true;
+			break;
+		case 'R': /* Number of reader threads */
+			opts->n_read_threads = (uint64_t)atoll(__wt_optarg);
+			break;
+		case 'T': /* Number of threads */
+			opts->nthreads = (uint64_t)atoll(__wt_optarg);
+			break;
+		case 't': /* Table type */
+			switch (__wt_optarg[0]) {
+			case 'C':
+			case 'c':
+				opts->table_type = TABLE_COL;
+				break;
+			case 'F':
+			case 'f':
+				opts->table_type = TABLE_FIX;
+				break;
+			case 'R':
+			case 'r':
+				opts->table_type = TABLE_ROW;
+				break;
+			}
+			break;
+		case 'v':
+			opts->verbose = true;
+			break;
+		case 'W': /* Number of writer threads */
+			opts->n_write_threads = (uint64_t)atoll(__wt_optarg);
+			break;
+		case '?':
+		default:
+			(void)fprintf(stderr, "usage: %s "
+			    "[-A append thread count] "
+			    "[-h home] "
+			    "[-n record count] "
+			    "[-o op count] "
+			    "[-p] "
+			    "[-R read thread count] "
+			    "[-T thread count] "
+			    "[-t c|f|r table type] "
+			    "[-v] "
+			    "[-W write thread count] ",
+			    opts->progname);
+			return (1);
+		}
+
+	/*
+	 * Setup the home directory. It needs to be unique for every test
+	 * or the auto make parallel tester gets upset.
+	 */
+	len = snprintf(NULL, 0, "WT_TEST.%s", opts->progname) + 1;
+	opts->home = (char *)malloc(len);
+	snprintf(opts->home, len, "WT_TEST.%s", opts->progname);
+
+	/* Setup the default URI string */
+	len = snprintf(NULL, 0, "table:%s", opts->progname) + 1;
+	opts->uri = (char *)malloc(len);
+	snprintf(opts->uri, len, "table:%s", opts->progname);
+
+	return (0);
+}
diff --git a/test/utility/test_util.h b/test/utility/test_util.h
new file mode 100644
index 00000000000..140ce5567db
--- /dev/null
+++ b/test/utility/test_util.h
@@ -0,0 +1,118 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+#include "wt_internal.h"			/* For __wt_XXX */
+
+#ifdef _WIN32
+	#define DIR_DELIM '\\'
+	#define RM_COMMAND "rd /s /q "
+#else
+	#define	DIR_DELIM '/'
+	#define RM_COMMAND "rm -rf "
+#endif
+
+#define	DEFAULT_DIR "WT_TEST"
+#define	MKDIR_COMMAND "mkdir "
+
+#ifdef _WIN32
+#include "windows_shim.h"
+#endif
+
+/* Generic option parsing structure shared by all test cases. */
+typedef struct {
+	char  *home;
+	char  *progname;
+	enum {	TABLE_COL=1,	/* Fixed-length column store */
+		TABLE_FIX=2,	/* Variable-length column store */
+		TABLE_ROW=3	/* Row-store */
+	} table_type;
+	bool	     preserve;			/* Don't remove files on exit */
+	bool	     verbose;			/* Run in verbose mode */
+	uint64_t     nrecords;			/* Number of records */
+	uint64_t     nops;			/* Number of operations */
+	uint64_t     nthreads;			/* Number of threads */
+	uint64_t     n_append_threads;		/* Number of append threads */
+	uint64_t     n_read_threads;		/* Number of read threads */
+	uint64_t     n_write_threads;		/* Number of write threads */
+
+	/*
+	 * Fields commonly shared within a test program. The test cleanup
+	 * function will attempt to automatically free and close non-null
+	 * resources.
+	 */
+	WT_CONNECTION *conn;
+	char	  *conn_config;
+	WT_SESSION    *session;
+	bool	   running;
+	char	  *table_config;
+	char	  *uri;
+	volatile uint64_t   next_threadid;
+	uint64_t   max_inserted_id;
+} TEST_OPTS;
+
+/*
+ * testutil_check --
+ *	Complain and quit if a function call fails.
+ */
+#define	testutil_check(call) do {					\
+	int __r;							\
+	if ((__r = (call)) != 0)					\
+		testutil_die(__r, "%s/%d: %s", __func__, __LINE__, #call);\
+} while (0)
+
+/*
+ * testutil_checkfmt --
+ *	Complain and quit if a function call fails, with additional arguments.
+ */
+#define	testutil_checkfmt(call, fmt, ...) do {				\
+	int __r;							\
+	if ((__r = (call)) != 0)					\
+		testutil_die(__r, "%s/%d: %s: " fmt,			\
+		    __func__, __LINE__, #call, __VA_ARGS__);		\
+} while (0)
+
+/* Allow tests to add their own death handling. */
+extern void (*custom_die)(void);
+
+void testutil_die(int, const char *, ...)
+#if defined(__GNUC__)
+__attribute__((__noreturn__))
+#endif
+;
+
+void *dcalloc(size_t, size_t);
+void *dmalloc(size_t);
+void *drealloc(void *, size_t);
+void *dstrdup(const void *);
+void  testutil_clean_work_dir(char *);
+void  testutil_cleanup(TEST_OPTS *);
+void  testutil_make_work_dir(char *);
+int   testutil_parse_opts(int, char *[], TEST_OPTS *);
+void  testutil_work_dir_from_path(char *, size_t, const char *);
+void *thread_append(void *);
+void *thread_insert_append(void *);
+void *thread_prev(void *);
diff --git a/test/utility/thread.c b/test/utility/thread.c
new file mode 100644
index 00000000000..c8665f376d9
--- /dev/null
+++ b/test/utility/thread.c
@@ -0,0 +1,141 @@
+/*-
+ * Public Domain 2014-2016 MongoDB, Inc.
+ * Public Domain 2008-2014 WiredTiger, Inc.
+ *
+ * This is free and unencumbered software released into the public domain.
+ *
+ * Anyone is free to copy, modify, publish, use, compile, sell, or
+ * distribute this software, either in source code form or as a compiled
+ * binary, for any purpose, commercial or non-commercial, and by any
+ * means.
+ *
+ * In jurisdictions that recognize copyright laws, the author or authors
+ * of this software dedicate any and all copyright interest in the
+ * software to the public domain. We make this dedication for the benefit
+ * of the public at large and to the detriment of our heirs and
+ * successors. We intend this dedication to be an overt act of
+ * relinquishment in perpetuity of all present and future rights to this
+ * software under copyright law.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include "test_util.h"
+
+/*
+ * A thread dedicated to appending records into a table. Works with fixed
+ * length column stores and variable length column stores.
+ * One thread (the first thread created by an application) checks for a
+ * terminating condition after each insert.
+ */
+void *
+thread_append(void *arg)
+{
+	TEST_OPTS *opts;
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	WT_CURSOR *cursor;
+	uint64_t id, recno;
+	char buf[64];
+
+	opts = (TEST_OPTS *)arg;
+	conn = opts->conn;
+
+	id = __wt_atomic_fetch_addv64(&opts->next_threadid, 1);
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(
+	    session->open_cursor(session, opts->uri, NULL, "append", &cursor));
+
+	buf[0] = '\2';
+	for (recno = 1; opts->running; ++recno) {
+		if (opts->table_type == TABLE_FIX)
+			cursor->set_value(cursor, buf[0]);
+		else {
+			snprintf(buf, sizeof(buf),
+			    "%" PRIu64 " VALUE ------", recno);
+			cursor->set_value(cursor, buf);
+		}
+		testutil_check(cursor->insert(cursor));
+		if (id == 0) {
+			testutil_check(
+			    cursor->get_key(cursor, &opts->max_inserted_id));
+			if (opts->max_inserted_id >= opts->nrecords)
+				opts->running = false;
+		}
+	}
+
+	return (NULL);
+}
+
+/*
+ * Append into a row store table.
+ */
+void *
+thread_insert_append(void *arg)
+{
+	TEST_OPTS *opts;
+	WT_CONNECTION *conn;
+	WT_SESSION *session;
+	WT_CURSOR *cursor;
+	uint64_t i;
+	char kbuf[64];
+
+	opts = (TEST_OPTS *)arg;
+	conn = opts->conn;
+
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(session->open_cursor(
+	    session, opts->uri, NULL, NULL, &cursor));
+
+	for (i = 0; i < opts->nrecords; ++i) {
+		snprintf(kbuf, sizeof(kbuf), "%010d KEY------", (int)i);
+		cursor->set_key(cursor, kbuf);
+		cursor->set_value(cursor, "========== VALUE =======");
+		testutil_check(cursor->insert(cursor));
+		if (i % 100000 == 0) {
+			printf("insert: %d\r", (int)i);
+			fflush(stdout);
+		}
+	}
+	printf("\n");
+
+	opts->running = false;
+
+	return (NULL);
+}
+
+/*
+ * Repeatedly walk backwards through the records in a table.
+ */
+void *
+thread_prev(void *arg)
+{
+	TEST_OPTS *opts;
+	WT_SESSION *session;
+	WT_CURSOR *cursor;
+	int ret;
+
+	opts = (TEST_OPTS *)arg;
+	ret = 0;
+
+	testutil_check(
+	    opts->conn->open_session(opts->conn, NULL, NULL, &session));
+	testutil_check(
+	    session->open_cursor(session, opts->uri, NULL, NULL, &cursor));
+	while (opts->running) {
+		while (opts->running && (ret = cursor->prev(cursor)) == 0)
+			;
+		if (ret == WT_NOTFOUND)
+			ret = 0;
+		testutil_check(ret);
+	}
+
+	testutil_check(session->close(session, NULL));
+	return (NULL);
+}