Merge branch 'develop' into mongodb-3.4mongodb-3.3.6

13 files changed, 904 insertions, 564 deletions

diff --git a/test/format/backup.c b/test/format/backup.c
index 2b1463bd0e3..69fdf771de9 100644
--- a/test/format/backup.c
+++ b/test/format/backup.c
@@ -38,7 +38,7 @@ check_copy(void)
 	WT_CONNECTION *conn;
 	WT_SESSION *session;
 
-	wts_open(g.home_backup, 0, &conn);
+	wts_open(g.home_backup, false, &conn);
 
 	testutil_checkfmt(
 	    conn->open_session(conn, NULL, NULL, &session),
@@ -53,27 +53,30 @@ check_copy(void)
 
 /*
  * copy_file --
- *	Copy a single file into the backup directory.
+ *	Copy a single file into the backup directories.
  */
 static void
-copy_file(const char *name)
+copy_file(WT_SESSION *session, const char *name)
 {
 	size_t len;
-	char *cmd;
-
-	len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20;
-	cmd = dmalloc(len);
-	(void)snprintf(cmd, len,
-	    "cp %s/%s %s/%s", g.home, name, g.home_backup, name);
-	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
-	free(cmd);
-
-	len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20;
-	cmd = dmalloc(len);
-	(void)snprintf(cmd, len,
-	    "cp %s/%s %s/%s", g.home, name, g.home_backup2, name);
-	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
-	free(cmd);
+	char *first, *second;
+
+	len = strlen("BACKUP") + strlen(name) + 10;
+	first = dmalloc(len);
+	(void)snprintf(first, len, "BACKUP/%s", name);
+	testutil_check(__wt_copy_and_sync(session, name, first));
+
+	/*
+	 * Save another copy of the original file to make debugging recovery
+	 * errors easier.
+	 */
+	len = strlen("BACKUP_COPY") + strlen(name) + 10;
+	second = dmalloc(len);
+	(void)snprintf(second, len, "BACKUP_COPY/%s", name);
+	testutil_check(__wt_copy_and_sync(session, first, second));
+
+	free(first);
+	free(second);
 }
 
 /*
@@ -85,10 +88,11 @@ backup(void *arg)
 {
 	WT_CONNECTION *conn;
 	WT_CURSOR *backup_cursor;
+	WT_DECL_RET;
 	WT_SESSION *session;
-	u_int period;
-	int ret;
-	const char *key;
+	u_int incremental, period;
+	bool full;
+	const char *config, *key;
 
 	(void)(arg);
 
@@ -102,48 +106,86 @@ backup(void *arg)
 	testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
 	/*
-	 * Perform a backup at somewhere under 10 seconds (so we get at
-	 * least one done), and then at 45 second intervals.
+	 * Perform a full backup at somewhere under 10 seconds (that way there's
+	 * at least one), then at larger intervals, optionally do incremental
+	 * backups between full backups.
 	 */
-	for (period = mmrand(NULL, 1, 10);; period = 45) {
+	incremental = 0;
+	for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) {
 		/* Sleep for short periods so we don't make the run wait. */
 		while (period > 0 && !g.workers_finished) {
 			--period;
 			sleep(1);
 		}
-		if (g.workers_finished)
-			break;
 
-		/* Lock out named checkpoints */
+		/*
+		 * We can't drop named checkpoints while there's a backup in
+		 * progress, serialize backups with named checkpoints. Wait
+		 * for the checkpoint to complete, otherwise backups might be
+		 * starved out.
+		 */
 		testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
+		if (g.workers_finished) {
+			testutil_check(pthread_rwlock_unlock(&g.backup_lock));
+			break;
+		}
 
-		/* Re-create the backup directory. */
-		testutil_checkfmt(
-		    system(g.home_backup_init),
-		    "%s", "backup directory creation failed");
+		if (incremental) {
+			config = "target=(\"log:\")";
+			full = false;
+		} else {
+			/* Re-create the backup directory. */
+			testutil_checkfmt(
+			    system(g.home_backup_init),
+			    "%s", "backup directory creation failed");
+
+			config = NULL;
+			full = true;
+		}
 
 		/*
-		 * open_cursor can return EBUSY if a metadata operation is
-		 * currently happening - retry in that case.
+		 * open_cursor can return EBUSY if concurrent with a metadata
+		 * operation, retry in that case.
 		 */
-		while ((ret = session->open_cursor(session,
-		    "backup:", NULL, NULL, &backup_cursor)) == EBUSY)
-			sleep(1);
+		while ((ret = session->open_cursor(
+		    session, "backup:", NULL, config, &backup_cursor)) == EBUSY)
+			__wt_yield();
 		if (ret != 0)
 			testutil_die(ret, "session.open_cursor: backup");
 
 		while ((ret = backup_cursor->next(backup_cursor)) == 0) {
 			testutil_check(
 			    backup_cursor->get_key(backup_cursor, &key));
-			copy_file(key);
+			copy_file(session, key);
 		}
+		if (ret != WT_NOTFOUND)
+			testutil_die(ret, "backup-cursor");
+
+		/* After an incremental backup, truncate the log files. */
+		if (incremental)
+			testutil_check(session->truncate(
+			    session, "log:", backup_cursor, NULL, NULL));
 
 		testutil_check(backup_cursor->close(backup_cursor));
 		testutil_check(pthread_rwlock_unlock(&g.backup_lock));
 
-		check_copy();
+		/*
+		 * If automatic log archival isn't configured, optionally do
+		 * incremental backups after each full backup. If we're not
+		 * doing any more incrementals, verify the backup (we can't
+		 * verify intermediate states, once we perform recovery on the
+		 * backup database, we can't do any more incremental backups).
+		 */
+		if (full)
+			incremental =
+			    g.c_logging_archive ? 1 : mmrand(NULL, 1, 5);
+		if (--incremental == 0)
+			check_copy();
 	}
 
+	if (incremental != 0)
+		check_copy();
+
 	testutil_check(session->close(session, NULL));
 
 	return (NULL);
diff --git a/test/format/bdb.c b/test/format/bdb.c
index 823fc8ff888..48229cfd5e7 100644
--- a/test/format/bdb.c
+++ b/test/format/bdb.c
@@ -30,7 +30,7 @@
 #include "format.h"
 
 static DBT key, value;
-static uint8_t *keybuf;
+static WT_ITEM keyitem;
 
 static int
 bdb_compare_reverse(DB *dbp, const DBT *k1, const DBT *k2
@@ -78,7 +78,7 @@ bdb_open(void)
 	assert(db->cursor(db, NULL, &dbc, 0) == 0);
 	g.dbc = dbc;
 
-	key_gen_setup(&keybuf);
+	key_gen_setup(&keyitem);
 }
 
 void
@@ -95,8 +95,7 @@ bdb_close(void)
 	assert(db->close(db, 0) == 0);
 	assert(dbenv->close(dbenv, 0) == 0);
 
-	free(keybuf);
-	keybuf = NULL;
+	free(keyitem.mem);
 }
 
 void
@@ -144,12 +143,11 @@ void
 bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp)
 {
 	DBC *dbc = g.dbc;
-	size_t size;
 	int ret;
 
-	key_gen(keybuf, &size, keyno);
-	key.data = keybuf;
-	key.size = (uint32_t)size;
+	key_gen(&keyitem, keyno);
+	key.data = (void *)keyitem.data;
+	key.size = keyitem.size;
 
 	*notfoundp = 0;
 	if ((ret = dbc->get(dbc, &key, &value, DB_SET)) != 0) {
@@ -165,7 +163,7 @@ bdb_read(uint64_t keyno, void *valuep, size_t *valuesizep, int *notfoundp)
 
 void
 bdb_update(const void *arg_key, size_t arg_key_size,
-    const void *arg_value, size_t arg_value_size, int *notfoundp)
+    const void *arg_value, size_t arg_value_size)
 {
 	DBC *dbc = g.dbc;
 	int ret;
@@ -175,15 +173,10 @@ bdb_update(const void *arg_key, size_t arg_key_size,
 	value.data = (void *)arg_value;
 	value.size = (uint32_t)arg_value_size;
 
-	*notfoundp = 0;
-	if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0) {
-		if (ret != DB_NOTFOUND) {
-			testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}",
-			    (int)key.size, (char *)key.data,
-			    (int)value.size, (char *)value.data);
-		}
-		*notfoundp = 1;
-	}
+	if ((ret = dbc->put(dbc, &key, &value, DB_KEYFIRST)) != 0)
+		testutil_die(ret, "dbc.put: DB_KEYFIRST: {%.*s}{%.*s}",
+		    (int)key.size, (char *)key.data,
+		    (int)value.size, (char *)value.data);
 }
 
 void
@@ -193,9 +186,9 @@ bdb_remove(uint64_t keyno, int *notfoundp)
 	size_t size;
 	int ret;
 
-	key_gen(keybuf, &size, keyno);
-	key.data = keybuf;
-	key.size = (uint32_t)size;
+	key_gen(&keyitem, keyno);
+	key.data = (void *)keyitem.data;
+	key.size = keyitem.size;
 
 	bdb_read(keyno, &value.data, &size, notfoundp);
 	value.size = (uint32_t)size;
diff --git a/test/format/bulk.c b/test/format/bulk.c
index 64b005d294f..dab23bed404 100644
--- a/test/format/bulk.c
+++ b/test/format/bulk.c
@@ -33,13 +33,12 @@ wts_load(void)
 {
 	WT_CONNECTION *conn;
 	WT_CURSOR *cursor;
+	WT_DECL_RET;
 	WT_ITEM key, value;
 	WT_SESSION *session;
-	uint8_t *keybuf, *valbuf;
 	bool is_bulk;
 
 	conn = g.wts_conn;
-	keybuf = valbuf = NULL;
 
 	testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
@@ -63,8 +62,8 @@ wts_load(void)
 	    is_bulk ? "bulk,append" : NULL, &cursor));
 
 	/* Set up the key/value buffers. */
-	key_gen_setup(&keybuf);
-	val_gen_setup(NULL, &valbuf);
+	key_gen_setup(&key);
+	val_gen_setup(NULL, &value);
 
 	for (;;) {
 		if (++g.key_cnt > g.c_rows) {
@@ -73,13 +72,11 @@ wts_load(void)
 		}
 
 		/* Report on progress every 100 inserts. */
-		if (g.key_cnt % 100 == 0)
+		if (g.key_cnt % 1000 == 0)
 			track("bulk load", g.key_cnt, NULL);
 
-		key_gen(keybuf, &key.size, (uint64_t)g.key_cnt);
-		key.data = keybuf;
-		val_gen(NULL, valbuf, &value.size, (uint64_t)g.key_cnt);
-		value.data = valbuf;
+		key_gen(&key, g.key_cnt);
+		val_gen(NULL, &value, g.key_cnt);
 
 		switch (g.type) {
 		case FIX:
@@ -88,7 +85,7 @@ wts_load(void)
 			cursor->set_value(cursor, *(uint8_t *)value.data);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {0x%02" PRIx8 "}",
+				    "%-10s %" PRIu64 " {0x%02" PRIx8 "}",
 				    "bulk V",
 				    g.key_cnt, ((uint8_t *)value.data)[0]);
 			break;
@@ -98,7 +95,7 @@ wts_load(void)
 			cursor->set_value(cursor, &value);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk V",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk V",
 				    g.key_cnt,
 				    (int)value.size, (char *)value.data);
 			break;
@@ -106,18 +103,40 @@ wts_load(void)
 			cursor->set_key(cursor, &key);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk K",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk K",
 				    g.key_cnt, (int)key.size, (char *)key.data);
 			cursor->set_value(cursor, &value);
 			if (g.logging == LOG_OPS)
 				(void)g.wt_api->msg_printf(g.wt_api, session,
-				    "%-10s %" PRIu32 " {%.*s}", "bulk V",
+				    "%-10s %" PRIu64 " {%.*s}", "bulk V",
 				    g.key_cnt,
 				    (int)value.size, (char *)value.data);
 			break;
 		}
 
-		testutil_check(cursor->insert(cursor));
+		/*
+		 * We don't want to size the cache to ensure the initial data
+		 * set can load in the in-memory case, guaranteeing the load
+		 * succeeds probably means future updates are also guaranteed
+		 * to succeed, which isn't what we want. If we run out of space
+		 * in the initial load, reset the row counter and continue.
+		 *
+		 * Decrease inserts, they can't be successful if we're at the
+		 * cache limit, and increase the delete percentage to get some
+		 * extra space once the run starts.
+		 */
+		if ((ret = cursor->insert(cursor)) != 0) {
+			if (ret != WT_CACHE_FULL)
+				testutil_die(ret, "cursor.insert");
+			g.rows = --g.key_cnt;
+			g.c_rows = (uint32_t)g.key_cnt;
+
+			if (g.c_insert_pct > 5)
+				g.c_insert_pct = 5;
+			if (g.c_delete_pct < 20)
+				g.c_delete_pct += 20;
+			break;
+		}
 
 #ifdef HAVE_BERKELEY_DB
 		if (SINGLETHREADED)
@@ -133,6 +152,6 @@ wts_load(void)
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
-	free(valbuf);
+	free(key.mem);
+	free(value.mem);
 }
diff --git a/test/format/compact.c b/test/format/compact.c
index a75ee4f2adf..240e5553697 100644
--- a/test/format/compact.c
+++ b/test/format/compact.c
@@ -36,9 +36,9 @@ void *
 compact(void *arg)
 {
 	WT_CONNECTION *conn;
+	WT_DECL_RET;
 	WT_SESSION *session;
 	u_int period;
-	int ret;
 
 	(void)(arg);
 
diff --git a/test/format/config.c b/test/format/config.c
index 042316d8344..1b09916bd88 100644
--- a/test/format/config.c
+++ b/test/format/config.c
@@ -35,6 +35,7 @@ static void	   config_encryption(void);
 static const char *config_file_type(u_int);
 static CONFIG	  *config_find(const char *, size_t);
 static void	   config_in_memory(void);
+static void	   config_in_memory_check(void);
 static int	   config_is_perm(const char *);
 static void	   config_isolation(void);
 static void	   config_lrt(void);
@@ -43,6 +44,7 @@ static void	   config_map_compression(const char *, u_int *);
 static void	   config_map_encryption(const char *, u_int *);
 static void	   config_map_file_type(const char *, u_int *);
 static void	   config_map_isolation(const char *, u_int *);
+static void	   config_reset(void);
 
 /*
  * config_setup --
@@ -54,14 +56,10 @@ config_setup(void)
 	CONFIG *cp;
 
 	/* Clear any temporary values. */
-	config_clear();
+	config_reset();
 
-	/*
-	 * Periodically, run in-memory; don't do it on the first run, all our
-	 * smoke tests would hit it.
-	 */
-	if (!config_is_perm("in_memory") && g.run_cnt % 20 == 19)
-		g.c_in_memory = 1;
+	/* Periodically run in-memory. */
+	config_in_memory();
 
 	/*
 	 * Choose a data source type and a file type: they're interrelated (LSM
@@ -145,7 +143,7 @@ config_setup(void)
 
 	/* Some data-sources don't support user-specified collations. */
 	if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
-		g.c_reverse = 0;
+		config_single("reverse=off", 0);
 
 	/*
 	 * Periodically, run single-threaded so we can compare the results to
@@ -159,7 +157,6 @@ config_setup(void)
 	config_compression("compression");
 	config_compression("logging_compression");
 	config_encryption();
-	config_in_memory();
 	config_isolation();
 	config_lrt();
 
@@ -169,7 +166,7 @@ config_setup(void)
 	 * Don't do it on the first run, all our smoke tests would hit it.
 	 */
 	if (!g.replay && g.run_cnt % 10 == 9 && !config_is_perm("delete_pct"))
-		g.c_delete_pct = 0;
+		config_single("delete_pct=0", 0);
 
 	/*
 	 * If this is an LSM run, set the cache size and crank up the insert
@@ -187,9 +184,12 @@ config_setup(void)
 	if (!config_is_perm("cache") && g.c_cache < g.c_threads)
 		g.c_cache = g.c_threads;
 
+	/* Give in-memory configuration a final review. */
+	config_in_memory_check();
+
 	/* Make the default maximum-run length 20 minutes. */
 	if (!config_is_perm("timer"))
-		g.c_timer = 20;
+		config_single("timer=20", 0);
 
 	/*
 	 * Key/value minimum/maximum are related, correct unless specified by
@@ -329,43 +329,89 @@ config_encryption(void)
 
 /*
  * config_in_memory --
- *	In-memory configuration.
+ *	Periodically set up an in-memory configuration.
  */
 static void
 config_in_memory(void)
 {
+	/*
+	 * Configure in-memory before configuring anything else, in-memory has
+	 * many related requirements. Don't configure in-memory if there's any
+	 * incompatible configurations, so we don't have to configure in-memory
+	 * every time we configure something like LSM, that's too painful.
+	 */
+	if (config_is_perm("backups"))
+		return;
+	if (config_is_perm("checkpoints"))
+		return;
+	if (config_is_perm("compression"))
+		return;
+	if (config_is_perm("data_source") && DATASOURCE("lsm"))
+		return;
+	if (config_is_perm("logging"))
+		return;
+	if (config_is_perm("rebalance"))
+		return;
+	if (config_is_perm("salvage"))
+		return;
+	if (config_is_perm("verify"))
+		return;
+
+	if (!config_is_perm("in_memory") && mmrand(NULL, 1, 20) == 1)
+		g.c_in_memory = 1;
+}
+
+/*
+ * config_in_memory_check --
+ *	In-memory configuration review.
+ */
+static void
+config_in_memory_check(void)
+{
+	uint32_t cache;
+
 	if (g.c_in_memory == 0)
 		return;
 
 	/* Turn off a lot of stuff. */
 	if (!config_is_perm("backups"))
-		g.c_backups = 0;
+		config_single("backups=off", 0);
 	if (!config_is_perm("checkpoints"))
-		g.c_checkpoints = 0;
-	if (!config_is_perm("compression")) {
-		g.c_compression = dstrdup("none");
-		g.c_compression_flag = COMPRESS_NONE;
-	}
+		config_single("checkpoints=off", 0);
+	if (!config_is_perm("compression"))
+		config_single("compression=none", 0);
 	if (!config_is_perm("logging"))
-		g.c_logging = 0;
+		config_single("logging=off", 0);
 	if (!config_is_perm("rebalance"))
-		g.c_rebalance = 0;
+		config_single("rebalance=off", 0);
 	if (!config_is_perm("salvage"))
-		g.c_salvage = 0;
+		config_single("salvage=off", 0);
 	if (!config_is_perm("verify"))
-		g.c_verify = 0;
+		config_single("verify=off", 0);
 
 	/*
-	 * Ensure there is 250MB of cache per thread; keep keys/values small,
-	 * overflow items aren't an issue for in-memory configurations and it
-	 * keeps us from overflowing the cache.
+	 * Keep keys/values small, overflow items aren't an issue for in-memory
+	 * configurations and it keeps us from overflowing the cache.
 	 */
-	if (!config_is_perm("cache"))
-		g.c_cache = g.c_threads * 250;
 	if (!config_is_perm("key_max"))
-		g.c_value_max = 64;
+		config_single("key_max=32", 0);
 	if (!config_is_perm("value_max"))
-		g.c_value_max = 128;
+		config_single("value_max=80", 0);
+
+	/*
+	 * Size the cache relative to the initial data set, use 2x the base
+	 * size as a minimum.
+	 */
+	if (!config_is_perm("cache")) {
+		cache = g.c_value_max;
+		if (g.type == ROW)
+			cache += g.c_key_max;
+		cache *= g.c_rows;
+		cache *= 2;
+		cache /= WT_MEGABYTE;
+		if (g.c_cache < cache)
+			g.c_cache = cache;
+	}
 }
 
 /*
@@ -413,11 +459,11 @@ config_lrt(void)
 	 * stores.
 	 */
 	if (g.type == FIX) {
-		if (g.c_long_running_txn && config_is_perm("long_running_txn"))
+		if (config_is_perm("long_running_txn"))
 			testutil_die(EINVAL,
 			    "long_running_txn not supported with fixed-length "
 			    "column store");
-		g.c_long_running_txn = 0;
+		config_single("long_running_txn=off", 0);
 	}
 }
 
@@ -503,18 +549,36 @@ config_file(const char *name)
 
 /*
  * config_clear --
- *	Clear per-run values.
+ *	Clear all configuration values.
  */
 void
 config_clear(void)
 {
 	CONFIG *cp;
 
-	/* Clear configuration data. */
+	/* Clear all allocated configuration data. */
+	for (cp = c; cp->name != NULL; ++cp)
+		if (cp->vstr != NULL) {
+			free((void *)*cp->vstr);
+			*cp->vstr = NULL;
+		}
+	free(g.uri);
+	g.uri = NULL;
+}
+
+/*
+ * config_reset --
+ *	Clear per-run configuration values.
+ */
+static void
+config_reset(void)
+{
+	CONFIG *cp;
+
+	/* Clear temporary allocated configuration data. */
 	for (cp = c; cp->name != NULL; ++cp) {
 		F_CLR(cp, C_TEMP);
-		if (!F_ISSET(cp, C_PERM) &&
-		    F_ISSET(cp, C_STRING) && cp->vstr != NULL) {
+		if (!F_ISSET(cp, C_PERM) && cp->vstr != NULL) {
 			free((void *)*cp->vstr);
 			*cp->vstr = NULL;
 		}
@@ -531,7 +595,7 @@ void
 config_single(const char *s, int perm)
 {
 	CONFIG *cp;
-	uint32_t v;
+	long v;
 	char *p;
 	const char *ep;
 
@@ -557,43 +621,59 @@ config_single(const char *s, int perm)
 			    exit(EXIT_FAILURE);
 		}
 
+		/*
+		 * Free the previous setting if a configuration has been
+		 * passed in twice.
+		 */
+		if (*cp->vstr != NULL) {
+			free(*cp->vstr);
+			*cp->vstr = NULL;
+		}
+
 		if (strncmp(s, "checksum", strlen("checksum")) == 0) {
 			config_map_checksum(ep, &g.c_checksum_flag);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		} else if (strncmp(
 		    s, "compression", strlen("compression")) == 0) {
 			config_map_compression(ep, &g.c_compression_flag);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		} else if (strncmp(
 		    s, "encryption", strlen("encryption")) == 0) {
 			config_map_encryption(ep, &g.c_encryption_flag);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		} else if (strncmp(s, "isolation", strlen("isolation")) == 0) {
 			config_map_isolation(ep, &g.c_isolation_flag);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		} else if (strncmp(s, "file_type", strlen("file_type")) == 0) {
 			config_map_file_type(ep, &g.type);
-			*cp->vstr = strdup(config_file_type(g.type));
+			*cp->vstr = dstrdup(config_file_type(g.type));
 		} else if (strncmp(s, "logging_compression",
 		    strlen("logging_compression")) == 0) {
 			config_map_compression(ep,
 			    &g.c_logging_compression_flag);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		} else {
 			free((void *)*cp->vstr);
-			*cp->vstr = strdup(ep);
+			*cp->vstr = dstrdup(ep);
 		}
-		if (*cp->vstr == NULL)
-			testutil_die(errno, "malloc");
 
 		return;
 	}
 
-	v = (uint32_t)strtoul(ep, &p, 10);
-	if (*p != '\0') {
-		fprintf(stderr, "%s: %s: illegal numeric value\n",
-		    g.progname, s);
-		exit(EXIT_FAILURE);
+	v = -1;
+	if (F_ISSET(cp, C_BOOL)) {
+		if (strncmp(ep, "off", strlen("off")) == 0)
+			v = 0;
+		else if (strncmp(ep, "on", strlen("on")) == 0)
+			v = 1;
+	}
+	if (v == -1) {
+		v = strtol(ep, &p, 10);
+		if (*p != '\0') {
+			fprintf(stderr, "%s: %s: illegal numeric value\n",
+			    g.progname, s);
+			exit(EXIT_FAILURE);
+		}
 	}
 	if (F_ISSET(cp, C_BOOL)) {
 		if (v != 0 && v != 1) {
@@ -607,7 +687,7 @@ config_single(const char *s, int perm)
 		    g.progname, s, cp->min, cp->maxset);
 		exit(EXIT_FAILURE);
 	}
-	*cp->v = v;
+	*cp->v = (uint32_t)v;
 }
 
 /*
diff --git a/test/format/config.h b/test/format/config.h
index a17614bc044..16fffb6fafe 100644
--- a/test/format/config.h
+++ b/test/format/config.h
@@ -294,6 +294,10 @@ static CONFIG c[] = {
 	  "maximum time to run in minutes (default 20 minutes)",
 	  C_IGNORE, 0, UINT_MAX, UINT_MAX, &g.c_timer, NULL },
 
+	{ "transaction-frequency",
+	  "percent operations done inside an explicit transaction",
+	  0x0, 1, 100, 100, &g.c_txn_freq, NULL },
+
 	{ "value_max",
 	  "maximum size of values",
 	  0x0, 32, 4096, MEGABYTE(10), &g.c_value_max, NULL },
diff --git a/test/format/format.h b/test/format/format.h
index a129c5395fd..beaabe7e83c 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -26,33 +26,10 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 
-#include <sys/stat.h>
-#ifndef _WIN32
-#include <sys/time.h>
-#endif
-#include <sys/types.h>
-
-#include <assert.h>
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <inttypes.h>
-#include <limits.h>
-#ifndef _WIN32
-#include <pthread.h>
-#endif
-#include <signal.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#ifndef _WIN32
-#include <unistd.h>
-#endif
-#include <time.h>
-
 #include "test_util.i"
 
 #ifdef BDB
+#include <assert.h>
 #include <db.h>
 #endif
 
@@ -109,7 +86,6 @@ typedef struct {
 
 	char *home;				/* Home directory */
 	char *home_backup;			/* Hot-backup directory */
-	char *home_backup2;			/* Saved Hot-backup directory */
 	char *home_backup_init;			/* Initialize backup command */
 	char *home_bdb;				/* BDB directory */
 	char *home_config;			/* Run CONFIG file path */
@@ -145,7 +121,8 @@ typedef struct {
 	int replay;				/* Replaying a run. */
 	int workers_finished;			/* Operations completed */
 
-	pthread_rwlock_t backup_lock;		/* Hot backup running */
+	pthread_rwlock_t backup_lock;		/* Backup running */
+	pthread_rwlock_t checkpoint_lock;	/* Checkpoint running */
 
 	WT_RAND_STATE rnd;			/* Global RNG state */
 
@@ -224,6 +201,7 @@ typedef struct {
 	uint32_t c_statistics_server;
 	uint32_t c_threads;
 	uint32_t c_timer;
+	uint32_t c_txn_freq;
 	uint32_t c_value_max;
 	uint32_t c_value_min;
 	uint32_t c_verify;
@@ -297,7 +275,7 @@ void	 bdb_np(int, void *, size_t *, void *, size_t *, int *);
 void	 bdb_open(void);
 void	 bdb_read(uint64_t, void *, size_t *, int *);
 void	 bdb_remove(uint64_t, int *);
-void	 bdb_update(const void *, size_t, const void *, size_t, int *);
+void	 bdb_update(const void *, size_t, const void *, size_t);
 #endif
 
 void	*backup(void *);
@@ -308,25 +286,23 @@ void	 config_file(const char *);
 void	 config_print(int);
 void	 config_setup(void);
 void	 config_single(const char *, int);
-void	*dmalloc(size_t);
-char	*dstrdup(const char *);
 void	 fclose_and_clear(FILE **);
-void	 key_gen(uint8_t *, size_t *, uint64_t);
-void	 key_gen_insert(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t);
-void	 key_gen_setup(uint8_t **);
+void	 key_gen(WT_ITEM *, uint64_t);
+void	 key_gen_insert(WT_RAND_STATE *, WT_ITEM *, uint64_t);
+void	 key_gen_setup(WT_ITEM *);
 void	 key_len_setup(void);
 void	*lrt(void *);
 void	 path_setup(const char *);
-int	 read_row(WT_CURSOR *, WT_ITEM *, uint64_t, int);
+int	 read_row(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
 uint32_t rng(WT_RAND_STATE *);
 void	 track(const char *, uint64_t, TINFO *);
-void	 val_gen(WT_RAND_STATE *, uint8_t *, size_t *, uint64_t);
-void	 val_gen_setup(WT_RAND_STATE *, uint8_t **);
+void	 val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t);
+void	 val_gen_setup(WT_RAND_STATE *, WT_ITEM *);
 void	 wts_close(void);
-void	 wts_create(void);
 void	 wts_dump(const char *, int);
+void	 wts_init(void);
 void	 wts_load(void);
-void	 wts_open(const char *, int, WT_CONNECTION **);
+void	 wts_open(const char *, bool, WT_CONNECTION **);
 void	 wts_ops(int);
 void	 wts_read_scan(void);
 void	 wts_rebalance(void);
diff --git a/test/format/lrt.c b/test/format/lrt.c
index 451d2f4fa3c..937525522fa 100644
--- a/test/format/lrt.c
+++ b/test/format/lrt.c
@@ -43,17 +43,15 @@ lrt(void *arg)
 	uint64_t keyno, saved_keyno;
 	u_int period;
 	int pinned, ret;
-	uint8_t bitfield, *keybuf;
+	uint8_t bitfield;
 	void *buf;
 
 	(void)(arg);			/* Unused parameter */
 
 	saved_keyno = 0;		/* [-Werror=maybe-uninitialized] */
 
-	key_gen_setup(&keybuf);
-	memset(&key, 0, sizeof(key));
-	key.data = keybuf;
-	memset(&value, 0, sizeof(value));
+	key_gen_setup(&key);
+	val_gen_setup(NULL, &value);
 
 	buf = NULL;
 	buf_len = buf_size = 0;
@@ -67,8 +65,8 @@ lrt(void *arg)
 	for (pinned = 0;;) {
 		if (pinned) {
 			/* Re-read the record at the end of the table. */
-			while ((ret = read_row(cursor,
-			    &key, saved_keyno, 1)) == WT_ROLLBACK)
+			while ((ret = read_row(
+			    cursor, &key, &value, saved_keyno)) == WT_ROLLBACK)
 				;
 			if (ret != 0)
 				testutil_die(ret,
@@ -112,7 +110,7 @@ lrt(void *arg)
 				    (u_int)(g.key_cnt - g.key_cnt / 10),
 				    (u_int)g.key_cnt);
 				while ((ret = read_row(cursor,
-				    &key, saved_keyno, 1)) == WT_ROLLBACK)
+				    &key, &value, saved_keyno)) == WT_ROLLBACK)
 					;
 			} while (ret == WT_NOTFOUND);
 			if (ret != 0)
@@ -129,9 +127,8 @@ lrt(void *arg)
 			if (ret != 0)
 				testutil_die(ret,
 				    "cursor.get_value: %" PRIu64, saved_keyno);
-			if (buf_len < value.size &&
-			    (buf = realloc(buf, buf_len = value.size)) == NULL)
-				testutil_die(errno, "malloc");
+			if (buf_len < value.size)
+				buf = drealloc(buf, buf_len = value.size);
 			memcpy(buf, value.data, buf_size = value.size);
 
 			/*
@@ -142,7 +139,7 @@ lrt(void *arg)
 			do {
 				keyno = mmrand(NULL, 1, (u_int)g.key_cnt / 5);
 				while ((ret = read_row(cursor,
-				    &key, keyno, 1)) == WT_ROLLBACK)
+				    &key, &value, keyno)) == WT_ROLLBACK)
 					;
 			} while (ret == WT_NOTFOUND);
 			if (ret != 0)
@@ -165,7 +162,8 @@ lrt(void *arg)
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
+	free(key.mem);
+	free(value.mem);
 	free(buf);
 
 	return (NULL);
diff --git a/test/format/ops.c b/test/format/ops.c
index 5d66f4d5391..9275d7f3856 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -28,14 +28,14 @@
 
 #include "format.h"
 
-static int   col_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *);
-static int   col_remove(WT_CURSOR *, WT_ITEM *, uint64_t, int *);
-static int   col_update(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
-static int   nextprev(WT_CURSOR *, int, int *);
+static int   col_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t *);
+static int   col_remove(WT_CURSOR *, WT_ITEM *, uint64_t);
+static int   col_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
+static int   nextprev(WT_CURSOR *, int);
 static void *ops(void *);
-static int   row_insert(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
-static int   row_remove(WT_CURSOR *, WT_ITEM *, uint64_t, int *);
-static int   row_update(TINFO *, WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
+static int   row_insert(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
+static int   row_remove(WT_CURSOR *, WT_ITEM *, uint64_t);
+static int   row_update(WT_CURSOR *, WT_ITEM *, WT_ITEM *, uint64_t);
 static void  table_append_init(void);
 
 #ifdef HAVE_BERKELEY_DB
@@ -103,8 +103,7 @@ wts_ops(int lastrun)
 	}
 
 	/* Create thread structure; start the worker threads. */
-	if ((tinfo = calloc((size_t)g.c_threads, sizeof(*tinfo))) == NULL)
-		testutil_die(errno, "calloc");
+	tinfo = dcalloc((size_t)g.c_threads, sizeof(*tinfo));
 	for (i = 0; i < g.c_threads; ++i) {
 		tinfo[i].id = (int)i + 1;
 		tinfo[i].state = TINFO_RUNNING;
@@ -184,6 +183,7 @@ wts_ops(int lastrun)
 		(void)pthread_join(compact_tid, NULL);
 	if (!SINGLETHREADED && g.c_long_running_txn)
 		(void)pthread_join(lrt_tid, NULL);
+	g.workers_finished = 0;
 
 	if (g.logging != 0) {
 		(void)g.wt_api->msg_printf(g.wt_api, session,
@@ -193,57 +193,229 @@ wts_ops(int lastrun)
 }
 
 /*
- * ops_session_config --
- *	Return the current session configuration.
+ * isolation_config --
+ *	Return an isolation configuration.
  */
-static const char *
-ops_session_config(WT_RAND_STATE *rnd)
+static inline const char *
+isolation_config(WT_RAND_STATE *rnd, bool *iso_snapshotp)
 {
 	u_int v;
 
-	/*
-	 * The only current session configuration is the isolation level.
-	 */
 	if ((v = g.c_isolation_flag) == ISOLATION_RANDOM)
 		v = mmrand(rnd, 2, 4);
 	switch (v) {
 	case ISOLATION_READ_UNCOMMITTED:
+		*iso_snapshotp = false;
 		return ("isolation=read-uncommitted");
 	case ISOLATION_READ_COMMITTED:
+		*iso_snapshotp = false;
 		return ("isolation=read-committed");
 	case ISOLATION_SNAPSHOT:
 	default:
+		*iso_snapshotp = true;
 		return ("isolation=snapshot");
 	}
 }
 
+typedef struct {
+	uint64_t keyno;			/* Row number */
+
+	void    *kdata;			/* If an insert, the generated key */
+	size_t   ksize;
+	size_t   kmemsize;
+
+	void    *vdata;			/* If not a delete, the value */
+	size_t   vsize;
+	size_t   vmemsize;
+
+	bool     deleted;		/* Delete operation */
+	bool     insert;		/* Insert operation */
+} SNAP_OPS;
+
+/*
+ * snap_track --
+ *     Add a single snapshot isolation returned value to the list.
+ */
+static void
+snap_track(SNAP_OPS *snap, uint64_t keyno, WT_ITEM *key, WT_ITEM *value)
+{
+	snap->keyno = keyno;
+	if (key == NULL)
+		snap->insert = false;
+	else {
+		snap->insert = true;
+
+		if (snap->kmemsize < key->size) {
+			snap->kdata = drealloc(snap->kdata, key->size);
+			snap->kmemsize = key->size;
+		}
+		memcpy(snap->kdata, key->data, snap->ksize = key->size);
+	}
+	if (value == NULL)
+		snap->deleted = true;
+	else  {
+		snap->deleted = false;
+		if (snap->vmemsize < value->size) {
+			snap->vdata = drealloc(snap->vdata, value->size);
+			snap->vmemsize = value->size;
+		}
+		memcpy(snap->vdata, value->data, snap->vsize = value->size);
+	}
+}
+
+/*
+ * snap_check --
+ *	Check snapshot isolation operations are repeatable.
+ */
+static int
+snap_check(WT_CURSOR *cursor,
+    SNAP_OPS *start, SNAP_OPS *stop, WT_ITEM *key, WT_ITEM *value)
+{
+	WT_DECL_RET;
+	SNAP_OPS *p;
+	uint8_t bitfield;
+
+	for (; start < stop; ++start) {
+		/* Check for subsequent changes to this record. */
+		for (p = start + 1; p < stop && p->keyno != start->keyno; ++p)
+			;
+		if (p != stop)
+			continue;
+
+		/*
+		 * Retrieve the key/value pair by key. Row-store inserts have a
+		 * unique generated key we saved, else generate the key from the
+		 * key number.
+		 */
+		if (start->insert == 0) {
+			switch (g.type) {
+			case FIX:
+			case VAR:
+				cursor->set_key(cursor, start->keyno);
+				break;
+			case ROW:
+				key_gen(key, start->keyno);
+				cursor->set_key(cursor, key);
+				break;
+			}
+		} else {
+			key->data = start->kdata;
+			key->size = start->ksize;
+			cursor->set_key(cursor, key);
+		}
+		if ((ret = cursor->search(cursor)) == 0) {
+			if (g.type == FIX) {
+				testutil_check(
+				    cursor->get_value(cursor, &bitfield));
+				*(uint8_t *)(value->data) = bitfield;
+				value->size = 1;
+			} else
+				testutil_check(
+				    cursor->get_value(cursor, value));
+		} else
+			if (ret != WT_NOTFOUND)
+				return (ret);
+
+		/* Check for simple matches. */
+		if (ret == 0 && !start->deleted &&
+		    value->size == start->vsize &&
+		    memcmp(value->data, start->vdata, value->size) == 0)
+			continue;
+		if (ret == WT_NOTFOUND && start->deleted)
+			continue;
+
+		/*
+		 * In fixed length stores, zero values at the end of the key
+		 * space are returned as not-found, and not-found row reads
+		 * are saved as zero values. Map back-and-forth for simplicity.
+		 */
+		if (g.type == FIX) {
+			if (ret == WT_NOTFOUND &&
+			    start->vsize == 1 && *(uint8_t *)start->vdata == 0)
+				continue;
+			if (start->deleted &&
+			    value->size == 1 && *(uint8_t *)value->data == 0)
+				continue;
+		}
+
+		/* Things went pear-shaped. */
+		switch (g.type) {
+		case FIX:
+			testutil_die(ret,
+			    "snap_check: %" PRIu64 " search: "
+			    "expected {0x%02x}, found {0x%02x}",
+			    start->keyno,
+			    start->deleted ? 0 : *(uint8_t *)start->vdata,
+			    ret == WT_NOTFOUND ? 0 : *(uint8_t *)value->data);
+			/* NOTREACHED */
+		case ROW:
+			testutil_die(ret,
+			    "snap_check: %.*s search: "
+			    "expected {%.*s}, found {%.*s}",
+			    (int)key->size, key->data,
+			    start->deleted ?
+			    (int)strlen("deleted") : (int)start->vsize,
+			    start->deleted ? "deleted" : start->vdata,
+			    ret == WT_NOTFOUND ?
+			    (int)strlen("deleted") : (int)value->size,
+			    ret == WT_NOTFOUND ? "deleted" : value->data);
+			/* NOTREACHED */
+		case VAR:
+			testutil_die(ret,
+			    "snap_check: %" PRIu64 " search: "
+			    "expected {%.*s}, found {%.*s}",
+			    start->keyno,
+			    start->deleted ?
+			    (int)strlen("deleted") : (int)start->vsize,
+			    start->deleted ? "deleted" : start->vdata,
+			    ret == WT_NOTFOUND ?
+			    (int)strlen("deleted") : (int)value->size,
+			    ret == WT_NOTFOUND ? "deleted" : value->data);
+			/* NOTREACHED */
+		}
+	}
+	return (0);
+}
+
+/*
+ * ops --
+ *     Per-thread operations.
+ */
 static void *
 ops(void *arg)
 {
+	SNAP_OPS *snap, snap_list[64];
 	TINFO *tinfo;
 	WT_CONNECTION *conn;
 	WT_CURSOR *cursor, *cursor_insert;
+	WT_DECL_RET;
+	WT_ITEM *key, _key, *value, _value;
 	WT_SESSION *session;
-	WT_ITEM key, value;
 	uint64_t keyno, ckpt_op, reset_op, session_op;
-	uint32_t op;
-	uint8_t *keybuf, *valbuf;
-	u_int np;
-	int ckpt_available, dir, insert, intxn, notfound, readonly;
+	uint32_t op, rnd;
+	u_int i;
+	int dir;
 	char *ckpt_config, ckpt_name[64];
+	bool ckpt_available, intxn, iso_snapshot, positioned, readonly;
 
 	tinfo = arg;
 
 	conn = g.wts_conn;
-	keybuf = valbuf = NULL;
-	readonly = 0;			/* -Wconditional-uninitialized */
+	readonly = false;		/* -Wconditional-uninitialized */
+
+	/* Initialize tracking of snapshot isolation transaction returns. */
+	snap = NULL;
+	iso_snapshot = false;
+	memset(snap_list, 0, sizeof(snap_list));
 
 	/* Initialize the per-thread random number generator. */
 	__wt_random_init(&tinfo->rnd);
 
 	/* Set up the default key and value buffers. */
-	key_gen_setup(&keybuf);
-	val_gen_setup(&tinfo->rnd, &valbuf);
+	key = &_key;
+	key_gen_setup(key);
+	value = &_value;
+	val_gen_setup(&tinfo->rnd, value);
 
 	/* Set the first operation where we'll create sessions and cursors. */
 	session_op = 0;
@@ -252,12 +424,12 @@ ops(void *arg)
 
 	/* Set the first operation where we'll perform checkpoint operations. */
 	ckpt_op = g.c_checkpoints ? mmrand(&tinfo->rnd, 100, 10000) : 0;
-	ckpt_available = 0;
+	ckpt_available = false;
 
 	/* Set the first operation where we'll reset the session. */
 	reset_op = mmrand(&tinfo->rnd, 100, 10000);
 
-	for (intxn = 0; !tinfo->quit; ++tinfo->ops) {
+	for (intxn = false; !tinfo->quit; ++tinfo->ops) {
 		/*
 		 * We can't checkpoint or swap sessions/cursors while in a
 		 * transaction, resolve any running transaction.
@@ -267,7 +439,7 @@ ops(void *arg)
 			testutil_check(
 			    session->commit_transaction(session, NULL));
 			++tinfo->commit;
-			intxn = 0;
+			intxn = false;
 		}
 
 		/* Open up a new session and cursors. */
@@ -276,8 +448,8 @@ ops(void *arg)
 			if (session != NULL)
 				testutil_check(session->close(session, NULL));
 
-			testutil_check(conn->open_session(conn, NULL,
-			    ops_session_config(&tinfo->rnd), &session));
+			testutil_check(
+			    conn->open_session(conn, NULL, NULL, &session));
 
 			/*
 			 * 10% of the time, perform some read-only operations
@@ -299,7 +471,7 @@ ops(void *arg)
 				session_op += 250;
 
 				/* Checkpoints are read-only. */
-				readonly = 1;
+				readonly = true;
 			} else {
 				/*
 				 * Open two cursors: one for overwriting and one
@@ -325,21 +497,32 @@ ops(void *arg)
 				session_op += mmrand(&tinfo->rnd, 100, 5000);
 
 				/* Updates supported. */
-				readonly = 0;
+				readonly = false;
 			}
 		}
 
 		/* Checkpoint the database. */
 		if (tinfo->ops == ckpt_op && g.c_checkpoints) {
 			/*
-			 * LSM and data-sources don't support named checkpoints,
+			 * Checkpoints are single-threaded inside WiredTiger,
+			 * skip our checkpoint if another thread is already
+			 * doing one.
+			 */
+			ret = pthread_rwlock_trywrlock(&g.checkpoint_lock);
+			if (ret == EBUSY)
+				goto skip_checkpoint;
+			testutil_check(ret);
+
+			/*
+			 * LSM and data-sources don't support named checkpoints
 			 * and we can't drop a named checkpoint while there's a
-			 * cursor open on it, otherwise 20% of the time name the
-			 * checkpoint.
+			 * backup in progress, otherwise name the checkpoint 5%
+			 * of the time.
 			 */
-			if (DATASOURCE("helium") || DATASOURCE("kvsbdb") ||
-			    DATASOURCE("lsm") ||
-			    readonly || mmrand(&tinfo->rnd, 1, 5) == 1)
+			if (mmrand(&tinfo->rnd, 1, 20) != 1 ||
+			    DATASOURCE("helium") ||
+			    DATASOURCE("kvsbdb") || DATASOURCE("lsm") ||
+			    pthread_rwlock_trywrlock(&g.backup_lock) == EBUSY)
 				ckpt_config = NULL;
 			else {
 				(void)snprintf(ckpt_name, sizeof(ckpt_name),
@@ -347,11 +530,6 @@ ops(void *arg)
 				ckpt_config = ckpt_name;
 			}
 
-			/* Named checkpoints lock out backups */
-			if (ckpt_config != NULL)
-				testutil_check(
-				    pthread_rwlock_wrlock(&g.backup_lock));
-
 			testutil_checkfmt(
 			    session->checkpoint(session, ckpt_config),
 			    "%s", ckpt_config == NULL ? "" : ckpt_config);
@@ -359,6 +537,8 @@ ops(void *arg)
 			if (ckpt_config != NULL)
 				testutil_check(
 				    pthread_rwlock_unlock(&g.backup_lock));
+			testutil_check(
+			    pthread_rwlock_unlock(&g.checkpoint_lock));
 
 			/* Rephrase the checkpoint name for cursor open. */
 			if (ckpt_config == NULL)
@@ -367,9 +547,9 @@ ops(void *arg)
 			else
 				(void)snprintf(ckpt_name, sizeof(ckpt_name),
 				    "checkpoint=thread-%d", tinfo->id);
-			ckpt_available = 1;
+			ckpt_available = true;
 
-			/* Pick the next checkpoint operation. */
+skip_checkpoint:	/* Pick the next checkpoint operation. */
 			ckpt_op += mmrand(&tinfo->rnd, 5000, 20000);
 		}
 
@@ -386,21 +566,24 @@ ops(void *arg)
 		}
 
 		/*
-		 * If we're not single-threaded and we're not in a transaction,
-		 * start a transaction 20% of the time.
+		 * If we're not single-threaded and not in a transaction, choose
+		 * an isolation level and start a transaction some percentage of
+		 * the time.
 		 */
 		if (!SINGLETHREADED &&
-		    !intxn && mmrand(&tinfo->rnd, 1, 10) >= 8) {
+		    !intxn && mmrand(&tinfo->rnd, 1, 100) >= g.c_txn_freq) {
+			testutil_check(
+			    session->reconfigure(session,
+				isolation_config(&tinfo->rnd, &iso_snapshot)));
 			testutil_check(
 			    session->begin_transaction(session, NULL));
-			intxn = 1;
-		}
 
-		insert = notfound = 0;
+			snap = iso_snapshot ? snap_list : NULL;
+			intxn = true;
+		}
 
 		keyno = mmrand(&tinfo->rnd, 1, (u_int)g.rows);
-		key.data = keybuf;
-		value.data = valbuf;
+		positioned = false;
 
 		/*
 		 * Perform some number of operations: the percentage of deletes,
@@ -414,27 +597,30 @@ ops(void *arg)
 			++tinfo->remove;
 			switch (g.type) {
 			case ROW:
-				/*
-				 * If deleting a non-existent record, the cursor
-				 * won't be positioned, and so can't do a next.
-				 */
-				if (row_remove(cursor, &key, keyno, &notfound))
-					goto deadlock;
+				ret = row_remove(cursor, key, keyno);
 				break;
 			case FIX:
 			case VAR:
-				if (col_remove(cursor, &key, keyno, &notfound))
-					goto deadlock;
+				ret = col_remove(cursor, key, keyno);
 				break;
 			}
+			if (ret == 0) {
+				positioned = true;
+				if (snap != NULL && (size_t)
+				    (snap - snap_list) < WT_ELEMENTS(snap_list))
+					snap_track(snap++, keyno, NULL, NULL);
+			} else {
+				positioned = false;
+				if (ret == WT_ROLLBACK && intxn)
+					goto deadlock;
+			}
 		} else if (op < g.c_delete_pct + g.c_insert_pct) {
 			++tinfo->insert;
 			switch (g.type) {
 			case ROW:
-				if (row_insert(
-				    tinfo, cursor, &key, &value, keyno))
-					goto deadlock;
-				insert = 1;
+				key_gen_insert(&tinfo->rnd, key, keyno);
+				val_gen(&tinfo->rnd, value, keyno);
+				ret = row_insert(cursor, key, value, keyno);
 				break;
 			case FIX:
 			case VAR:
@@ -447,37 +633,60 @@ ops(void *arg)
 					goto skip_insert;
 
 				/* Insert, then reset the insert cursor. */
-				if (col_insert(tinfo,
-				    cursor_insert, &key, &value, &keyno))
-					goto deadlock;
+				val_gen(&tinfo->rnd, value, g.rows + 1);
+				ret = col_insert(
+				    cursor_insert, key, value, &keyno);
 				testutil_check(
 				    cursor_insert->reset(cursor_insert));
-
-				insert = 1;
 				break;
 			}
+			positioned = false;
+			if (ret == 0) {
+				if (snap != NULL && (size_t)
+				    (snap - snap_list) < WT_ELEMENTS(snap_list))
+					snap_track(snap++, keyno,
+					    g.type == ROW ? key : NULL, value);
+			} else
+				if (ret == WT_ROLLBACK && intxn)
+					goto deadlock;
 		} else if (
 		    op < g.c_delete_pct + g.c_insert_pct + g.c_write_pct) {
 			++tinfo->update;
 			switch (g.type) {
 			case ROW:
-				if (row_update(
-				    tinfo, cursor, &key, &value, keyno))
-					goto deadlock;
+				key_gen(key, keyno);
+				val_gen(&tinfo->rnd, value, keyno);
+				ret = row_update(cursor, key, value, keyno);
 				break;
 			case FIX:
 			case VAR:
-skip_insert:			if (col_update(tinfo,
-				    cursor, &key, &value, keyno))
-					goto deadlock;
+skip_insert:			val_gen(&tinfo->rnd, value, keyno);
+				ret = col_update(cursor, key, value, keyno);
 				break;
 			}
+			if (ret == 0) {
+				positioned = true;
+				if (snap != NULL && (size_t)
+				    (snap - snap_list) < WT_ELEMENTS(snap_list))
+					snap_track(snap++, keyno, NULL, value);
+			} else {
+				positioned = false;
+				if (ret == WT_ROLLBACK && intxn)
+					goto deadlock;
+			}
 		} else {
 			++tinfo->search;
-			if (read_row(cursor, &key, keyno, 0))
-				if (intxn)
+			ret = read_row(cursor, key, value, keyno);
+			if (ret == 0) {
+				positioned = true;
+				if (snap != NULL && (size_t)
+				    (snap - snap_list) < WT_ELEMENTS(snap_list))
+					snap_track(snap++, keyno, NULL, value);
+			} else {
+				positioned = false;
+				if (ret == WT_ROLLBACK && intxn)
 					goto deadlock;
-			continue;
+			}
 		}
 
 		/*
@@ -485,55 +694,64 @@ skip_insert:			if (col_update(tinfo,
 		 * insert, do a small number of next/prev cursor operations in
 		 * a random direction.
 		 */
-		if (!insert) {
+		if (positioned) {
 			dir = (int)mmrand(&tinfo->rnd, 0, 1);
-			for (np = 0; np < mmrand(&tinfo->rnd, 1, 100); ++np) {
-				if (notfound)
-					break;
-				if (nextprev(cursor, dir, &notfound))
+			for (i = 0; i < mmrand(&tinfo->rnd, 1, 100); ++i) {
+				if ((ret = nextprev(cursor, dir)) == 0)
+					continue;
+				if (ret == WT_ROLLBACK && intxn)
 					goto deadlock;
+				break;
 			}
 		}
 
-		/* Read to confirm the operation. */
-		++tinfo->search;
-		if (read_row(cursor, &key, keyno, 0))
-			goto deadlock;
-
 		/* Reset the cursor: there is no reason to keep pages pinned. */
 		testutil_check(cursor->reset(cursor));
 
 		/*
-		 * If we're in the transaction, commit 40% of the time and
+		 * If we're in a transaction, commit 40% of the time and
 		 * rollback 10% of the time.
 		 */
-		if (intxn)
-			switch (mmrand(&tinfo->rnd, 1, 10)) {
-			case 1: case 2: case 3: case 4:		/* 40% */
-				testutil_check(session->commit_transaction(
-				    session, NULL));
-				++tinfo->commit;
-				intxn = 0;
-				break;
-			case 5:					/* 10% */
-				if (0) {
-deadlock:				++tinfo->deadlock;
-				}
-				testutil_check(session->rollback_transaction(
-				    session, NULL));
-				++tinfo->rollback;
-				intxn = 0;
-				break;
-			default:
-				break;
+		if (!intxn || (rnd = mmrand(&tinfo->rnd, 1, 10)) > 5)
+			continue;
+
+		/*
+		 * Ending the transaction. If in snapshot isolation, repeat the
+		 * operations and confirm they're unchanged.
+		 */
+		if (snap != NULL && (ret = snap_check(
+		    cursor, snap_list, snap, key, value)) == WT_ROLLBACK)
+			goto deadlock;
+
+		switch (rnd) {
+		case 1: case 2: case 3: case 4:			/* 40% */
+			testutil_check(
+			    session->commit_transaction(session, NULL));
+			++tinfo->commit;
+			break;
+		case 5:						/* 10% */
+			if (0) {
+deadlock:			++tinfo->deadlock;
 			}
+			testutil_check(
+			    session->rollback_transaction(session, NULL));
+			++tinfo->rollback;
+			break;
+		}
+
+		intxn = false;
+		snap = NULL;
 	}
 
 	if (session != NULL)
 		testutil_check(session->close(session, NULL));
 
-	free(keybuf);
-	free(valbuf);
+	for (i = 0; i < WT_ELEMENTS(snap_list); ++i) {
+		free(snap_list[i].kdata);
+		free(snap_list[i].vdata);
+	}
+	free(key->mem);
+	free(value->mem);
 
 	tinfo->state = TINFO_COMPLETE;
 	return (NULL);
@@ -548,40 +766,47 @@ wts_read_scan(void)
 {
 	WT_CONNECTION *conn;
 	WT_CURSOR *cursor;
-	WT_ITEM key;
+	WT_DECL_RET;
+	WT_ITEM key, value;
 	WT_SESSION *session;
-	uint64_t cnt, last_cnt;
-	uint8_t *keybuf;
+	uint64_t keyno, last_keyno;
 
 	conn = g.wts_conn;
 
-	/* Set up the default key buffer. */
-	key_gen_setup(&keybuf);
+	/* Set up the default key/value buffers. */
+	key_gen_setup(&key);
+	val_gen_setup(NULL, &value);
 
 	/* Open a session and cursor pair. */
-	testutil_check(conn->open_session(
-	    conn, NULL, ops_session_config(NULL), &session));
-	testutil_check(session->open_cursor(
-	    session, g.uri, NULL, NULL, &cursor));
+	testutil_check(conn->open_session(conn, NULL, NULL, &session));
+	testutil_check(
+	    session->open_cursor(session, g.uri, NULL, NULL, &cursor));
 
 	/* Check a random subset of the records using the key. */
-	for (last_cnt = cnt = 0; cnt < g.key_cnt;) {
-		cnt += mmrand(NULL, 1, 17);
-		if (cnt > g.rows)
-			cnt = g.rows;
-		if (cnt - last_cnt > 1000) {
-			track("read row scan", cnt, NULL);
-			last_cnt = cnt;
+	for (last_keyno = keyno = 0; keyno < g.key_cnt;) {
+		keyno += mmrand(NULL, 1, 17);
+		if (keyno > g.rows)
+			keyno = g.rows;
+		if (keyno - last_keyno > 1000) {
+			track("read row scan", keyno, NULL);
+			last_keyno = keyno;
 		}
 
-		key.data = keybuf;
-		testutil_checkfmt(
-		    read_row(cursor, &key, cnt, 0), "%s", "read_scan");
+		switch (ret = read_row(cursor, &key, &value, keyno)) {
+		case 0:
+		case WT_NOTFOUND:
+		case WT_ROLLBACK:
+			break;
+		default:
+			testutil_die(
+			    ret, "wts_read_scan: read row %" PRIu64, keyno);
+		}
 	}
 
 	testutil_check(session->close(session, NULL));
 
-	free(keybuf);
+	free(key.mem);
+	free(value.mem);
 }
 
 /*
@@ -589,10 +814,9 @@ wts_read_scan(void)
  *	Read and verify a single element in a row- or column-store file.
  */
 int
-read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
+read_row(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
 {
 	static int sn = 0;
-	WT_ITEM value;
 	WT_SESSION *session;
 	int exact, ret;
 	uint8_t bitfield;
@@ -611,7 +835,7 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
 		cursor->set_key(cursor, keyno);
 		break;
 	case ROW:
-		key_gen((uint8_t *)key->data, &key->size, keyno);
+		key_gen(key, keyno);
 		cursor->set_key(cursor, key);
 		break;
 	}
@@ -628,37 +852,33 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
 	switch (ret) {
 	case 0:
 		if (g.type == FIX) {
-			ret = cursor->get_value(cursor, &bitfield);
-			value.data = &bitfield;
-			value.size = 1;
+			testutil_check(cursor->get_value(cursor, &bitfield));
+			*(uint8_t *)(value->data) = bitfield;
+			value->size = 1;
 		} else
-			ret = cursor->get_value(cursor, &value);
+			testutil_check(cursor->get_value(cursor, value));
 		break;
-	case WT_ROLLBACK:
-		return (WT_ROLLBACK);
 	case WT_NOTFOUND:
-		if (notfound_err)
-			return (WT_NOTFOUND);
+		/*
+		 * In fixed length stores, zero values at the end of the key
+		 * space are returned as not found.  Treat this the same as
+		 * a zero value in the key space, to match BDB's behavior.
+		 */
+		if (g.type == FIX) {
+			*(uint8_t *)(value->data) = 0;
+			value->size = 1;
+			ret = 0;
+		}
 		break;
+	case WT_ROLLBACK:
+		return (WT_ROLLBACK);
 	default:
 		testutil_die(ret, "read_row: read row %" PRIu64, keyno);
 	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
-		return (0);
-
-	/*
-	 * In fixed length stores, zero values at the end of the key space are
-	 * returned as not found.  Treat this the same as a zero value in the
-	 * key space, to match BDB's behavior.
-	 */
-	if (ret == WT_NOTFOUND && g.type == FIX) {
-		bitfield = 0;
-		value.data = &bitfield;
-		value.size = 1;
-		ret = 0;
-	}
+		return (ret);
 
 	/* Retrieve the BDB value. */
 	{
@@ -669,20 +889,20 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
 
 	/* Check for not-found status. */
 	if (notfound_chk("read_row", ret, notfound, keyno))
-		return (0);
+		return (ret);
 
 	/* Compare the two. */
-	if (value.size != bdb_value.size ||
-	    memcmp(value.data, bdb_value.data, value.size) != 0) {
+	if (value->size != bdb_value.size ||
+	    memcmp(value->data, bdb_value.data, value->size) != 0) {
 		fprintf(stderr,
 		    "read_row: value mismatch %" PRIu64 ":\n", keyno);
 		print_item("bdb", &bdb_value);
-		print_item(" wt", &value);
+		print_item(" wt", value);
 		testutil_die(0, NULL);
 	}
 	}
 #endif
-	return (0);
+	return (ret);
 }
 
 /*
@@ -690,21 +910,19 @@ read_row(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int notfound_err)
  *	Read and verify the next/prev element in a row- or column-store file.
  */
 static int
-nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
+nextprev(WT_CURSOR *cursor, int next)
 {
+	WT_DECL_RET;
 	WT_ITEM key, value;
 	uint64_t keyno;
-	int ret;
 	uint8_t bitfield;
 	const char *which;
 
+	keyno = 0;
 	which = next ? "next" : "prev";
 
-	keyno = 0;
-	ret = next ? cursor->next(cursor) : cursor->prev(cursor);
-	if (ret == WT_ROLLBACK)
-		return (WT_ROLLBACK);
-	if (ret == 0)
+	switch (ret = (next ? cursor->next(cursor) : cursor->prev(cursor))) {
+	case 0:
 		switch (g.type) {
 		case FIX:
 			if ((ret = cursor->get_key(cursor, &keyno)) == 0 &&
@@ -722,13 +940,20 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
 				ret = cursor->get_value(cursor, &value);
 			break;
 		}
-	if (ret != 0 && ret != WT_NOTFOUND)
+		if (ret != 0)
+			testutil_die(ret, "nextprev: get_key/get_value");
+		break;
+	case WT_NOTFOUND:
+		break;
+	case WT_ROLLBACK:
+		return (WT_ROLLBACK);
+	default:
 		testutil_die(ret, "%s", which);
-	*notfoundp = (ret == WT_NOTFOUND);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
-		return (0);
+		return (ret);
 
 	{
 	WT_ITEM bdb_key, bdb_value;
@@ -743,7 +968,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
 	    &bdb_value.data, &bdb_value.size, &notfound);
 	if (notfound_chk(
 	    next ? "nextprev(next)" : "nextprev(prev)", ret, notfound, keyno))
-		return (0);
+		return (ret);
 
 	/* Compare the two. */
 	if (g.type == ROW) {
@@ -794,7 +1019,7 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
 		}
 	}
 #endif
-	return (0);
+	return (ret);
 }
 
 /*
@@ -802,43 +1027,38 @@ nextprev(WT_CURSOR *cursor, int next, int *notfoundp)
  *	Update a row in a row-store file.
  */
 static int
-row_update(TINFO *tinfo,
-    WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
+row_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	session = cursor->session;
 
-	key_gen((uint8_t *)key->data, &key->size, keyno);
-	val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno);
-
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
 		(void)g.wt_api->msg_printf(g.wt_api, session,
-		    "%-10s{%.*s}\n%-10s{%.*s}",
-		    "putK", (int)key->size, (char *)key->data,
-		    "putV", (int)value->size, (char *)value->data);
+		    "%-10s{%.*s}, {%.*s}",
+		    "put",
+		    (int)key->size, key->data, (int)value->size, value->data);
 
 	cursor->set_key(cursor, key);
 	cursor->set_value(cursor, value);
-	ret = cursor->update(cursor);
-	if (ret == WT_ROLLBACK)
+	switch (ret = cursor->update(cursor)) {
+	case 0:
+		break;
+	case WT_CACHE_FULL:
+	case WT_ROLLBACK:
 		return (WT_ROLLBACK);
-	if (ret != 0 && ret != WT_NOTFOUND)
+	default:
 		testutil_die(ret,
 		    "row_update: update row %" PRIu64 " by key", keyno);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
 		return (0);
 
-	{
-	int notfound;
-
-	bdb_update(key->data, key->size, value->data, value->size, &notfound);
-	(void)notfound_chk("row_update", ret, notfound, keyno);
-	}
+	bdb_update(key->data, key->size, value->data, value->size);
 #endif
 	return (0);
 }
@@ -848,16 +1068,13 @@ row_update(TINFO *tinfo,
  *	Update a row in a column-store file.
  */
 static int
-col_update(TINFO *tinfo,
-    WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
+col_update(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	session = cursor->session;
 
-	val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno);
-
 	/* Log the operation */
 	if (g.logging == LOG_OPS) {
 		if (g.type == FIX)
@@ -877,23 +1094,22 @@ col_update(TINFO *tinfo,
 		cursor->set_value(cursor, *(uint8_t *)value->data);
 	else
 		cursor->set_value(cursor, value);
-	ret = cursor->update(cursor);
-	if (ret == WT_ROLLBACK)
+	switch (ret = cursor->update(cursor)) {
+	case 0:
+		break;
+	case WT_CACHE_FULL:
+	case WT_ROLLBACK:
 		return (WT_ROLLBACK);
-	if (ret != 0 && ret != WT_NOTFOUND)
+	default:
 		testutil_die(ret, "col_update: %" PRIu64, keyno);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
 		return (0);
 
-	{
-	int notfound;
-
-	key_gen((uint8_t *)key->data, &key->size, keyno);
-	bdb_update(key->data, key->size, value->data, value->size, &notfound);
-	(void)notfound_chk("col_update", ret, notfound, keyno);
-	}
+	key_gen(key, keyno);
+	bdb_update(key->data, key->size, value->data, value->size);
 #else
 	(void)key;				/* [-Wunused-variable] */
 #endif
@@ -912,8 +1128,7 @@ table_append_init(void)
 	g.append_cnt = 0;
 
 	free(g.append);
-	if ((g.append = calloc(g.append_max, sizeof(uint64_t))) == NULL)
-		testutil_die(errno, "calloc");
+	g.append = dcalloc(g.append_max, sizeof(uint64_t));
 }
 
 /*
@@ -1005,43 +1220,38 @@ table_append(uint64_t keyno)
  *	Insert a row in a row-store file.
  */
 static int
-row_insert(TINFO *tinfo,
-    WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
+row_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t keyno)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	session = cursor->session;
 
-	key_gen_insert(&tinfo->rnd, (uint8_t *)key->data, &key->size, keyno);
-	val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, keyno);
-
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
 		(void)g.wt_api->msg_printf(g.wt_api, session,
-		    "%-10s{%.*s}\n%-10s{%.*s}",
-		    "insertK", (int)key->size, (char *)key->data,
-		    "insertV", (int)value->size, (char *)value->data);
+		    "%-10s{%.*s}, {%.*s}",
+		    "insert",
+		    (int)key->size, key->data, (int)value->size, value->data);
 
 	cursor->set_key(cursor, key);
 	cursor->set_value(cursor, value);
-	ret = cursor->insert(cursor);
-	if (ret == WT_ROLLBACK)
+	switch (ret = cursor->insert(cursor)) {
+	case 0:
+		break;
+	case WT_CACHE_FULL:
+	case WT_ROLLBACK:
 		return (WT_ROLLBACK);
-	if (ret != 0 && ret != WT_NOTFOUND)
+	default:
 		testutil_die(ret,
 		    "row_insert: insert row %" PRIu64 " by key", keyno);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
 		return (0);
 
-	{
-	int notfound;
-
-	bdb_update(key->data, key->size, value->data, value->size, &notfound);
-	(void)notfound_chk("row_insert", ret, notfound, keyno);
-	}
+	bdb_update(key->data, key->size, value->data, value->size);
 #endif
 	return (0);
 }
@@ -1051,24 +1261,25 @@ row_insert(TINFO *tinfo,
  *	Insert an element in a column-store file.
  */
 static int
-col_insert(TINFO *tinfo,
-    WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop)
+col_insert(WT_CURSOR *cursor, WT_ITEM *key, WT_ITEM *value, uint64_t *keynop)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
 	uint64_t keyno;
-	int ret;
 
 	session = cursor->session;
 
-	val_gen(&tinfo->rnd, (uint8_t *)value->data, &value->size, g.rows + 1);
-
 	if (g.type == FIX)
 		cursor->set_value(cursor, *(uint8_t *)value->data);
 	else
 		cursor->set_value(cursor, value);
-	if ((ret = cursor->insert(cursor)) != 0) {
-		if (ret == WT_ROLLBACK)
-			return (WT_ROLLBACK);
+	switch (ret = cursor->insert(cursor)) {
+	case 0:
+		break;
+	case WT_CACHE_FULL:
+	case WT_ROLLBACK:
+		return (WT_ROLLBACK);
+	default:
 		testutil_die(ret, "cursor.insert");
 	}
 	testutil_check(cursor->get_key(cursor, &keyno));
@@ -1093,12 +1304,8 @@ col_insert(TINFO *tinfo,
 	if (!SINGLETHREADED)
 		return (0);
 
-	{
-	int notfound;
-
-	key_gen((uint8_t *)key->data, &key->size, keyno);
-	bdb_update(key->data, key->size, value->data, value->size, &notfound);
-	}
+	key_gen(key, keyno);
+	bdb_update(key->data, key->size, value->data, value->size);
 #else
 	(void)key;				/* [-Wunused-variable] */
 #endif
@@ -1110,14 +1317,14 @@ col_insert(TINFO *tinfo,
  *	Remove an row from a row-store file.
  */
 static int
-row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
+row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	session = cursor->session;
 
-	key_gen((uint8_t *)key->data, &key->size, keyno);
+	key_gen(key, keyno);
 
 	/* Log the operation */
 	if (g.logging == LOG_OPS)
@@ -1128,16 +1335,20 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
 	/* We use the cursor in overwrite mode, check for existence. */
 	if ((ret = cursor->search(cursor)) == 0)
 		ret = cursor->remove(cursor);
-	if (ret == WT_ROLLBACK)
+	switch (ret) {
+	case 0:
+	case WT_NOTFOUND:
+		break;
+	case WT_ROLLBACK:
 		return (WT_ROLLBACK);
-	if (ret != 0 && ret != WT_NOTFOUND)
+	default:
 		testutil_die(ret,
 		    "row_remove: remove %" PRIu64 " by key", keyno);
-	*notfoundp = (ret == WT_NOTFOUND);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
-		return (0);
+		return (ret);
 
 	{
 	int notfound;
@@ -1148,7 +1359,7 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
 #else
 	(void)key;				/* [-Wunused-variable] */
 #endif
-	return (0);
+	return (ret);
 }
 
 /*
@@ -1156,10 +1367,10 @@ row_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
  *	Remove a row from a column-store file.
  */
 static int
-col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
+col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno)
 {
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	session = cursor->session;
 
@@ -1172,35 +1383,38 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, int *notfoundp)
 	/* We use the cursor in overwrite mode, check for existence. */
 	if ((ret = cursor->search(cursor)) == 0)
 		ret = cursor->remove(cursor);
-	if (ret == WT_ROLLBACK)
+	switch (ret) {
+	case 0:
+	case WT_NOTFOUND:
+		break;
+	case WT_ROLLBACK:
 		return (WT_ROLLBACK);
-	if (ret != 0 && ret != WT_NOTFOUND)
+	default:
 		testutil_die(ret,
 		    "col_remove: remove %" PRIu64 " by key", keyno);
-	*notfoundp = (ret == WT_NOTFOUND);
+	}
 
 #ifdef HAVE_BERKELEY_DB
 	if (!SINGLETHREADED)
-		return (0);
-
-	{
-	int notfound;
+		return (ret);
 
 	/*
 	 * Deleting a fixed-length item is the same as setting the bits to 0;
 	 * do the same thing for the BDB store.
 	 */
 	if (g.type == FIX) {
-		key_gen((uint8_t *)key->data, &key->size, keyno);
-		bdb_update(key->data, key->size, "\0", 1, &notfound);
-	} else
+		key_gen(key, keyno);
+		bdb_update(key->data, key->size, "\0", 1);
+	} else {
+		int notfound;
+
 		bdb_remove(keyno, &notfound);
-	(void)notfound_chk("col_remove", ret, notfound, keyno);
+		(void)notfound_chk("col_remove", ret, notfound, keyno);
 	}
 #else
 	(void)key;				/* [-Wunused-variable] */
 #endif
-	return (0);
+	return (ret);
 }
 
 #ifdef HAVE_BERKELEY_DB
diff --git a/test/format/salvage.c b/test/format/salvage.c
index 526e1563390..8274c556364 100644
--- a/test/format/salvage.c
+++ b/test/format/salvage.c
@@ -36,8 +36,8 @@ static void
 salvage(void)
 {
 	WT_CONNECTION *conn;
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	conn = g.wts_conn;
 	track("salvage", 0ULL, NULL);
@@ -141,7 +141,7 @@ found:	if (fstat(fd, &sb) == -1)
 void
 wts_salvage(void)
 {
-	int ret;
+	WT_DECL_RET;
 
 	/* Some data-sources don't support salvage. */
 	if (DATASOURCE("helium") || DATASOURCE("kvsbdb"))
@@ -158,7 +158,7 @@ wts_salvage(void)
 		testutil_die(ret, "salvage copy step failed");
 
 	/* Salvage, then verify. */
-	wts_open(g.home, 1, &g.wts_conn);
+	wts_open(g.home, true, &g.wts_conn);
 	salvage();
 	wts_verify("post-salvage verify");
 	wts_close();
@@ -174,7 +174,7 @@ wts_salvage(void)
 
 	/* Corrupt the file randomly, salvage, then verify. */
 	if (corrupt()) {
-		wts_open(g.home, 1, &g.wts_conn);
+		wts_open(g.home, true, &g.wts_conn);
 		salvage();
 		wts_verify("post-corrupt-salvage verify");
 		wts_close();
diff --git a/test/format/t.c b/test/format/t.c
index 28c22e23cb8..085163befe2 100644
--- a/test/format/t.c
+++ b/test/format/t.c
@@ -181,6 +181,7 @@ main(int argc, char *argv[])
 	 */
 	testutil_check(pthread_rwlock_init(&g.append_lock, NULL));
 	testutil_check(pthread_rwlock_init(&g.backup_lock, NULL));
+	testutil_check(pthread_rwlock_init(&g.checkpoint_lock, NULL));
 	testutil_check(pthread_rwlock_init(&g.death_lock, NULL));
 
 	printf("%s: process %" PRIdMAX "\n", g.progname, (intmax_t)getpid());
@@ -198,8 +199,8 @@ main(int argc, char *argv[])
 		if (SINGLETHREADED)
 			bdb_open();		/* Initial file config */
 #endif
-		wts_open(g.home, 1, &g.wts_conn);
-		wts_create();
+		wts_open(g.home, true, &g.wts_conn);
+		wts_init();
 
 		wts_load();			/* Load initial records */
 		wts_verify("post-bulk verify");	/* Verify */
@@ -275,6 +276,8 @@ main(int argc, char *argv[])
 
 	testutil_check(pthread_rwlock_destroy(&g.append_lock));
 	testutil_check(pthread_rwlock_destroy(&g.backup_lock));
+	testutil_check(pthread_rwlock_destroy(&g.checkpoint_lock));
+	testutil_check(pthread_rwlock_destroy(&g.death_lock));
 
 	config_clear();
 
@@ -288,7 +291,7 @@ main(int argc, char *argv[])
 static void
 startup(void)
 {
-	int ret;
+	WT_DECL_RET;
 
 	/* Flush/close any logging information. */
 	fclose_and_clear(&g.logfp);
diff --git a/test/format/util.c b/test/format/util.c
index 2e4c869366c..f2b4d18029e 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -32,56 +32,11 @@
 #define	MAX(a, b)	(((a) > (b)) ? (a) : (b))
 #endif
 
-/*
- * dmalloc --
- *	Call malloc, dying on failure.
- */
-void *
-dmalloc(size_t len)
-{
-	void *p;
-
-	if ((p = malloc(len)) == NULL)
-		testutil_die(errno, "malloc");
-	return (p);
-}
-
-/*
- * dstrdup --
- *	Call strdup, dying on failure.
- */
-char *
-dstrdup(const char *str)
-{
-	char *p;
-
-	if ((p = strdup(str)) == NULL)
-		testutil_die(errno, "strdup");
-	return (p);
-}
-
-static inline uint32_t
-kv_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max)
-{
-	/*
-	 * Focus on relatively small key/value items, admitting the possibility
-	 * of larger items.  Pick a size close to the minimum most of the time,
-	 * only create a larger item 1 in 20 times, and a really big item 1 in
-	 * 1000 times. (Configuration can force large key/value minimum sizes,
-	 * where every key/value item is an overflow.)
-	 */
-	if (keyno % 1000 == 0 && max < KILOBYTE(80)) {
-		min = KILOBYTE(80);
-		max = KILOBYTE(100);
-	} else if (keyno % 20 != 0 && max > min + 20)
-		max = min + 20;
-	return (mmrand(rnd, min, max));
-}
-
 void
 key_len_setup(void)
 {
 	size_t i;
+	uint32_t max;
 
 	/*
 	 * The key is a variable length item with a leading 10-digit value.
@@ -91,72 +46,113 @@ key_len_setup(void)
 	 * the pre-loaded lengths.
 	 *
 	 * Fill in the random key lengths.
+	 *
+	 * Focus on relatively small items, admitting the possibility of larger
+	 * items. Pick a size close to the minimum most of the time, only create
+	 * a larger item 1 in 20 times.
 	 */
-	for (i = 0; i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i)
-		g.key_rand_len[i] =
-		    kv_len(NULL, (uint64_t)i, g.c_key_min, g.c_key_max);
+	for (i = 0;
+	    i < sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]); ++i) {
+		max = g.c_key_max;
+		if (i % 20 != 0 && max > g.c_key_min + 20)
+			max = g.c_key_min + 20;
+		g.key_rand_len[i] = mmrand(NULL, g.c_key_min, max);
+	}
 }
 
 void
-key_gen_setup(uint8_t **keyp)
+key_gen_setup(WT_ITEM *key)
 {
-	uint8_t *key;
 	size_t i, len;
-
-	*keyp = NULL;
+	char *p;
 
 	len = MAX(KILOBYTE(100), g.c_key_max);
-	key = dmalloc(len);
+	p = dmalloc(len);
 	for (i = 0; i < len; ++i)
-		key[i] = (uint8_t)("abcdefghijklmnopqrstuvwxyz"[i % 26]);
-	*keyp = key;
+		p[i] = "abcdefghijklmnopqrstuvwxyz"[i % 26];
+
+	key->mem = p;
+	key->memsize = len;
+	key->data = key->mem;
+	key->size = 0;
 }
 
 static void
-key_gen_common(uint8_t *key, size_t *sizep, uint64_t keyno, int suffix)
+key_gen_common(WT_ITEM *key, uint64_t keyno, int suffix)
 {
 	int len;
+	char *p;
+
+	p = key->mem;
 
 	/*
 	 * The key always starts with a 10-digit string (the specified cnt)
 	 * followed by two digits, a random number between 1 and 15 if it's
 	 * an insert, otherwise 00.
 	 */
-	len = sprintf((char *)key, "%010" PRIu64 ".%02d", keyno, suffix);
+	len = sprintf(p, "%010" PRIu64 ".%02d", keyno, suffix);
 
 	/*
-	 * In a column-store, the key is only used for BDB, and so it doesn't
-	 * need a random length.
+	 * In a column-store, the key is only used for Berkeley DB inserts,
+	 * and so it doesn't need a random length.
 	 */
 	if (g.type == ROW) {
-		key[len] = '/';
-		len = (int)g.key_rand_len[keyno %
-		    (sizeof(g.key_rand_len) / sizeof(g.key_rand_len[0]))];
+		p[len] = '/';
+
+		/*
+		 * Because we're doing table lookup for key sizes, we weren't
+		 * able to set really big keys sizes in the table, the table
+		 * isn't big enough to keep our hash from selecting too many
+		 * big keys and blowing out the cache. Handle that here, use a
+		 * really big key 1 in 2500 times.
+		 */
+		len = keyno % 2500 == 0 && g.c_key_max < KILOBYTE(80) ?
+		    KILOBYTE(80) :
+		    (int)g.key_rand_len[keyno % WT_ELEMENTS(g.key_rand_len)];
 	}
-	*sizep = (size_t)len;
+
+	key->data = key->mem;
+	key->size = (size_t)len;
 }
 
 void
-key_gen(uint8_t *key, size_t *sizep, uint64_t keyno)
+key_gen(WT_ITEM *key, uint64_t keyno)
 {
-	key_gen_common(key, sizep, keyno, 0);
+	key_gen_common(key, keyno, 0);
 }
 
 void
-key_gen_insert(WT_RAND_STATE *rnd, uint8_t *key, size_t *sizep, uint64_t keyno)
+key_gen_insert(WT_RAND_STATE *rnd, WT_ITEM *key, uint64_t keyno)
 {
-	key_gen_common(key, sizep, keyno, (int)mmrand(rnd, 1, 15));
+	key_gen_common(key, keyno, (int)mmrand(rnd, 1, 15));
 }
 
 static uint32_t val_dup_data_len;	/* Length of duplicate data items */
 
+static inline uint32_t
+value_len(WT_RAND_STATE *rnd, uint64_t keyno, uint32_t min, uint32_t max)
+{
+	/*
+	 * Focus on relatively small items, admitting the possibility of larger
+	 * items. Pick a size close to the minimum most of the time, only create
+	 * a larger item 1 in 20 times, and a really big item 1 in somewhere
+	 * around 2500 items.
+	 */
+	if (keyno % 2500 == 0 && max < KILOBYTE(80)) {
+		min = KILOBYTE(80);
+		max = KILOBYTE(100);
+	} else if (keyno % 20 != 0 && max > min + 20)
+		max = min + 20;
+	return (mmrand(rnd, min, max));
+}
+
 void
-val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp)
+val_gen_setup(WT_RAND_STATE *rnd, WT_ITEM *value)
 {
-	uint8_t *val;
 	size_t i, len;
+	char *p;
 
-	*valp = NULL;
+	memset(value, 0, sizeof(WT_ITEM));
 
 	/*
 	 * Set initial buffer contents to recognizable text.
@@ -166,35 +162,43 @@ val_gen_setup(WT_RAND_STATE *rnd, uint8_t **valp)
 	 * data for column-store run-length encoded files.
 	 */
 	len = MAX(KILOBYTE(100), g.c_value_max) + 20;
-	val = dmalloc(len);
+	p = dmalloc(len);
 	for (i = 0; i < len; ++i)
-		val[i] = (uint8_t)("ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26]);
+		p[i] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"[i % 26];
 
-	*valp = val;
+	value->mem = p;
+	value->memsize = len;
+	value->data = value->mem;
+	value->size = 0;
 
-	val_dup_data_len = kv_len(rnd,
+	val_dup_data_len = value_len(rnd,
 	    (uint64_t)mmrand(rnd, 1, 20), g.c_value_min, g.c_value_max);
 }
 
 void
-val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno)
+val_gen(WT_RAND_STATE *rnd, WT_ITEM *value, uint64_t keyno)
 {
+	char *p;
+
+	p = value->mem;
+	value->data = value->mem;
+
 	/*
 	 * Fixed-length records: take the low N bits from the last digit of
 	 * the record number.
 	 */
 	if (g.type == FIX) {
 		switch (g.c_bitcnt) {
-		case 8: val[0] = (uint8_t)mmrand(rnd, 1, 0xff); break;
-		case 7: val[0] = (uint8_t)mmrand(rnd, 1, 0x7f); break;
-		case 6: val[0] = (uint8_t)mmrand(rnd, 1, 0x3f); break;
-		case 5: val[0] = (uint8_t)mmrand(rnd, 1, 0x1f); break;
-		case 4: val[0] = (uint8_t)mmrand(rnd, 1, 0x0f); break;
-		case 3: val[0] = (uint8_t)mmrand(rnd, 1, 0x07); break;
-		case 2: val[0] = (uint8_t)mmrand(rnd, 1, 0x03); break;
-		case 1: val[0] = 1; break;
+		case 8: p[0] = (char)mmrand(rnd, 1, 0xff); break;
+		case 7: p[0] = (char)mmrand(rnd, 1, 0x7f); break;
+		case 6: p[0] = (char)mmrand(rnd, 1, 0x3f); break;
+		case 5: p[0] = (char)mmrand(rnd, 1, 0x1f); break;
+		case 4: p[0] = (char)mmrand(rnd, 1, 0x0f); break;
+		case 3: p[0] = (char)mmrand(rnd, 1, 0x07); break;
+		case 2: p[0] = (char)mmrand(rnd, 1, 0x03); break;
+		case 1: p[0] = 1; break;
 		}
-		*sizep = 1;
+		value->size = 1;
 		return;
 	}
 
@@ -203,8 +207,8 @@ val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno)
 	 * test that by inserting a zero-length data item every so often.
 	 */
 	if (keyno % 63 == 0) {
-		val[0] = '\0';
-		*sizep = 0;
+		p[0] = '\0';
+		value->size = 0;
 		return;
 	}
 
@@ -219,13 +223,14 @@ val_gen(WT_RAND_STATE *rnd, uint8_t *val, size_t *sizep, uint64_t keyno)
 	if ((g.type == ROW || g.type == VAR) &&
 	    g.c_repeat_data_pct != 0 &&
 	    mmrand(rnd, 1, 100) < g.c_repeat_data_pct) {
-		(void)strcpy((char *)val, "DUPLICATEV");
-		val[10] = '/';
-		*sizep = val_dup_data_len;
+		(void)strcpy(p, "DUPLICATEV");
+		p[10] = '/';
+		value->size = val_dup_data_len;
 	} else {
-		(void)sprintf((char *)val, "%010" PRIu64, keyno);
-		val[10] = '/';
-		*sizep = kv_len(rnd, keyno, g.c_value_min, g.c_value_max);
+		(void)sprintf(p, "%010" PRIu64, keyno);
+		p[10] = '/';
+		value->size =
+		    value_len(rnd, keyno, g.c_value_min, g.c_value_max);
 	}
 }
 
@@ -305,15 +310,6 @@ path_setup(const char *home)
 	g.home_stats = dmalloc(len);
 	snprintf(g.home_stats, len, "%s/%s", g.home, "stats");
 
-	/* Backup directory. */
-	len = strlen(g.home) + strlen("BACKUP") + 2;
-	g.home_backup = dmalloc(len);
-	snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP");
-
-	len = strlen(g.home) + strlen("BACKUP2") + 2;
-	g.home_backup2 = dmalloc(len);
-	snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2");
-
 	/* BDB directory. */
 	len = strlen(g.home) + strlen("bdb") + 2;
 	g.home_bdb = dmalloc(len);
@@ -341,18 +337,27 @@ path_setup(const char *home)
 	g.home_init = dmalloc(len);
 	snprintf(g.home_init, len, CMD, g.home, g.home, g.home);
 
-	/* Backup directory initialize command, remove and re-create it. */
+	/* Primary backup directory. */
+	len = strlen(g.home) + strlen("BACKUP") + 2;
+	g.home_backup = dmalloc(len);
+	snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP");
+
+	/*
+	 * Backup directory initialize command, remove and re-create the primary
+	 * backup directory, plus a copy we maintain for recovery testing.
+	 */
 #undef	CMD
 #ifdef _WIN32
-#define	CMD	"del /s /q >:nul && mkdir %s %s"
+#define	CMD	"del %s/%s %s/%s /s /q >:nul && mkdir %s/%s %s/%s"
 #else
-#define	CMD	"rm -rf %s %s && mkdir %s %s"
+#define	CMD	"rm -rf %s/%s %s/%s && mkdir %s/%s %s/%s"
 #endif
-	len = strlen(g.home_backup) * 2 +
-	    strlen(g.home_backup2) * 2 + strlen(CMD) + 1;
+	len = strlen(g.home) * 4 +
+	    strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1;
 	g.home_backup_init = dmalloc(len);
-	snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2,
-	    g.home_backup, g.home_backup2);
+	snprintf(g.home_backup_init, len, CMD,
+	    g.home, "BACKUP", g.home, "BACKUP_COPY",
+	    g.home, "BACKUP", g.home, "BACKUP_COPY");
 
 	/*
 	 * Salvage command, save the interesting files so we can replay the
diff --git a/test/format/wts.c b/test/format/wts.c
index 81e484296e2..2ee01aa75b5 100644
--- a/test/format/wts.c
+++ b/test/format/wts.c
@@ -126,10 +126,10 @@ static WT_EVENT_HANDLER event_handler = {
  *	Open a connection to a WiredTiger database.
  */
 void
-wts_open(const char *home, int set_api, WT_CONNECTION **connp)
+wts_open(const char *home, bool set_api, WT_CONNECTION **connp)
 {
 	WT_CONNECTION *conn;
-	int ret;
+	WT_DECL_RET;
 	char *config, *end, *p, helium_config[1024];
 
 	*connp = NULL;
@@ -138,10 +138,11 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
 	end = config + sizeof(g.wiredtiger_open_config);
 
 	p += snprintf(p, REMAIN(p, end),
-	    "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB",
-	    g.c_cache);
-
-	p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname);
+	    "create=true,"
+	    "cache_size=%" PRIu32 "MB,"
+	    "checkpoint_sync=false,"
+	    "error_prefix=\"%s\"",
+	    g.c_cache, g.progname);
 
 	/* In-memory configuration. */
 	if (g.c_in_memory != 0)
@@ -273,8 +274,13 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
 void
 wts_reopen(void)
 {
+	WT_CONNECTION *conn;
+
 	testutil_checkfmt(wiredtiger_open(g.home, &event_handler,
-	    g.wiredtiger_open_config, &g.wts_conn), "%s", g.home);
+	    g.wiredtiger_open_config, &conn), "%s", g.home);
+
+	g.wt_api = conn->get_extension_api(conn);
+	g.wts_conn = conn;
 }
 
 /*
@@ -282,7 +288,7 @@ wts_reopen(void)
  *	Create the underlying store.
  */
 void
-wts_create(void)
+wts_init(void)
 {
 	WT_CONNECTION *conn;
 	WT_SESSION *session;
@@ -497,8 +503,8 @@ void
 wts_verify(const char *tag)
 {
 	WT_CONNECTION *conn;
+	WT_DECL_RET;
 	WT_SESSION *session;
-	int ret;
 
 	if (g.c_verify == 0)
 		return;
@@ -531,12 +537,12 @@ wts_stats(void)
 {
 	WT_CONNECTION *conn;
 	WT_CURSOR *cursor;
+	WT_DECL_RET;
 	WT_SESSION *session;
 	FILE *fp;
 	char *stat_name;
 	const char *pval, *desc;
 	uint64_t v;
-	int ret;
 
 	/* Ignore statistics if they're not configured. */
 	if (g.c_statistics == 0)