1 files changed, 62 insertions, 41 deletions

diff --git a/src/third_party/wiredtiger/test/format/ops.c b/src/third_party/wiredtiger/test/format/ops.c
index 8e118772500..f4770465628 100644
--- a/src/third_party/wiredtiger/test/format/ops.c
+++ b/src/third_party/wiredtiger/test/format/ops.c
@@ -76,10 +76,10 @@ wts_ops(int lastrun)
 	TINFO **tinfo_list, *tinfo, total;
 	WT_CONNECTION *conn;
 	WT_SESSION *session;
-	wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid;
-	int64_t fourths, thread_ops;
+	wt_thread_t alter_tid, backup_tid, compact_tid, lrt_tid, timestamp_tid;
+	int64_t fourths, quit_fourths, thread_ops;
 	uint32_t i;
-	int running;
+	bool running;
 
 	conn = g.wts_conn;
 
@@ -88,16 +88,20 @@ wts_ops(int lastrun)
 	memset(&backup_tid, 0, sizeof(backup_tid));
 	memset(&compact_tid, 0, sizeof(compact_tid));
 	memset(&lrt_tid, 0, sizeof(lrt_tid));
+	memset(&timestamp_tid, 0, sizeof(timestamp_tid));
 
 	modify_repl_init();
 
 	/*
 	 * There are two mechanisms to specify the length of the run, a number
 	 * of operations and a timer, when either expire the run terminates.
+	 *
 	 * Each thread does an equal share of the total operations (and make
 	 * sure that it's not 0).
 	 *
-	 * Calculate how many fourth-of-a-second sleeps until any timer expires.
+	 * Calculate how many fourth-of-a-second sleeps until the timer expires.
+	 * If the timer expires and threads don't return in 15 minutes, assume
+	 * there is something hung, and force the quit.
 	 */
 	if (g.c_ops == 0)
 		thread_ops = -1;
@@ -107,9 +111,11 @@ wts_ops(int lastrun)
 		thread_ops = g.c_ops / g.c_threads;
 	}
 	if (g.c_timer == 0)
-		fourths = -1;
-	else
+		fourths = quit_fourths = -1;
+	else {
 		fourths = ((int64_t)g.c_timer * 4 * 60) / FORMAT_OPERATION_REPS;
+		quit_fourths = fourths + 15 * 4 * 60;
+	}
 
 	/* Initialize the table extension code. */
 	table_append_init();
@@ -119,7 +125,7 @@ wts_ops(int lastrun)
 	 * after threaded operations start, there's no point.
 	 */
 	if (!SINGLETHREADED)
-		g.rand_log_stop = 1;
+		g.rand_log_stop = true;
 
 	/* Open a session. */
 	if (g.logging != 0) {
@@ -135,7 +141,23 @@ wts_ops(int lastrun)
 	tinfo_list = dcalloc((size_t)g.c_threads, sizeof(TINFO *));
 	for (i = 0; i < g.c_threads; ++i) {
 		tinfo_list[i] = tinfo = dcalloc(1, sizeof(TINFO));
+
 		tinfo->id = (int)i + 1;
+
+		/*
+		 * Characterize the per-thread random number generator. Normally
+		 * we want independent behavior so threads start in different
+		 * parts of the RNG space, but we've found bugs by having the
+		 * threads pound on the same key/value pairs, that is, by making
+		 * them traverse the same RNG space. 75% of the time we run in
+		 * independent RNG space.
+		 */
+		if (g.c_independent_thread_rng)
+			__wt_random_init_seed(
+			    (WT_SESSION_IMPL *)session, &tinfo->rnd);
+		else
+			__wt_random_init(&tinfo->rnd);
+
 		tinfo->state = TINFO_RUNNING;
 		testutil_check(
 		    __wt_thread_create(NULL, &tinfo->tid, ops, tinfo));
@@ -156,12 +178,15 @@ wts_ops(int lastrun)
 		    __wt_thread_create(NULL, &compact_tid, compact, NULL));
 	if (!SINGLETHREADED && g.c_long_running_txn)
 		testutil_check(__wt_thread_create(NULL, &lrt_tid, lrt, NULL));
+	if (g.c_txn_timestamps)
+		testutil_check(__wt_thread_create(
+		    NULL, &timestamp_tid, timestamp, tinfo_list));
 
 	/* Spin on the threads, calculating the totals. */
 	for (;;) {
 		/* Clear out the totals each pass. */
 		memset(&total, 0, sizeof(total));
-		for (i = 0, running = 0; i < g.c_threads; ++i) {
+		for (i = 0, running = false; i < g.c_threads; ++i) {
 			tinfo = tinfo_list[i];
 			total.commit += tinfo->commit;
 			total.deadlock += tinfo->deadlock;
@@ -173,7 +198,7 @@ wts_ops(int lastrun)
 
 			switch (tinfo->state) {
 			case TINFO_RUNNING:
-				running = 1;
+				running = true;
 				break;
 			case TINFO_COMPLETE:
 				tinfo->state = TINFO_JOINED;
@@ -199,22 +224,25 @@ wts_ops(int lastrun)
 					static char *core = NULL;
 					*core = 0;
 				}
-				tinfo->quit = 1;
+				tinfo->quit = true;
 			}
 		}
 		track("ops", 0ULL, &total);
 		if (!running)
 			break;
-		(void)usleep(250000);		/* 1/4th of a second */
+		__wt_sleep(0, 250000);		/* 1/4th of a second */
 		if (fourths != -1)
 			--fourths;
+		if (quit_fourths != -1 && --quit_fourths == 0) {
+			fprintf(stderr, "%s\n",
+			    "format run exceeded 15 minutes past the maximum "
+			    "time, aborting the process.");
+			abort();
+		}
 	}
-	for (i = 0; i < g.c_threads; ++i)
-		free(tinfo_list[i]);
-	free(tinfo_list);
 
 	/* Wait for the other threads. */
-	g.workers_finished = 1;
+	g.workers_finished = true;
 	if (g.c_alter)
 		testutil_check(__wt_thread_join(NULL, alter_tid));
 	if (g.c_backups)
@@ -223,13 +251,19 @@ wts_ops(int lastrun)
 		testutil_check(__wt_thread_join(NULL, compact_tid));
 	if (!SINGLETHREADED && g.c_long_running_txn)
 		testutil_check(__wt_thread_join(NULL, lrt_tid));
-	g.workers_finished = 0;
+	if (g.c_txn_timestamps)
+		testutil_check(__wt_thread_join(NULL, timestamp_tid));
+	g.workers_finished = false;
 
 	if (g.logging != 0) {
 		(void)g.wt_api->msg_printf(g.wt_api, session,
 		    "=============== thread ops stop ===============");
 		testutil_check(session->close(session, NULL));
 	}
+
+	for (i = 0; i < g.c_threads; ++i)
+		free(tinfo_list[i]);
+	free(tinfo_list);
 }
 
 /*
@@ -239,8 +273,8 @@ wts_ops(int lastrun)
 static inline u_int
 isolation_config(WT_RAND_STATE *rnd, WT_SESSION *session)
 {
-	const char *config;
 	u_int v;
+	const char *config;
 
 	if ((v = g.c_isolation_flag) == ISOLATION_RANDOM)
 		v = mmrand(rnd, 2, 4);
@@ -447,31 +481,21 @@ snap_check(WT_CURSOR *cursor,
 static void
 commit_transaction(TINFO *tinfo, WT_SESSION *session)
 {
-	WT_CONNECTION *conn;
 	uint64_t ts;
-	char *commit_conf, config_buf[64];
-
-	conn = g.wts_conn;
+	char config_buf[64];
 
 	if (g.c_txn_timestamps) {
 		ts = __wt_atomic_addv64(&g.timestamp, 1);
-
-		/* Periodically bump the oldest timestamp. */
-		if (ts > 100 && ts % 100 == 0) {
-			testutil_check(__wt_snprintf(
-			    config_buf, sizeof(config_buf),
-			    "oldest_timestamp=%" PRIx64, ts));
-			testutil_check(conn->set_timestamp(conn, config_buf));
-		}
-
 		testutil_check(__wt_snprintf(
 		    config_buf, sizeof(config_buf),
 		    "commit_timestamp=%" PRIx64, ts));
-		commit_conf = config_buf;
-	} else
-		commit_conf = NULL;
+		testutil_check(
+		    session->commit_transaction(session, config_buf));
 
-	testutil_check(session->commit_transaction(session, commit_conf));
+		/* After the commit, update our last timestamp. */
+		tinfo->timestamp = ts;
+	} else
+		testutil_check(session->commit_transaction(session, NULL));
 	++tinfo->commit;
 }
 
@@ -490,7 +514,7 @@ ops(void *arg)
 	WT_DECL_RET;
 	WT_ITEM *key, _key, *value, _value;
 	WT_SESSION *session;
-	uint64_t keyno, ckpt_op, reset_op, session_op;
+	uint64_t ckpt_op, keyno, reset_op, session_op;
 	uint32_t rnd;
 	u_int i, iso_config;
 	int dir;
@@ -507,9 +531,6 @@ ops(void *arg)
 	iso_config = 0;
 	memset(snap_list, 0, sizeof(snap_list));
 
-	/* Initialize the per-thread random number generator. */
-	__wt_random_init(&tinfo->rnd);
-
 	/* Set up the default key and value buffers. */
 	key = &_key;
 	key_gen_init(key);
@@ -1524,7 +1545,7 @@ table_append_init(void)
 static void
 table_append(uint64_t keyno)
 {
-	uint64_t *p, *ep;
+	uint64_t *ep, *p;
 	int done;
 
 	ep = g.append + g.append_max;
@@ -1597,7 +1618,7 @@ table_append(uint64_t keyno)
 
 		if (done)
 			break;
-		sleep(1);
+		__wt_sleep(1, 0);
 	}
 }
 
@@ -1792,7 +1813,7 @@ col_remove(WT_CURSOR *cursor, WT_ITEM *key, uint64_t keyno, bool positioned)
 	 */
 	if (g.type == FIX) {
 		key_gen(key, keyno);
-		bdb_update(key->data, key->size, "\0", 1);
+		bdb_update(key->data, key->size, "", 1);
 	} else {
 		int notfound;