Allow replay of threaded runs.

6 files changed, 62 insertions, 53 deletions

diff --git a/dist/s_string.ok b/dist/s_string.ok
index 6e9d23242da..f1e7bb8d80f 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -158,6 +158,7 @@ Memrata
 Metadata
 Mewhort
 Multi
+Multithreaded
 Mutex
 Mutexes
 NEEDKEY
@@ -728,6 +729,7 @@ ret
 retp
 rf
 rle
+rng
 rpc
 rref
 run's
diff --git a/test/format/config.c b/test/format/config.c
index 8027c0c8b1b..f3d0f7d698a 100644
--- a/test/format/config.c
+++ b/test/format/config.c
@@ -152,10 +152,6 @@ config_setup(void)
 			if (cp->flags & C_OPS)
 				*cp->v = 0;
 
-	/* Multi-threaded runs cannot be replayed. */
-	if (g.replay && !SINGLETHREADED)
-		die(0, "-r is incompatible with threaded runs");
-
 	/*
 	 * Periodically, set the delete percentage to 0 so salvage gets run,
 	 * as long as the delete percentage isn't nailed down.
diff --git a/test/format/format.h b/test/format/format.h
index 28306dee757..572fa6b5171 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -72,7 +72,7 @@ extern WT_EXTENSION_API *wt_api;
 #define	M(v)		((v) * 1000000)		/* Million */
 
 /* Get a random value between a min/max pair. */
-#define	MMRAND(min, max)	(wts_rand() % (((max) + 1) - (min)) + (min))
+#define	MMRAND(min, max)	(rng() % (((max) + 1) - (min)) + (min))
 
 #define	WT_NAME	"wt"				/* Object name */
 
@@ -100,6 +100,7 @@ typedef struct {
 	WT_CONNECTION	 *wts_conn;
 	WT_EXTENSION_API *wt_api;
 
+	int   rand_log_stop;			/* Logging turned off */
 	FILE *rand_log;				/* Random number log */
 
 	uint32_t run_cnt;			/* Run counter */
@@ -237,6 +238,8 @@ void	 key_len_setup(void);
 void	 key_gen_setup(uint8_t **);
 void	 key_gen(uint8_t *, uint32_t *, uint64_t, int);
 void	 path_setup(const char *);
+uint32_t rng(void);
+void	 rng_init(void);
 void	 syserr(const char *);
 void	 track(const char *, uint64_t, TINFO *);
 void	 val_gen_setup(uint8_t **);
@@ -247,8 +250,6 @@ void	 wts_dump(const char *, int);
 void	 wts_load(void);
 void	 wts_open(const char *, int, WT_CONNECTION **);
 void	 wts_ops(void);
-uint32_t wts_rand(void);
-void	 wts_rand_init(void);
 void	 wts_read_scan(void);
 void	 wts_salvage(void);
 void	 wts_stats(void);
diff --git a/test/format/ops.c b/test/format/ops.c
index e2e49acaed2..5ab13021f03 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -56,6 +56,13 @@ wts_ops(void)
 
 	conn = g.wts_conn;
 
+	/*
+	 * We support replay of threaded runs, but don't log random numbers
+	 * after threaded operations start, there's no point.
+	 */
+	if (!SINGLETHREADED)
+		g.rand_log_stop = 1;
+
 	/* Initialize the table extension code. */
 	table_append_init();
 
@@ -303,7 +310,7 @@ ops(void *arg)
 		 * of deletes will mean fewer inserts and writes.  Modifications
 		 * are always followed by a read to confirm it worked.
 		 */
-		op = (uint32_t)(wts_rand() % 100);
+		op = (uint32_t)(rng() % 100);
 		if (op < g.c_delete_pct) {
 			++tinfo->remove;
 			switch (g.type) {
@@ -465,7 +472,7 @@ wts_read_scan(void)
 
 	/* Check a random subset of the records using the key. */
 	for (last_cnt = cnt = 0; cnt < g.key_cnt;) {
-		cnt += wts_rand() % 17 + 1;
+		cnt += rng() % 17 + 1;
 		if (cnt > g.rows)
 			cnt = g.rows;
 		if (cnt - last_cnt > 1000) {
diff --git a/test/format/t.c b/test/format/t.c
index 5f6a77bac7c..96d6e461e93 100644
--- a/test/format/t.c
+++ b/test/format/t.c
@@ -90,7 +90,6 @@ main(int argc, char *argv[])
 			break;
 		case 'r':			/* Replay a run */
 			g.replay = 1;
-			g.c_runs = 1;
 			break;
 		default:
 			usage();
@@ -117,6 +116,21 @@ main(int argc, char *argv[])
 	for (; *argv != NULL; ++argv)
 		config_single(*argv, 1);
 
+	/*
+	 * Multithreaded runs can be replayed: it's useful and we'll get the
+	 * configuration correct.  Obviously the order of operations changes,
+	 * warn the user.
+	 */
+	if (g.replay && !SINGLETHREADED)
+		printf("Warning: replaying a threaded run\n");
+
+	/*
+	 * Single-threaded runs historically exited after a single replay, which
+	 * makes sense when you're debugging, leave that semantic in place.
+	 */
+	if (g.replay && SINGLETHREADED)
+		g.c_runs = 1;
+
 	/* Use line buffering on stdout so status updates aren't buffered. */
 	(void)setvbuf(stdout, NULL, _IOLBF, 0);
 
@@ -132,12 +146,8 @@ main(int argc, char *argv[])
 	/* Clean up on signal. */
 	(void)signal(SIGINT, onint);
 
-	/*
-	 * Initialize the random number generator (don't reinitialize on each
-	 * new run, reinitializing won't be more random than continuing on from
-	 * the current state).
-	 */
-	wts_rand_init();
+	/* Seed the random number generator. */
+	srand((u_int)(0xdeadbeef ^ (u_int)time(NULL)));
 
 	/* Set up paths. */
 	path_setup(home);
@@ -257,7 +267,7 @@ startup(void)
 		g.logfp = NULL;
 	}
 
-	/* Close the random number file. */
+	/* Close the random number logging file. */
 	if (g.rand_log != NULL) {
 		(void)fclose(g.rand_log);
 		g.rand_log = NULL;
@@ -275,12 +285,23 @@ startup(void)
 	if (mkdir(g.home_kvs, 0777) != 0)
 		die(errno, "mkdir: %s", g.home_kvs);
 
-	/* Open/truncate the logging file. */
+	/*
+	 * Open/truncate the logging file; line buffer so we see up-to-date
+	 * information on error.
+	 */
 	if (g.logging != 0) {
 		if ((g.logfp = fopen(g.home_log, "w")) == NULL)
 			die(errno, "fopen: %s", g.home_log);
 		(void)setvbuf(g.logfp, NULL, _IOLBF, 0);
 	}
+
+	/*
+	 * Open/truncate the random number logging file; line buffer so we see
+	 * up-to-date information on error.
+	 */
+	if ((g.rand_log = fopen(g.home_rand, g.replay ? "r" : "w")) == NULL)
+		die(errno, "%s", g.home_rand);
+	(void)setvbuf(g.rand_log, NULL, _IOLBF, 0);
 }
 
 /*
diff --git a/test/format/util.c b/test/format/util.c
index 800e3e578ba..a2023048879 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -290,44 +290,31 @@ path_setup(const char *home)
 }
 
 /*
- * wts_rand_init --
- *	Initialize the random number generator.
- */
-void
-wts_rand_init(void)
-{
-	/* Seed the random number generator. */
-	if (!g.replay)
-		srand((u_int)(0xdeadbeef ^ (u_int)time(NULL)));
-}
-
-/*
- * wts_rand --
+ * rng --
  *	Return a random number.
  */
 uint32_t
-wts_rand(void)
+rng(void)
 {
 	char buf[64];
 	uint32_t r;
 
-	/* If we're threaded, it's not repeatable, ignore the log. */
-	if (!SINGLETHREADED)
-		return ((uint32_t)rand());
-
 	/*
 	 * We can entirely reproduce a run based on the random numbers used
 	 * in the initial run, plus the configuration files.  It would be
 	 * nice to just log the initial RNG seed, rather than logging every
-	 * random number generated, but we can't -- Berkeley DB calls rand()
-	 * internally, and so that messes up the pattern of random numbers
-	 * (and WT might call rand() in the future, who knows?)
+	 * random number generated, but we'd have to include our own RNG,
+	 * Berkeley DB calls rand() internally, and that messes up the pattern
+	 * of random numbers.
+	 *
+	 * Check g.replay and g.rand_log_stop: multithreaded runs log/replay
+	 * until they get to the operations phase, then turn off log/replay,
+	 * threaded operation order can't be replayed.
 	 */
-	if (g.replay) {
-		if (g.rand_log == NULL &&
-		   (g.rand_log = fopen(g.home_rand, "r")) == NULL)
-			die(errno, "fopen: %s", g.home_rand);
+	if (g.rand_log_stop)
+		return ((uint32_t)rand());
 
+	if (g.replay) {
 		if (fgets(buf, sizeof(buf), g.rand_log) == NULL) {
 			if (feof(g.rand_log)) {
 				fprintf(stderr,
@@ -335,18 +322,13 @@ wts_rand(void)
 				    "exiting\n");
 				exit(EXIT_SUCCESS);
 			}
-			die(errno, "feof: random number log");
+			die(errno, "random number log");
 		}
 
-		r = (uint32_t)strtoul(buf, NULL, 10);
-	} else {
-		if (g.rand_log == NULL) {
-			if ((g.rand_log = fopen(g.home_rand, "w")) == NULL)
-				die(errno, "fopen: %s", g.home_rand);
-			(void)setvbuf(g.rand_log, NULL, _IOLBF, 0);
-		}
-		r = (uint32_t)rand();
-		fprintf(g.rand_log, "%" PRIu32 "\n", r);
+		return ((uint32_t)strtoul(buf, NULL, 10));
 	}
+
+	r = (uint32_t)rand();
+	fprintf(g.rand_log, "%" PRIu32 "\n", r);
 	return (r);
 }