Add a -F option to thread, to create a separate file for every thread.

With this option, the command line combination "-FS -R 0 -W 10" will
trigger the checkpoint problem where closing a file fails because there
is an update for the file we can't write.

Reference #844, #957.

5 files changed, 91 insertions, 38 deletions

diff --git a/test/thread/file.c b/test/thread/file.c
index 2a19477e955..ac37b27a4d8 100644
--- a/test/thread/file.c
+++ b/test/thread/file.c
@@ -28,7 +28,7 @@
 #include "thread.h"
 
 static void
-file_create(void)
+file_create(const char *name)
 {
 	WT_SESSION *session;
 	int ret;
@@ -47,7 +47,7 @@ file_create(void)
 	if (ftype == FIX)
 		(void)snprintf(p, (size_t)(end - p), ",value_format=3t");
 
-	if ((ret = session->create(session, FNAME, config)) != 0)
+	if ((ret = session->create(session, name, config)) != 0)
 		if (ret != EEXIST)
 			die("session.create", ret);
 
@@ -56,7 +56,7 @@ file_create(void)
 }
 
 void
-load(void)
+load(const char *name)
 {
 	WT_CURSOR *cursor;
 	WT_ITEM *key, _key, *value, _value;
@@ -65,13 +65,13 @@ load(void)
 	u_int keyno;
 	int ret;
 
-	file_create();
+	file_create(name);
 
 	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
 		die("conn.session", ret);
 
 	if ((ret =
-	    session->open_cursor(session, FNAME, NULL, "bulk", &cursor)) != 0)
+	    session->open_cursor(session, name, NULL, "bulk", &cursor)) != 0)
 		die("cursor.open", ret);
 
 	key = &_key;
@@ -99,3 +99,19 @@ load(void)
 	if ((ret = session->close(session, NULL)) != 0)
 		die("session.close", ret);
 }
+
+void
+verify(const char *name)
+{
+	WT_SESSION *session;
+	int ret;
+
+	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
+		die("conn.session", ret);
+
+	if ((ret = session->verify(session, name, NULL)) != 0)
+		die("session.create", ret);
+
+	if ((ret = session->close(session, NULL)) != 0)
+		die("session.close", ret);
+}
diff --git a/test/thread/rw.c b/test/thread/rw.c
index 97af78924e6..795673a0890 100644
--- a/test/thread/rw.c
+++ b/test/thread/rw.c
@@ -32,12 +32,14 @@ static void *reader(void *);
 static void *writer(void *);
 
 typedef struct {
+	char *name;				/* object name */
+
 	int remove;				/* cursor.remove */
 	int update;				/* cursor.update */
 	int reads;				/* cursor.search */
-} STATS;
+} INFO;
 
-static STATS *run_stats;
+static INFO *run_info;
 
 /*
  * r --
@@ -74,12 +76,23 @@ rw_start(u_int readers, u_int writers)
 	int ret;
 	void *thread_ret;
 
-	/* Create statistics and thread structures. */
-	if ((run_stats = calloc(
-	    (size_t)(readers + writers), sizeof(*run_stats))) == NULL ||
+	/* Create per-thread structures. */
+	if ((run_info = calloc(
+	    (size_t)(readers + writers), sizeof(*run_info))) == NULL ||
 	    (tids = calloc((size_t)(readers + writers), sizeof(*tids))) == NULL)
 		die("calloc", errno);
 
+	/* Create the files and load the initial records. */
+	for (i = 0; i < readers + writers; ++i) {
+		if ((run_info[i].name = malloc(64)) == NULL)
+			die("malloc", errno);
+		if (i == 0 || multiple_files) {
+			snprintf(run_info[i].name, 64, FNAME, i);
+			load(run_info[i].name);
+		} else
+			run_info[i].name = run_info[0].name;
+	}
+
 	(void)gettimeofday(&start, NULL);
 
 	/* Create threads. */
@@ -103,9 +116,24 @@ rw_start(u_int readers, u_int writers)
 	fprintf(stderr, "timer: %.2lf seconds (%d ops/second)\n",
 	    seconds, (int)(((readers + writers) * nops) / seconds));
 
+	/* Verify the files. */
+	for (i = 0; i < readers + writers; ++i) {
+		verify(run_info[i].name);
+		if (!multiple_files)
+			break;
+	}
+
+	/* Output run statistics. */
 	print_stats(readers + writers);
 
-	free(run_stats);
+	/* Free allocated memory. */
+	for (i = 0; i < readers + writers; ++i) {
+		free(run_info[i].name);
+		if (!multiple_files)
+			break;
+	}
+
+	free(run_info);
 	free(tids);
 
 	return (0);
@@ -147,7 +175,7 @@ reader_op(WT_SESSION *session, WT_CURSOR *cursor)
 static void *
 reader(void *arg)
 {
-	STATS *s;
+	INFO *s;
 	WT_CURSOR *cursor;
 	WT_SESSION *session;
 	pthread_t tid;
@@ -159,7 +187,7 @@ reader(void *arg)
 	printf(" read thread %2d starting: tid: %p\n", id, (void *)tid);
 	sched_yield();		/* Get all the threads created. */
 
-	s = &run_stats[id];
+	s = &run_info[id];
 
 	if (session_per_op) {
 		for (i = 0; i < nops; ++i, ++s->reads, sched_yield()) {
@@ -167,7 +195,7 @@ reader(void *arg)
 			    conn, NULL, NULL, &session)) != 0)
 				die("conn.open_session", ret);
 			if ((ret = session->open_cursor(
-			    session, FNAME, NULL, NULL, &cursor)) != 0)
+			    session, s->name, NULL, NULL, &cursor)) != 0)
 				die("session.open_cursor", ret);
 			reader_op(session, cursor);
 			if ((ret = session->close(session, NULL)) != 0)
@@ -178,7 +206,7 @@ reader(void *arg)
 		    conn, NULL, NULL, &session)) != 0)
 			die("conn.open_session", ret);
 		if ((ret = session->open_cursor(
-		    session, FNAME, NULL, NULL, &cursor)) != 0)
+		    session, s->name, NULL, NULL, &cursor)) != 0)
 			die("session.open_cursor", ret);
 		for (i = 0; i < nops; ++i, ++s->reads, sched_yield())
 			reader_op(session, cursor);
@@ -194,7 +222,7 @@ reader(void *arg)
  *	Write operation.
  */
 static inline void
-writer_op(WT_SESSION *session, WT_CURSOR *cursor, STATS *s)
+writer_op(WT_SESSION *session, WT_CURSOR *cursor, INFO *s)
 {
 	WT_ITEM *key, _key, *value, _value;
 	u_int keyno;
@@ -242,7 +270,7 @@ writer_op(WT_SESSION *session, WT_CURSOR *cursor, STATS *s)
 static void *
 writer(void *arg)
 {
-	STATS *s;
+	INFO *s;
 	WT_CURSOR *cursor;
 	WT_SESSION *session;
 	pthread_t tid;
@@ -254,7 +282,7 @@ writer(void *arg)
 	printf("write thread %2d starting: tid: %p\n", id, (void *)tid);
 	sched_yield();		/* Get all the threads created. */
 
-	s = &run_stats[id];
+	s = &run_info[id];
 
 	if (session_per_op) {
 		for (i = 0; i < nops; ++i, sched_yield()) {
@@ -262,7 +290,7 @@ writer(void *arg)
 			    conn, NULL, NULL, &session)) != 0)
 				die("conn.open_session", ret);
 			if ((ret = session->open_cursor(
-			    session, FNAME, NULL, NULL, &cursor)) != 0)
+			    session, s->name, NULL, NULL, &cursor)) != 0)
 				die("session.open_cursor", ret);
 			writer_op(session, cursor, s);
 			if ((ret = session->close(session, NULL)) != 0)
@@ -273,7 +301,7 @@ writer(void *arg)
 		    conn, NULL, NULL, &session)) != 0)
 			die("conn.open_session", ret);
 		if ((ret = session->open_cursor(
-		    session, FNAME, NULL, NULL, &cursor)) != 0)
+		    session, s->name, NULL, NULL, &cursor)) != 0)
 			die("session.open_cursor", ret);
 		for (i = 0; i < nops; ++i, sched_yield())
 			writer_op(session, cursor, s);
@@ -291,10 +319,10 @@ writer(void *arg)
 static void
 print_stats(u_int nthreads)
 {
-	STATS *s;
+	INFO *s;
 	u_int id;
 
-	s = run_stats;
+	s = run_info;
 	for (id = 0; id < nthreads; ++id, ++s)
 		printf("%3d: read %6d, remove %6d, update %6d\n",
 		    id, s->reads, s->remove, s->update);
diff --git a/test/thread/stats.c b/test/thread/stats.c
index 30d288c9d45..fda069d6bf5 100644
--- a/test/thread/stats.c
+++ b/test/thread/stats.c
@@ -34,12 +34,13 @@
 void
 stats(void)
 {
+	FILE *fp;
 	WT_CURSOR *cursor;
-	uint64_t v;
-	const char *pval, *desc;
 	WT_SESSION *session;
-	FILE *fp;
+	uint64_t v;
 	int ret;
+	char name[64];
+	const char *pval, *desc;
 
 	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
 		die("conn.session", ret);
@@ -61,9 +62,13 @@ stats(void)
 	if ((ret = cursor->close(cursor)) != 0)
 		die("cursor.close", ret);
 
+	if (multiple_files)
+		return;
+
 	/* File statistics. */
-	if ((ret = session->open_cursor(session,
-	    "statistics:" FNAME, NULL, NULL, &cursor)) != 0)
+	(void)snprintf(name, sizeof(name), "statistics:" FNAME, 0);
+	if ((ret =
+	    session->open_cursor(session, name, NULL, NULL, &cursor)) != 0)
 		die("session.open_cursor", ret);
 
 	while ((ret = cursor->next(cursor)) == 0 &&
diff --git a/test/thread/t.c b/test/thread/t.c
index 9d78e63fc73..530c2cef7f5 100644
--- a/test/thread/t.c
+++ b/test/thread/t.c
@@ -31,6 +31,7 @@ WT_CONNECTION *conn;				/* WiredTiger connection */
 __ftype ftype;					/* File type */
 u_int nkeys, nops;				/* Keys, Operations */
 int log_print;					/* Log print per operation */
+int multiple_files;				/* File per thread */
 int session_per_op;				/* New session per operation */
 
 static char *progname;				/* Program name */
@@ -59,6 +60,7 @@ main(int argc, char *argv[])
 	config_open = NULL;
 	ftype = ROW;
 	log_print = 0;
+	multiple_files = 0;
 	nkeys = 1000;
 	nops = 10000;
 	readers = 10;
@@ -66,11 +68,14 @@ main(int argc, char *argv[])
 	session_per_op = 0;
 	writers = 10;
 
-	while ((ch = getopt(argc, argv, "C:k:Ll:n:R:r:St:W:")) != EOF)
+	while ((ch = getopt(argc, argv, "C:Fk:Ll:n:R:r:St:W:")) != EOF)
 		switch (ch) {
 		case 'C':			/* wiredtiger_open config */
 			config_open = optarg;
 			break;
+		case 'F':			/* multiple files */
+			multiple_files = 1;
+			break;
 		case 'k':			/* rows */
 			nkeys = (u_int)atoi(optarg);
 			break;
@@ -135,9 +140,7 @@ main(int argc, char *argv[])
 
 		wt_connect(config_open);	/* WiredTiger connection */
 
-		load();				/* Load initial records */
-						/* Loop operations */
-		if (rw_start(readers, writers))
+		if (rw_start(readers, writers))	/* Loop operations */
 			return (EXIT_FAILURE);
 
 		stats();			/* Statistics */
@@ -186,9 +189,6 @@ wt_shutdown(void)
 	if ((ret = conn->open_session(conn, NULL, NULL, &session)) != 0)
 		die("conn.session", ret);
 
-	if ((ret = session->verify(session, FNAME, NULL)) != 0)
-		die("session.verify", ret);
-
 	if ((ret = session->checkpoint(session, NULL)) != 0)
 		die("session.checkpoint", ret);
 
@@ -203,7 +203,7 @@ wt_shutdown(void)
 static void
 shutdown(void)
 {
-	WT_UNUSED_RET(system("rm -f WiredTiger* __wt*"));
+	WT_UNUSED_RET(system("rm -f WiredTiger* wt.*"));
 }
 
 static int
@@ -265,12 +265,14 @@ usage(void)
 {
 	fprintf(stderr,
 	    "usage: %s "
-	    "[-S] [-C wiredtiger-config] [-k keys] [-l log]\n\t"
+	    "[-FLS] [-C wiredtiger-config] [-k keys] [-l log]\n\t"
 	    "[-n ops] [-R readers] [-r runs] [-t f|r|v] [-W writers]\n",
 	    progname);
 	fprintf(stderr, "%s",
 	    "\t-C specify wiredtiger_open configuration arguments\n"
+	    "\t-F create a file per thread\n"
 	    "\t-k set number of keys to load\n"
+	    "\t-L log print per operation\n"
 	    "\t-l specify a log file\n"
 	    "\t-n set number of operations each thread does\n"
 	    "\t-R set number of reading threads\n"
diff --git a/test/thread/thread.h b/test/thread/thread.h
index c677f8eeae8..c9d9313ff6d 100644
--- a/test/thread/thread.h
+++ b/test/thread/thread.h
@@ -40,7 +40,7 @@
 #include <wiredtiger.h>
 #include <gcc.h>				/* WiredTiger internal */
 
-#define	FNAME		"file:__wt"		/* File name */
+#define	FNAME		"file:wt.%03d"		/* File name */
 #define	FNAME_STAT	"__stats"		/* File name for statistics */
 
 extern WT_CONNECTION *conn;			/* WiredTiger connection */
@@ -49,11 +49,13 @@ typedef enum { FIX, ROW, VAR } __ftype;		/* File type */
 extern __ftype ftype;
 
 extern int   log_print;				/* Log print per operation */
+extern int   multiple_files;			/* File per thread */
 extern u_int nkeys;				/* Keys to load */
 extern u_int nops;				/* Operations per thread */
 extern int   session_per_op;			/* New session per operation */
 
 void die(const char *, int) WT_GCC_ATTRIBUTE((noreturn));
-void load(void);
+void load(const char *);
 int  rw_start(u_int, u_int);
 void stats(void);
+void verify(const char *);