Merge branch 'develop' into wt-2360-join-disjunction

7 files changed, 103 insertions, 65 deletions

diff --git a/test/format/backup.c b/test/format/backup.c
index 0ce40d0e631..977b44c2e11 100644
--- a/test/format/backup.c
+++ b/test/format/backup.c
@@ -38,7 +38,7 @@ check_copy(void)
 	WT_CONNECTION *conn;
 	WT_SESSION *session;
 
-	wts_open(g.home_backup, 0, &conn);
+	wts_open(g.home_backup, false, &conn);
 
 	testutil_checkfmt(
 	    conn->open_session(conn, NULL, NULL, &session),
@@ -53,27 +53,30 @@ check_copy(void)
 
 /*
  * copy_file --
- *	Copy a single file into the backup directory.
+ *	Copy a single file into the backup directories.
  */
 static void
-copy_file(const char *name)
+copy_file(WT_SESSION *session, const char *name)
 {
 	size_t len;
-	char *cmd;
-
-	len = strlen(g.home) + strlen(g.home_backup) + strlen(name) * 2 + 20;
-	cmd = dmalloc(len);
-	(void)snprintf(cmd, len,
-	    "cp %s/%s %s/%s", g.home, name, g.home_backup, name);
-	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
-	free(cmd);
-
-	len = strlen(g.home) + strlen(g.home_backup2) + strlen(name) * 2 + 20;
-	cmd = dmalloc(len);
-	(void)snprintf(cmd, len,
-	    "cp %s/%s %s/%s", g.home, name, g.home_backup2, name);
-	testutil_checkfmt(system(cmd), "backup copy: %s", cmd);
-	free(cmd);
+	char *first, *second;
+
+	len = strlen("BACKUP") + strlen(name) + 10;
+	first = dmalloc(len);
+	(void)snprintf(first, len, "BACKUP/%s", name);
+	testutil_check(__wt_copy_and_sync(session, name, first));
+
+	/*
+	 * Save another copy of the original file to make debugging recovery
+	 * errors easier.
+	 */
+	len = strlen("BACKUP_COPY") + strlen(name) + 10;
+	second = dmalloc(len);
+	(void)snprintf(second, len, "BACKUP_COPY/%s", name);
+	testutil_check(__wt_copy_and_sync(session, first, second));
+
+	free(first);
+	free(second);
 }
 
 /*
@@ -87,8 +90,9 @@ backup(void *arg)
 	WT_CURSOR *backup_cursor;
 	WT_DECL_RET;
 	WT_SESSION *session;
-	u_int period;
-	const char *key;
+	u_int incremental, period;
+	bool full;
+	const char *config, *key;
 
 	(void)(arg);
 
@@ -102,48 +106,81 @@ backup(void *arg)
 	testutil_check(conn->open_session(conn, NULL, NULL, &session));
 
 	/*
-	 * Perform a backup at somewhere under 10 seconds (so we get at
-	 * least one done), and then at 45 second intervals.
+	 * Perform a full backup at somewhere under 10 seconds (that way there's
+	 * at least one), then at larger intervals, optionally do incremental
+	 * backups between full backups.
 	 */
-	for (period = mmrand(NULL, 1, 10);; period = 45) {
+	incremental = 0;
+	for (period = mmrand(NULL, 1, 10);; period = mmrand(NULL, 20, 45)) {
 		/* Sleep for short periods so we don't make the run wait. */
 		while (period > 0 && !g.workers_finished) {
 			--period;
 			sleep(1);
 		}
-		if (g.workers_finished)
-			break;
 
 		/* Lock out named checkpoints */
 		testutil_check(pthread_rwlock_wrlock(&g.backup_lock));
+		if (g.workers_finished) {
+			testutil_check(pthread_rwlock_unlock(&g.backup_lock));
+			break;
+		}
 
-		/* Re-create the backup directory. */
-		testutil_checkfmt(
-		    system(g.home_backup_init),
-		    "%s", "backup directory creation failed");
+		if (incremental) {
+			config = "target=(\"log:\")";
+			full = false;
+		} else {
+			/* Re-create the backup directory. */
+			testutil_checkfmt(
+			    system(g.home_backup_init),
+			    "%s", "backup directory creation failed");
+
+			config = NULL;
+			full = true;
+		}
 
 		/*
-		 * open_cursor can return EBUSY if a metadata operation is
-		 * currently happening - retry in that case.
+		 * open_cursor can return EBUSY if concurrent with a metadata
+		 * operation, retry in that case.
 		 */
-		while ((ret = session->open_cursor(session,
-		    "backup:", NULL, NULL, &backup_cursor)) == EBUSY)
-			sleep(1);
+		while ((ret = session->open_cursor(
+		    session, "backup:", NULL, config, &backup_cursor)) == EBUSY)
+			__wt_yield();
 		if (ret != 0)
 			testutil_die(ret, "session.open_cursor: backup");
 
 		while ((ret = backup_cursor->next(backup_cursor)) == 0) {
 			testutil_check(
 			    backup_cursor->get_key(backup_cursor, &key));
-			copy_file(key);
+			copy_file(session, key);
 		}
+		if (ret != WT_NOTFOUND)
+			testutil_die(ret, "backup-cursor");
 
 		testutil_check(backup_cursor->close(backup_cursor));
 		testutil_check(pthread_rwlock_unlock(&g.backup_lock));
 
-		check_copy();
+		/* After an incremental backup, truncate the log files. */
+		if (incremental)
+			testutil_check(session->truncate(
+			    session, "log:", backup_cursor, NULL, NULL));
+
+		/*
+		 * If automatic log archival isn't configured, optionally do
+		 * incremental backups after each full backup. If we're not
+		 * doing any more incrementals, verify the backup (we can't
+		 * verify intermediate states, once we perform recovery on the
+		 * backup database, we can't do any more incremental backups).
+		 */
+		if (full)
+			incremental =
+			    g.c_logging_archive ? 1 : mmrand(NULL, 1, 5);
+		if (--incremental == 0)
+			check_copy();
 	}
 
+	if (incremental != 0)
+		check_copy();
+
 	testutil_check(session->close(session, NULL));
 
 	return (NULL);
diff --git a/test/format/format.h b/test/format/format.h
index ce45ca5a76e..0d5201309b2 100644
--- a/test/format/format.h
+++ b/test/format/format.h
@@ -109,7 +109,6 @@ typedef struct {
 
 	char *home;				/* Home directory */
 	char *home_backup;			/* Hot-backup directory */
-	char *home_backup2;			/* Saved Hot-backup directory */
 	char *home_backup_init;			/* Initialize backup command */
 	char *home_bdb;				/* BDB directory */
 	char *home_config;			/* Run CONFIG file path */
@@ -325,10 +324,10 @@ void	 track(const char *, uint64_t, TINFO *);
 void	 val_gen(WT_RAND_STATE *, WT_ITEM *, uint64_t);
 void	 val_gen_setup(WT_RAND_STATE *, WT_ITEM *);
 void	 wts_close(void);
-void	 wts_create(void);
 void	 wts_dump(const char *, int);
+void	 wts_init(void);
 void	 wts_load(void);
-void	 wts_open(const char *, int, WT_CONNECTION **);
+void	 wts_open(const char *, bool, WT_CONNECTION **);
 void	 wts_ops(int);
 void	 wts_read_scan(void);
 void	 wts_rebalance(void);
diff --git a/test/format/ops.c b/test/format/ops.c
index 6dc45fde768..b075c8c7d22 100644
--- a/test/format/ops.c
+++ b/test/format/ops.c
@@ -184,6 +184,7 @@ wts_ops(int lastrun)
 		(void)pthread_join(compact_tid, NULL);
 	if (!SINGLETHREADED && g.c_long_running_txn)
 		(void)pthread_join(lrt_tid, NULL);
+	g.workers_finished = 0;
 
 	if (g.logging != 0) {
 		(void)g.wt_api->msg_printf(g.wt_api, session,
diff --git a/test/format/salvage.c b/test/format/salvage.c
index b2a38e37c8b..8274c556364 100644
--- a/test/format/salvage.c
+++ b/test/format/salvage.c
@@ -158,7 +158,7 @@ wts_salvage(void)
 		testutil_die(ret, "salvage copy step failed");
 
 	/* Salvage, then verify. */
-	wts_open(g.home, 1, &g.wts_conn);
+	wts_open(g.home, true, &g.wts_conn);
 	salvage();
 	wts_verify("post-salvage verify");
 	wts_close();
@@ -174,7 +174,7 @@ wts_salvage(void)
 
 	/* Corrupt the file randomly, salvage, then verify. */
 	if (corrupt()) {
-		wts_open(g.home, 1, &g.wts_conn);
+		wts_open(g.home, true, &g.wts_conn);
 		salvage();
 		wts_verify("post-corrupt-salvage verify");
 		wts_close();
diff --git a/test/format/t.c b/test/format/t.c
index f991e49a9b0..b4f0fffda54 100644
--- a/test/format/t.c
+++ b/test/format/t.c
@@ -198,8 +198,8 @@ main(int argc, char *argv[])
 		if (SINGLETHREADED)
 			bdb_open();		/* Initial file config */
 #endif
-		wts_open(g.home, 1, &g.wts_conn);
-		wts_create();
+		wts_open(g.home, true, &g.wts_conn);
+		wts_init();
 
 		wts_load();			/* Load initial records */
 		wts_verify("post-bulk verify");	/* Verify */
diff --git a/test/format/util.c b/test/format/util.c
index 7c5cc05e68f..22c2ab329bf 100644
--- a/test/format/util.c
+++ b/test/format/util.c
@@ -333,15 +333,6 @@ path_setup(const char *home)
 	g.home_stats = dmalloc(len);
 	snprintf(g.home_stats, len, "%s/%s", g.home, "stats");
 
-	/* Backup directory. */
-	len = strlen(g.home) + strlen("BACKUP") + 2;
-	g.home_backup = dmalloc(len);
-	snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP");
-
-	len = strlen(g.home) + strlen("BACKUP2") + 2;
-	g.home_backup2 = dmalloc(len);
-	snprintf(g.home_backup2, len, "%s/%s", g.home, "BACKUP2");
-
 	/* BDB directory. */
 	len = strlen(g.home) + strlen("bdb") + 2;
 	g.home_bdb = dmalloc(len);
@@ -369,18 +360,27 @@ path_setup(const char *home)
 	g.home_init = dmalloc(len);
 	snprintf(g.home_init, len, CMD, g.home, g.home, g.home);
 
-	/* Backup directory initialize command, remove and re-create it. */
+	/* Primary backup directory. */
+	len = strlen(g.home) + strlen("BACKUP") + 2;
+	g.home_backup = dmalloc(len);
+	snprintf(g.home_backup, len, "%s/%s", g.home, "BACKUP");
+
+	/*
+	 * Backup directory initialize command, remove and re-create the primary
+	 * backup directory, plus a copy we maintain for recovery testing.
+	 */
 #undef	CMD
 #ifdef _WIN32
-#define	CMD	"del /s /q >:nul && mkdir %s %s"
+#define	CMD	"del %s/%s %s/%s /s /q >:nul && mkdir %s/%s %s/%s"
 #else
-#define	CMD	"rm -rf %s %s && mkdir %s %s"
+#define	CMD	"rm -rf %s/%s %s/%s && mkdir %s/%s %s/%s"
 #endif
-	len = strlen(g.home_backup) * 2 +
-	    strlen(g.home_backup2) * 2 + strlen(CMD) + 1;
+	len = strlen(g.home) * 4 +
+	    strlen("BACKUP") * 2 + strlen("BACKUP_COPY") * 2 + strlen(CMD) + 1;
 	g.home_backup_init = dmalloc(len);
-	snprintf(g.home_backup_init, len, CMD, g.home_backup, g.home_backup2,
-	    g.home_backup, g.home_backup2);
+	snprintf(g.home_backup_init, len, CMD,
+	    g.home, "BACKUP", g.home, "BACKUP_COPY",
+	    g.home, "BACKUP", g.home, "BACKUP_COPY");
 
 	/*
 	 * Salvage command, save the interesting files so we can replay the
diff --git a/test/format/wts.c b/test/format/wts.c
index 7b114edec1e..2ee01aa75b5 100644
--- a/test/format/wts.c
+++ b/test/format/wts.c
@@ -126,7 +126,7 @@ static WT_EVENT_HANDLER event_handler = {
  *	Open a connection to a WiredTiger database.
  */
 void
-wts_open(const char *home, int set_api, WT_CONNECTION **connp)
+wts_open(const char *home, bool set_api, WT_CONNECTION **connp)
 {
 	WT_CONNECTION *conn;
 	WT_DECL_RET;
@@ -138,10 +138,11 @@ wts_open(const char *home, int set_api, WT_CONNECTION **connp)
 	end = config + sizeof(g.wiredtiger_open_config);
 
 	p += snprintf(p, REMAIN(p, end),
-	    "create,checkpoint_sync=false,cache_size=%" PRIu32 "MB",
-	    g.c_cache);
-
-	p += snprintf(p, REMAIN(p, end), ",error_prefix=\"%s\"", g.progname);
+	    "create=true,"
+	    "cache_size=%" PRIu32 "MB,"
+	    "checkpoint_sync=false,"
+	    "error_prefix=\"%s\"",
+	    g.c_cache, g.progname);
 
 	/* In-memory configuration. */
 	if (g.c_in_memory != 0)
@@ -287,7 +288,7 @@ wts_reopen(void)
  *	Create the underlying store.
  */
 void
-wts_create(void)
+wts_init(void)
 {
 	WT_CONNECTION *conn;
 	WT_SESSION *session;