WT-2583 Detect incremental file left in source directory (#2696)

* Remove unused metadata backup file for incremental backups.
* Write backup information to a temp file and then rename.

4 files changed, 47 insertions, 31 deletions

diff --git a/dist/s_string.ok b/dist/s_string.ok
index 52f4f0dc74c..81d09a55225 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -728,6 +728,7 @@ io
 ip
 islocked
 ispo
+isrc
 iter
 iteratively
 jnr
diff --git a/src/cursor/cur_backup.c b/src/cursor/cur_backup.c
index b3c3e899eed..4ee23008687 100644
--- a/src/cursor/cur_backup.c
+++ b/src/cursor/cur_backup.c
@@ -10,7 +10,6 @@
 
 static int __backup_all(WT_SESSION_IMPL *);
 static int __backup_cleanup_handles(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *);
-static int __backup_file_create(WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, bool);
 static int __backup_list_append(
     WT_SESSION_IMPL *, WT_CURSOR_BACKUP *, const char *);
 static int __backup_list_uri_append(WT_SESSION_IMPL *, const char *, bool *);
@@ -192,9 +191,13 @@ __backup_start(
 {
 	WT_CONNECTION_IMPL *conn;
 	WT_DECL_RET;
+	WT_FSTREAM *srcfs;
+	const char *dest;
 	bool exist, log_only, target_list;
 
 	conn = S2C(session);
+	srcfs = NULL;
+	dest = NULL;
 
 	cb->next = 0;
 	cb->list = NULL;
@@ -223,11 +226,16 @@ __backup_start(
 	conn->hot_backup = true;
 	WT_ERR(__wt_writeunlock(session, conn->hot_backup_lock));
 
-	/* Create the hot backup file. */
-	WT_ERR(__backup_file_create(session, cb, false));
-
-	/* Add log files if logging is enabled. */
-
+	/*
+	 * Create a temporary backup file.  This must be opened before
+	 * generating the list of targets in backup_uri.  This file will
+	 * later be renamed to the correct name depending on whether or not
+	 * we're doing an incremental backup.  We need a temp file so that if
+	 * we fail or crash while filling it, the existence of a partial file
+	 * doesn't confuse restarting in the source database.
+	 */
+	WT_ERR(__wt_fopen(session, WT_BACKUP_TMP,
+	    WT_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs));
 	/*
 	 * If a list of targets was specified, work our way through them.
 	 * Else, generate a list of all database objects.
@@ -247,14 +255,17 @@ __backup_start(
 	/* Add the hot backup and standard WiredTiger files to the list. */
 	if (log_only) {
 		/*
-		 * Close any hot backup file.
-		 * We're about to open the incremental backup file.
+		 * We also open an incremental backup source file so that we
+		 * can detect a crash with an incremental backup existing in
+		 * the source directory versus an improper destination.
 		 */
-		WT_TRET(__wt_fclose(session, &cb->bfs));
-		WT_ERR(__backup_file_create(session, cb, log_only));
+		dest = WT_INCREMENTAL_BACKUP;
+		WT_ERR(__wt_fopen(session, WT_INCREMENTAL_SRC,
+		    WT_OPEN_CREATE, WT_STREAM_WRITE, &srcfs));
 		WT_ERR(__backup_list_append(
 		    session, cb, WT_INCREMENTAL_BACKUP));
 	} else {
+		dest = WT_METADATA_BACKUP;
 		WT_ERR(__backup_list_append(session, cb, WT_METADATA_BACKUP));
 		WT_ERR(__wt_fs_exist(session, WT_BASECONFIG, &exist));
 		if (exist)
@@ -269,9 +280,14 @@ __backup_start(
 
 err:	/* Close the hot backup file. */
 	WT_TRET(__wt_fclose(session, &cb->bfs));
+	if (srcfs != NULL)
+		WT_TRET(__wt_fclose(session, &srcfs));
 	if (ret != 0) {
 		WT_TRET(__backup_cleanup_handles(session, cb));
 		WT_TRET(__backup_stop(session));
+	} else {
+		WT_ASSERT(session, dest != NULL);
+		WT_TRET(__wt_fs_rename(session, WT_BACKUP_TMP, dest));
 	}
 
 	return (ret);
@@ -398,7 +414,8 @@ __backup_uri(WT_SESSION_IMPL *session,
 				    "incremental backup not possible when "
 				    "automatic log archival configured");
 			*log_only = !target_list;
-			WT_ERR(__backup_list_uri_append(session, uri, NULL));
+			WT_ERR(__backup_log_append(
+			    session, session->bkp_cursor, false));
 		} else {
 			*log_only = false;
 			WT_ERR(__wt_schema_worker(session,
@@ -412,19 +429,6 @@ err:	__wt_scr_free(session, &tmp);
 }
 
 /*
- * __backup_file_create --
- *	Create the meta-data backup file.
- */
-static int
-__backup_file_create(
-    WT_SESSION_IMPL *session, WT_CURSOR_BACKUP *cb, bool incremental)
-{
-	return (__wt_fopen(session,
-	    incremental ? WT_INCREMENTAL_BACKUP : WT_METADATA_BACKUP,
-	    WT_OPEN_CREATE, WT_STREAM_WRITE, &cb->bfs));
-}
-
-/*
  * __wt_backup_file_remove --
  *	Remove the incremental and meta-data backup files.
  */
@@ -433,7 +437,15 @@ __wt_backup_file_remove(WT_SESSION_IMPL *session)
 {
 	WT_DECL_RET;
 
+	/*
+	 * Note that order matters for removing the incremental files.  We must
+	 * remove the backup file before removing the source file so that we
+	 * always know we were a source directory while there's any chance of
+	 * an incremental backup file existing.
+	 */
+	WT_TRET(__wt_remove_if_exists(session, WT_BACKUP_TMP));
 	WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_BACKUP));
+	WT_TRET(__wt_remove_if_exists(session, WT_INCREMENTAL_SRC));
 	WT_TRET(__wt_remove_if_exists(session, WT_METADATA_BACKUP));
 	return (ret);
 }
@@ -461,11 +473,6 @@ __backup_list_uri_append(
 	 * if there's an entry backed by anything other than a file or lsm
 	 * entry, we're confused.
 	 */
-	if (WT_PREFIX_MATCH(name, "log:")) {
-		WT_RET(__backup_log_append(session, cb, false));
-		return (0);
-	}
-
 	if (!WT_PREFIX_MATCH(name, "file:") &&
 	    !WT_PREFIX_MATCH(name, "colgroup:") &&
 	    !WT_PREFIX_MATCH(name, "index:") &&
diff --git a/src/include/meta.h b/src/include/meta.h
index ac0f5fedac4..ba4149979ef 100644
--- a/src/include/meta.h
+++ b/src/include/meta.h
@@ -14,8 +14,10 @@
 
 #define	WT_USERCONFIG		"WiredTiger.config"	/* User configuration */
 
+#define	WT_BACKUP_TMP		"WiredTiger.backup.tmp"	/* Backup tmp file */
 #define	WT_METADATA_BACKUP	"WiredTiger.backup"	/* Hot backup file */
 #define	WT_INCREMENTAL_BACKUP	"WiredTiger.ibackup"	/* Incremental backup */
+#define	WT_INCREMENTAL_SRC	"WiredTiger.isrc"	/* Incremental source */
 
 #define	WT_METADATA_TURTLE	"WiredTiger.turtle"	/* Metadata metadata */
 #define	WT_METADATA_TURTLE_SET	"WiredTiger.turtle.set"	/* Turtle temp file */
diff --git a/src/meta/meta_turtle.c b/src/meta/meta_turtle.c
index ee9ee522748..635daf63d7f 100644
--- a/src/meta/meta_turtle.c
+++ b/src/meta/meta_turtle.c
@@ -156,7 +156,7 @@ int
 __wt_turtle_init(WT_SESSION_IMPL *session)
 {
 	WT_DECL_RET;
-	bool exist_backup, exist_incr, exist_turtle, load;
+	bool exist_backup, exist_incr, exist_isrc, exist_turtle, load;
 	char *metaconf;
 
 	metaconf = NULL;
@@ -183,10 +183,16 @@ __wt_turtle_init(WT_SESSION_IMPL *session)
 	 * done.
 	 */
 	WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_BACKUP, &exist_incr));
+	WT_RET(__wt_fs_exist(session, WT_INCREMENTAL_SRC, &exist_isrc));
 	WT_RET(__wt_fs_exist(session, WT_METADATA_BACKUP, &exist_backup));
 	WT_RET(__wt_fs_exist(session, WT_METADATA_TURTLE, &exist_turtle));
 	if (exist_turtle) {
-		if (exist_incr)
+		/*
+		 * We need to detect the difference between a source database
+		 * that may have crashed with an incremental backup file
+		 * and a destination database that incorrectly ran recovery.
+		 */
+		if (exist_incr && !exist_isrc)
 			WT_RET_MSG(session, EINVAL,
 			    "Incremental backup after running recovery "
 			    "is not allowed");