SERVER-36269: Add backup_restore test for $backupCursors

4 files changed, 79 insertions, 7 deletions

diff --git a/jstests/noPassthrough/backup_restore_backup_cursor.js b/jstests/noPassthrough/backup_restore_backup_cursor.js
new file mode 100644
index 00000000000..445c3592fe9
--- /dev/null
+++ b/jstests/noPassthrough/backup_restore_backup_cursor.js
@@ -0,0 +1,25 @@
+/**
+ * Test the backup/restore process:
+ * - 3 node replica set
+ * - Mongo CRUD client
+ * - Mongo FSM client
+ * - open a $backupCursor on the Secondary
+ * - cp files returned by the $backupCursor
+ * - close the $backupCursor
+ * - Start mongod as hidden secondary
+ * - Wait until new hidden node becomes secondary
+ *
+ * Some methods for backup used in this test checkpoint the files in the dbpath. This technique will
+ * not work for ephemeral storage engines, as they do not store any data in the dbpath.
+ * @tags: [requires_persistence, requires_replication]
+ */
+
+load("jstests/noPassthrough/libs/backup_restore.js");
+
+(function() {
+    "use strict";
+
+    // Run the fsyncLock test. Will return before testing for any engine that doesn't
+    // support fsyncLock
+    new BackupRestoreTest({backup: 'backupCursor'}).run();
+}());
diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js
index 051b63298e3..65af92af13b 100644
--- a/jstests/noPassthrough/libs/backup_restore.js
+++ b/jstests/noPassthrough/libs/backup_restore.js
@@ -3,9 +3,9 @@
  * - 3 node replica set
  * - Mongo CRUD client
  * - Mongo FSM client
- * - fsyncLock (or stop) Secondary
+ * - fsyncLock, stop or open a backupCursor on a Secondary
  * - cp (or rsync) DB files
- * - fsyncUnlock (or start) Secondary
+ * - fsyncUnlock, start or close a backupCursor on the Secondary
  * - Start mongod as hidden secondary
  * - Wait until new hidden node becomes secondary
  *
@@ -233,7 +233,7 @@ var BackupRestoreTest = function(options) {
         var testName = jsTest.name();
 
         // Backup type (must be specified)
-        var allowedBackupKeys = ['fsyncLock', 'stopStart', 'rolling'];
+        var allowedBackupKeys = ['fsyncLock', 'stopStart', 'rolling', 'backupCursor'];
         assert(options.backup, "Backup option not supplied");
         assert.contains(options.backup,
                         allowedBackupKeys,
@@ -359,6 +359,53 @@ var BackupRestoreTest = function(options) {
             print("Copied files:", tojson(copiedFiles));
             assert.gt(copiedFiles.length, 0, testName + ' no files copied');
             rst.start(secondary.nodeId, {}, true);
+        } else if (options.backup == 'backupCursor') {
+            resetDbpath(hiddenDbpath);
+            mkdir(hiddenDbpath + '/journal');
+            jsTestLog("Copying start: " + tojson({
+                          source: dbpathSecondary,
+                          destination: hiddenDbpath,
+                          sourceFiles: ls(dbpathSecondary),
+                          journalFiles: ls(dbpathSecondary + '/journal')
+                      }));
+
+            let backupCursor = secondary.getDB('admin').aggregate([{$backupCursor: {}}]);
+            assert(backupCursor.hasNext());
+            let doc = backupCursor.next();
+            assert(doc.hasOwnProperty('metadata'));
+            jsTestLog("Metadata doc: " + tojson({doc: doc}));
+
+            // Grab the dbpath, ensure it ends with a slash. Note that while the `dbpath` inputs
+            // may be in windows or unix format, the FS helpers for listing a directory and
+            // copying files (etc..)  treat them equally.
+            let dbpath = doc['metadata']['dbpath'];
+            if (dbpath.lastIndexOf('/') + 1 != dbpath.length &&
+                dbpath.lastIndexOf('\\') + 1 != dbpath.length) {
+                dbpath += '/';
+            }
+
+            while (backupCursor.hasNext()) {
+                doc = backupCursor.next();
+
+                let fileToCopy = doc['filename'];
+                // Ensure that the full path starts with the returned dbpath.
+                assert.eq(0, fileToCopy.search(dbpath));
+
+                // Grab the file path relative to the dbpath. Maintain that relation when copying
+                // to the `hiddenDbpath`.
+                let relativePath = fileToCopy.substr(dbpath.length);
+                jsTestLog("File copy: " +
+                          tojson({fileToCopy: fileToCopy, relativePath: relativePath}));
+                copyFile(fileToCopy, hiddenDbpath + '/' + relativePath);
+            }
+
+            copiedFiles = ls(hiddenDbpath);
+            jsTestLog("Copying End: " + tojson({
+                          destinationFiles: copiedFiles,
+                          destinationJournal: ls(hiddenDbpath + '/journal')
+                      }));
+            backupCursor.close();
+            assert.gt(copiedFiles.length, 0, testName + ' no files copied');
         }
 
         // Wait up to 5 minutes until restarted node is in state secondary.
diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index 0867728507d..50c884dd6e6 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -308,6 +308,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/repl/oplog_entry',
         '$BUILD_DIR/mongo/db/repl/optime',
         'encryption_hooks',
+        'storage_options',
     ],
 )
 
diff --git a/src/mongo/db/storage/backup_cursor_service.cpp b/src/mongo/db/storage/backup_cursor_service.cpp
index 09bbd7fb6a9..e83dc22cfc9 100644
--- a/src/mongo/db/storage/backup_cursor_service.cpp
+++ b/src/mongo/db/storage/backup_cursor_service.cpp
@@ -39,6 +39,7 @@
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/encryption_hooks.h"
 #include "mongo/db/storage/storage_engine.h"
+#include "mongo/db/storage/storage_options.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
 #include "mongo/util/scopeguard.h"
@@ -167,6 +168,7 @@ BackupCursorState BackupCursorService::openBackupCursor(OperationContext* opCtx)
     }
 
     BSONObjBuilder builder;
+    builder << "dbpath" << storageGlobalParams.dbpath;
     if (!oplogStart.isNull()) {
         builder << "oplogStart" << oplogStart.toBSON();
         builder << "oplogEnd" << oplogEnd.toBSON();
@@ -177,10 +179,7 @@ BackupCursorState BackupCursorService::openBackupCursor(OperationContext* opCtx)
         builder << "checkpointTimestamp" << checkpointTimestamp.get();
     }
 
-    boost::optional<Document> preamble = boost::none;
-    if (!oplogStart.isNull()) {
-        preamble = Document{{"metadata", builder.obj()}};
-    }
+    Document preamble{{"metadata", builder.obj()}};
 
     closeCursorGuard.Dismiss();
     return {_openCursor.get(), preamble, filesToBackup};