SERVER-44407 Connect the incremental backup cursor pipes with WT output

(cherry picked from commit ed35208e7faeef1856eccf3e724c2838ad4a6dca)

9 files changed, 65 insertions, 11 deletions

diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 6f12bd8c816..58e345ac8a3 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -75,7 +75,10 @@ public:
     }
     void endBackup(OperationContext* opCtx) final {}
     StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) final {
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) final {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine does not support a concurrent mode.");
     }
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 17c70079754..db0939d854f 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -271,7 +271,10 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup(
-    OperationContext* opCtx) {
+    OperationContext* opCtx,
+    bool incrementalBackup,
+    boost::optional<std::string> thisBackupName,
+    boost::optional<std::string> srcBackupName) {
     std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}};
     return blocksToCopy;
 }
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index 0dae32f61b1..c335a5b53ea 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -141,7 +141,10 @@ public:
     virtual void endBackup(OperationContext* opCtx) {}
 
     virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) override;
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override {}
 
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index a87983f9ef7..16513516b7e 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -224,7 +224,10 @@ public:
     }
 
     virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) {
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
     }
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index fcd5b045ada..e1429a91f09 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -273,8 +273,23 @@ public:
         return;
     }
 
+    /**
+     * When performing an incremental backup, we first need a basis for future incremental backups.
+     * The basis will be a full backup called 'thisBackupName'. For future incremental backups, the
+     * storage engine will take a backup called 'thisBackupName' which will contain the changes made
+     * to data files since the backup named 'srcBackupName'.
+     *
+     * The first full backup meant for incremental and future incremental backups must pass
+     * 'incrementalBackup' as true.
+     * 'thisBackupName' must exist only if 'incrementalBackup' is true.
+     * 'srcBackupName' must not exist when 'incrementalBackup' is false but may or may not exist
+     * when 'incrementalBackup' is true.
+     */
     virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) = 0;
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) = 0;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) {
         return;
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 1ae6c8783bd..1035ced7874 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -648,8 +648,11 @@ void StorageEngineImpl::endBackup(OperationContext* opCtx) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup(
-    OperationContext* opCtx) {
-    return _engine->beginNonBlockingBackup(opCtx);
+    OperationContext* opCtx,
+    bool incrementalBackup,
+    boost::optional<std::string> thisBackupName,
+    boost::optional<std::string> srcBackupName) {
+    return _engine->beginNonBlockingBackup(opCtx, incrementalBackup, thisBackupName, srcBackupName);
 }
 
 void StorageEngineImpl::endNonBlockingBackup(OperationContext* opCtx) {
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 66f6951e21b..4b99211c55d 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -98,7 +98,10 @@ public:
     virtual void endBackup(OperationContext* opCtx);
 
     virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) override;
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override;
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 013db40b696..3d90af240b4 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -40,6 +40,7 @@
 #endif
 
 #include <fmt/format.h>
+#include <iomanip>
 #include <memory>
 
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
@@ -1046,9 +1047,25 @@ void WiredTigerKVEngine::endBackup(OperationContext* opCtx) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup(
-    OperationContext* opCtx) {
+    OperationContext* opCtx,
+    bool incrementalBackup,
+    boost::optional<std::string> thisBackupName,
+    boost::optional<std::string> srcBackupName) {
     uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral());
 
+    std::stringstream ss;
+    if (incrementalBackup) {
+        invariant(thisBackupName);
+        ss << "incremental=(enabled=true,force_stop=false,";
+        ss << "this_id=" << std::quoted(str::escape(*thisBackupName)) << ",";
+
+        if (srcBackupName) {
+            ss << "src_id=" << std::quoted(str::escape(*srcBackupName)) << ",";
+        }
+
+        ss << ")";
+    }
+
     // Oplog truncation thread won't remove oplog since the checkpoint pinned by the backup cursor.
     stdx::lock_guard<Latch> lock(_oplogPinnedByBackupMutex);
     _checkpointThread->assignOplogNeededForCrashRecoveryTo(&_oplogPinnedByBackup);
@@ -1063,7 +1080,8 @@ StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNon
     auto sessionRaii = stdx::make_unique<WiredTigerSession>(_conn);
     WT_CURSOR* cursor = NULL;
     WT_SESSION* session = sessionRaii->getSession();
-    int wtRet = session->open_cursor(session, "backup:", NULL, NULL, &cursor);
+    const std::string config = ss.str();
+    int wtRet = session->open_cursor(session, "backup:", nullptr, config.c_str(), &cursor);
     if (wtRet != 0) {
         return wtRCToStatus(wtRet);
     }
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 00dea19eb08..9c40d5874b5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -175,7 +175,10 @@ public:
     void endBackup(OperationContext* opCtx) override;
 
     StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx) override;
+        OperationContext* opCtx,
+        bool incrementalBackup,
+        boost::optional<std::string> thisBackupName,
+        boost::optional<std::string> srcBackupName) override;
 
     void endNonBlockingBackup(OperationContext* opCtx) override;