SERVER-43794: Incremental backup cursor API.

(cherry picked from commit cff25ee25c1ad064777c9e8ad58d36857e73035e)

10 files changed, 72 insertions, 29 deletions

diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 827afdaed4b..6f12bd8c816 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -69,6 +69,21 @@ public:
     int flushAllFiles(OperationContext* opCtx, bool sync) final {
         return 0;
     }
+    Status beginBackup(OperationContext* opCtx) final {
+        return Status(ErrorCodes::CommandNotSupported,
+                      "The current storage engine doesn't support backup mode");
+    }
+    void endBackup(OperationContext* opCtx) final {}
+    StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+        OperationContext* opCtx) final {
+        return Status(ErrorCodes::CommandNotSupported,
+                      "The current storage engine does not support a concurrent mode.");
+    }
+    void endNonBlockingBackup(OperationContext* opCtx) final {}
+    StatusWith<std::vector<std::string>> extendBackupCursor(OperationContext* opCtx) final {
+        return Status(ErrorCodes::CommandNotSupported,
+                      "The current storage engine does not support a concurrent mode.");
+    }
     Status repairRecordStore(OperationContext* opCtx, const NamespaceString& ns) final {
         return Status::OK();
     }
diff --git a/src/mongo/db/storage/backup_cursor_state.h b/src/mongo/db/storage/backup_cursor_state.h
index 4f880fdafd7..834c5156719 100644
--- a/src/mongo/db/storage/backup_cursor_state.h
+++ b/src/mongo/db/storage/backup_cursor_state.h
@@ -34,13 +34,14 @@
 #include <vector>
 
 #include "mongo/db/pipeline/document.h"
+#include "mongo/db/storage/storage_engine.h"
 
 namespace mongo {
 
 struct BackupCursorState {
     UUID backupId;
     boost::optional<Document> preamble;
-    std::vector<std::string> filenames;
+    std::vector<StorageEngine::BackupBlock> blocksToCopy;
 };
 
 struct BackupCursorExtendState {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index 6ae745c5587..17c70079754 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -270,10 +270,10 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) {
     _cachePressureForTest = pressure;
 }
 
-StatusWith<std::vector<std::string>> DevNullKVEngine::beginNonBlockingBackup(
+StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup(
     OperationContext* opCtx) {
-    std::vector<std::string> filesToCopy = {"filename.wt"};
-    return filesToCopy;
+    std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}};
+    return blocksToCopy;
 }
 
 StatusWith<std::vector<std::string>> DevNullKVEngine::extendBackupCursor(OperationContext* opCtx) {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index ddf0406bf10..0dae32f61b1 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -140,7 +140,7 @@ public:
 
     virtual void endBackup(OperationContext* opCtx) {}
 
-    virtual StatusWith<std::vector<std::string>> beginNonBlockingBackup(
+    virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
         OperationContext* opCtx) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override {}
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 12054cb6ec0..a87983f9ef7 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -223,7 +223,8 @@ public:
         MONGO_UNREACHABLE;
     }
 
-    virtual StatusWith<std::vector<std::string>> beginNonBlockingBackup(OperationContext* opCtx) {
+    virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+        OperationContext* opCtx) {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
     }
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index a1a68e421dc..fcd5b045ada 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -68,6 +68,12 @@ public:
     using OldestActiveTransactionTimestampCallback =
         std::function<OldestActiveTransactionTimestampResult(Timestamp stableTimestamp)>;
 
+    struct BackupBlock {
+        std::string filename;
+        std::uint64_t offset;
+        std::uint64_t length;
+    };
+
     /**
      * The interface for creating new instances of storage engines.
      *
@@ -267,10 +273,8 @@ public:
         return;
     }
 
-    virtual StatusWith<std::vector<std::string>> beginNonBlockingBackup(OperationContext* opCtx) {
-        return Status(ErrorCodes::CommandNotSupported,
-                      "The current storage engine does not support a concurrent mode.");
-    }
+    virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
+        OperationContext* opCtx) = 0;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) {
         return;
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index c0e3e76bad3..1ae6c8783bd 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -647,7 +647,7 @@ void StorageEngineImpl::endBackup(OperationContext* opCtx) {
     _inBackupMode = false;
 }
 
-StatusWith<std::vector<std::string>> StorageEngineImpl::beginNonBlockingBackup(
+StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup(
     OperationContext* opCtx) {
     return _engine->beginNonBlockingBackup(opCtx);
 }
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 097c16ac2bc..66f6951e21b 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -97,9 +97,10 @@ public:
 
     virtual void endBackup(OperationContext* opCtx);
 
-    virtual StatusWith<std::vector<std::string>> beginNonBlockingBackup(OperationContext* opCtx);
+    virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
+        OperationContext* opCtx) override;
 
-    virtual void endNonBlockingBackup(OperationContext* opCtx);
+    virtual void endNonBlockingBackup(OperationContext* opCtx) override;
 
     virtual StatusWith<std::vector<std::string>> extendBackupCursor(OperationContext* opCtx);
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index a1f498724e2..013db40b696 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -39,6 +39,7 @@
 #define NVALGRIND
 #endif
 
+#include <fmt/format.h>
 #include <memory>
 
 #include "mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h"
@@ -510,11 +511,10 @@ stdx::function<bool(StringData)> initRsOplogBackgroundThreadCallback = [](String
     fassertFailed(40358);
 };
 
-StatusWith<std::vector<std::string>> getDataFilesFromBackupCursor(WT_CURSOR* cursor,
-                                                                  std::string dbPath,
-                                                                  const char* statusPrefix) {
+StatusWith<std::vector<StorageEngine::BackupBlock>> getDataBlocksFromBackupCursor(
+    WT_CURSOR* cursor, std::string dbPath, const char* statusPrefix) {
     int wtRet;
-    std::vector<std::string> files;
+    std::vector<StorageEngine::BackupBlock> blocks;
     const char* filename;
     const auto directoryPath = boost::filesystem::path(dbPath);
     const auto wiredTigerLogFilePrefix = "WiredTigerLog";
@@ -530,12 +530,19 @@ StatusWith<std::vector<std::string>> getDataFilesFromBackupCursor(WT_CURSOR* cur
         }
         filePath /= name;
 
-        files.push_back(filePath.string());
+        boost::system::error_code errorCode;
+        const std::uint64_t filesize = boost::filesystem::file_size(filePath, errorCode);
+        uassert(31317,
+                "Failed to get a file's size. Filename: {} Error: {}"_format(filePath.string(),
+                                                                             errorCode.message()),
+                !errorCode);
+        blocks.push_back({filePath.string(), 0, filesize});
     }
+
     if (wtRet != WT_NOTFOUND) {
         return wtRCToStatus(wtRet, statusPrefix);
     }
-    return files;
+    return blocks;
 }
 
 }  // namespace
@@ -1038,7 +1045,7 @@ void WiredTigerKVEngine::endBackup(OperationContext* opCtx) {
     _backupSession.reset();
 }
 
-StatusWith<std::vector<std::string>> WiredTigerKVEngine::beginNonBlockingBackup(
+StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup(
     OperationContext* opCtx) {
     uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral());
 
@@ -1061,18 +1068,18 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::beginNonBlockingBackup(
         return wtRCToStatus(wtRet);
     }
 
-    auto swFilesToCopy =
-        getDataFilesFromBackupCursor(cursor, _path, "Error opening backup cursor.");
+    auto swBlocksToCopy =
+        getDataBlocksFromBackupCursor(cursor, _path, "Error opening backup cursor.");
 
-    if (!swFilesToCopy.isOK()) {
-        return swFilesToCopy;
+    if (!swBlocksToCopy.isOK()) {
+        return swBlocksToCopy;
     }
 
     pinOplogGuard.dismiss();
     _backupSession = std::move(sessionRaii);
     _backupCursor = cursor;
 
-    return swFilesToCopy;
+    return swBlocksToCopy;
 }
 
 void WiredTigerKVEngine::endNonBlockingBackup(OperationContext* opCtx) {
@@ -1095,15 +1102,28 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::extendBackupCursor(
         return wtRCToStatus(wtRet);
     }
 
-    auto swFilesToCopy =
-        getDataFilesFromBackupCursor(cursor, _path, "Error extending backup cursor.");
+    StatusWith<std::vector<StorageEngine::BackupBlock>> swBlocksToCopy =
+        getDataBlocksFromBackupCursor(cursor, _path, "Error extending backup cursor.");
 
     wtRet = cursor->close(cursor);
     if (wtRet != 0) {
         return wtRCToStatus(wtRet);
     }
 
-    return swFilesToCopy;
+    if (!swBlocksToCopy.isOK()) {
+        return swBlocksToCopy.getStatus();
+    }
+
+    // Journal files returned from the future backup cursor are not intended to have block level
+    // information. For now, this is being explicitly codified by transforming the result into a
+    // vector of filename strings. The lack of block/filesize information instructs the client to
+    // copy the whole file.
+    std::vector<std::string> filenames;
+    for (const auto& block : swBlocksToCopy.getValue()) {
+        filenames.push_back(std::move(block.filename));
+    }
+
+    return {filenames};
 }
 
 void WiredTigerKVEngine::syncSizeInfo(bool sync) const {
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 8b8507aa492..00dea19eb08 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -174,7 +174,8 @@ public:
 
     void endBackup(OperationContext* opCtx) override;
 
-    StatusWith<std::vector<std::string>> beginNonBlockingBackup(OperationContext* opCtx) override;
+    StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+        OperationContext* opCtx) override;
 
     void endNonBlockingBackup(OperationContext* opCtx) override;