SERVER-45481 Change the backup API to return the blocks to copy for incremental backup

10 files changed, 98 insertions, 42 deletions

diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 7b85e350cfa..6b15acb840f 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -90,7 +90,7 @@ public:
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
     }
-    StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+    StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const StorageEngine::BackupOptions& options) final {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
diff --git a/src/mongo/db/storage/backup_cursor_state.h b/src/mongo/db/storage/backup_cursor_state.h
index d0af9c45b2b..814e81c4765 100644
--- a/src/mongo/db/storage/backup_cursor_state.h
+++ b/src/mongo/db/storage/backup_cursor_state.h
@@ -41,7 +41,7 @@ namespace mongo {
 struct BackupCursorState {
     UUID backupId;
     boost::optional<Document> preamble;
-    std::vector<StorageEngine::BackupBlock> blocksToCopy;
+    StorageEngine::BackupInformation backupInformation;
 };
 
 struct BackupCursorExtendState {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index f1982f314d8..b73bb4a9d4d 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -253,10 +253,12 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) {
     _cachePressureForTest = pressure;
 }
 
-StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup(
+StatusWith<StorageEngine::BackupInformation> DevNullKVEngine::beginNonBlockingBackup(
     OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
-    std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}};
-    return blocksToCopy;
+    StorageEngine::BackupInformation backupInformation;
+    StorageEngine::BackupFile backupFile(0);
+    backupInformation.insert({"filename.wt", backupFile});
+    return backupInformation;
 }
 
 StatusWith<std::vector<std::string>> DevNullKVEngine::extendBackupCursor(OperationContext* opCtx) {
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index 67989a4774f..3f73b4e4ade 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -139,7 +139,7 @@ public:
 
     virtual void endBackup(OperationContext* opCtx) {}
 
-    virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+    virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const StorageEngine::BackupOptions& options) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override {}
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index 337a40d13b9..7c3316cb6a5 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -237,7 +237,7 @@ public:
         MONGO_UNREACHABLE;
     }
 
-    virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+    virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 5f609793bfa..45e2c444228 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -78,12 +78,27 @@ public:
     };
 
     struct BackupBlock {
-        std::string filename;
-        std::uint64_t offset;
-        std::uint64_t length;
+        std::uint64_t offset = 0;
+        std::uint64_t length = 0;
     };
 
     /**
+     * Contains the size of the file to be backed up. This allows the backup application to safely
+     * truncate the file for incremental backups. Files that have had changes since the last
+     * incremental backup will have their changed file blocks listed.
+     */
+    struct BackupFile {
+        BackupFile() = delete;
+        explicit BackupFile(std::uint64_t fileSize) : fileSize(fileSize){};
+
+        std::uint64_t fileSize;
+        std::vector<BackupBlock> blocksToCopy;
+    };
+
+    // Map of filenames to backup file information.
+    using BackupInformation = stdx::unordered_map<std::string, BackupFile>;
+
+    /**
      * The interface for creating new instances of storage engines.
      *
      * A storage engine provides an instance of this class (along with an associated
@@ -331,7 +346,7 @@ public:
      * 'srcBackupName' must not exist when 'incrementalBackup' is false but may or may not exist
      * when 'incrementalBackup' is true.
      */
-    virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
+    virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const BackupOptions& options) = 0;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) = 0;
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index dbe69146dd3..15dc613ebe3 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -670,7 +670,7 @@ Status StorageEngineImpl::disableIncrementalBackup(OperationContext* opCtx) {
     return _engine->disableIncrementalBackup(opCtx);
 }
 
-StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup(
+StatusWith<StorageEngine::BackupInformation> StorageEngineImpl::beginNonBlockingBackup(
     OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
     return _engine->beginNonBlockingBackup(opCtx, options);
 }
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 44c08054f1d..f34c5bbac29 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -99,7 +99,7 @@ public:
 
     virtual Status disableIncrementalBackup(OperationContext* opCtx) override;
 
-    virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
+    virtual StatusWith<BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const BackupOptions& options) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override;
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index f85299d0422..1aa462b67a8 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -596,10 +596,14 @@ Status OpenReadTransactionParam::setFromString(const std::string& str) {
 
 namespace {
 
-StatusWith<std::vector<StorageEngine::BackupBlock>> getDataBlocksFromBackupCursor(
-    WT_CURSOR* cursor, std::string dbPath, const char* statusPrefix) {
+StatusWith<StorageEngine::BackupInformation> getBackupInformationFromBackupCursor(
+    WT_SESSION* session,
+    WT_CURSOR* cursor,
+    bool incrementalBackup,
+    std::string dbPath,
+    const char* statusPrefix) {
     int wtRet;
-    std::vector<StorageEngine::BackupBlock> blocks;
+    StorageEngine::BackupInformation backupInformation;
     const char* filename;
     const auto directoryPath = boost::filesystem::path(dbPath);
     const auto wiredTigerLogFilePrefix = "WiredTigerLog";
@@ -608,26 +612,57 @@ StatusWith<std::vector<StorageEngine::BackupBlock>> getDataBlocksFromBackupCurso
 
         std::string name(filename);
 
-        auto filePath = directoryPath;
+        boost::filesystem::path filePath = directoryPath;
+        boost::filesystem::path relativePath;
         if (name.find(wiredTigerLogFilePrefix) == 0) {
             // TODO SERVER-13455:replace `journal/` with the configurable journal path.
             filePath /= boost::filesystem::path("journal");
+            relativePath /= boost::filesystem::path("journal");
         }
         filePath /= name;
+        relativePath /= name;
 
-        boost::system::error_code errorCode;
-        const std::uint64_t filesize = boost::filesystem::file_size(filePath, errorCode);
-        uassert(31317,
-                "Failed to get a file's size. Filename: {} Error: {}"_format(filePath.string(),
-                                                                             errorCode.message()),
-                !errorCode);
-        blocks.push_back({filePath.string(), 0, filesize});
+        // TODO: SERVER-44410 Implement fileSize.
+        std::uint64_t fileSize = 0;
+        StorageEngine::BackupFile backupFile(fileSize);
+        backupInformation.insert({filePath.string(), backupFile});
+
+        if (!incrementalBackup) {
+            continue;
+        }
+
+        // For each file listed, open a duplicate backup cursor and get the blocks to copy.
+        std::stringstream ss;
+        ss << "incremental=(file=\"" << str::escape(relativePath.string()) << "\")";
+        const std::string config = ss.str();
+        WT_CURSOR* dupCursor;
+        wtRet = session->open_cursor(session, nullptr, cursor, config.c_str(), &dupCursor);
+        if (wtRet != 0) {
+            return wtRCToStatus(wtRet);
+        }
+
+        while ((wtRet = dupCursor->next(dupCursor)) == 0) {
+            uint64_t offset, size, type;
+            invariantWTOK(dupCursor->get_key(dupCursor, &offset, &size, &type));
+            LOG(2) << "Block to copy for incremental backup: filename: " << filePath.string()
+                   << ", offset: " << offset << ", size: " << size << ", type: " << type;
+            backupInformation.at(filePath.string()).blocksToCopy.push_back({offset, size});
+        }
+
+        if (wtRet != WT_NOTFOUND) {
+            return wtRCToStatus(wtRet);
+        }
+
+        wtRet = dupCursor->close(dupCursor);
+        if (wtRet != 0) {
+            return wtRCToStatus(wtRet);
+        }
     }
 
     if (wtRet != WT_NOTFOUND) {
         return wtRCToStatus(wtRet, statusPrefix);
     }
-    return blocks;
+    return backupInformation;
 }
 
 }  // namespace
@@ -1167,7 +1202,7 @@ Status WiredTigerKVEngine::disableIncrementalBackup(OperationContext* opCtx) {
     return Status::OK();
 }
 
-StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup(
+StatusWith<StorageEngine::BackupInformation> WiredTigerKVEngine::beginNonBlockingBackup(
     OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
     uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral());
 
@@ -1205,18 +1240,18 @@ StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNon
         return wtRCToStatus(wtRet);
     }
 
-    auto swBlocksToCopy =
-        getDataBlocksFromBackupCursor(cursor, _path, "Error opening backup cursor.");
+    auto swBackupInfo = getBackupInformationFromBackupCursor(
+        session, cursor, options.incrementalBackup, _path, "Error opening backup cursor.");
 
-    if (!swBlocksToCopy.isOK()) {
-        return swBlocksToCopy;
+    if (!swBackupInfo.isOK()) {
+        return swBackupInfo;
     }
 
     pinOplogGuard.dismiss();
     _backupSession = std::move(sessionRaii);
     _backupCursor = cursor;
 
-    return swBlocksToCopy;
+    return swBackupInfo;
 }
 
 void WiredTigerKVEngine::endNonBlockingBackup(OperationContext* opCtx) {
@@ -1232,6 +1267,8 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::extendBackupCursor(
     uassert(51033, "Cannot extend backup cursor with in-memory mode.", !isEphemeral());
     invariant(_backupCursor);
 
+    // The "target=(\"log:\")" configuration string for the cursor will ensure that we only see the
+    // log files when iterating on the cursor.
     WT_CURSOR* cursor = nullptr;
     WT_SESSION* session = _backupSession->getSession();
     int wtRet = session->open_cursor(session, nullptr, _backupCursor, "target=(\"log:\")", &cursor);
@@ -1239,25 +1276,27 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::extendBackupCursor(
         return wtRCToStatus(wtRet);
     }
 
-    StatusWith<std::vector<StorageEngine::BackupBlock>> swBlocksToCopy =
-        getDataBlocksFromBackupCursor(cursor, _path, "Error extending backup cursor.");
+    StatusWith<StorageEngine::BackupInformation> swBackupInfo =
+        getBackupInformationFromBackupCursor(
+            session, cursor, /*incrementalBackup=*/false, _path, "Error extending backup cursor.");
 
     wtRet = cursor->close(cursor);
     if (wtRet != 0) {
         return wtRCToStatus(wtRet);
     }
 
-    if (!swBlocksToCopy.isOK()) {
-        return swBlocksToCopy.getStatus();
+    if (!swBackupInfo.isOK()) {
+        return swBackupInfo.getStatus();
     }
 
-    // Journal files returned from the future backup cursor are not intended to have block level
-    // information. For now, this is being explicitly codified by transforming the result into a
-    // vector of filename strings. The lack of block/filesize information instructs the client to
-    // copy the whole file.
+    // Once all the backup cursors have been opened on a sharded cluster, we need to ensure that the
+    // data being copied from each shard is at the same point-in-time across the entire cluster to
+    // have a consistent view of the data. For shards that opened their backup cursor before the
+    // established point-in-time for backup, they will need to create a full copy of the additional
+    // journal files returned by this method to ensure a consistent backup of the data is taken.
     std::vector<std::string> filenames;
-    for (const auto& block : swBlocksToCopy.getValue()) {
-        filenames.push_back(std::move(block.filename));
+    for (const auto& entry : swBackupInfo.getValue()) {
+        filenames.push_back(entry.first);
     }
 
     return {filenames};
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 4dccf63f627..40295cb4670 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -181,7 +181,7 @@ public:
 
     Status disableIncrementalBackup(OperationContext* opCtx) override;
 
-    StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
+    StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup(
         OperationContext* opCtx, const StorageEngine::BackupOptions& options) override;
 
     void endNonBlockingBackup(OperationContext* opCtx) override;