SERVER-45581 Configure the incremental backup API to allow block size granularity to be specified

(cherry picked from commit a6f0b19ab9c886d15ff691cfcd73685d621b5ef5)

16 files changed, 36 insertions, 57 deletions

diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp
index 2c25e41b047..0ead1ab40f0 100644
--- a/src/mongo/db/catalog/catalog_control_test.cpp
+++ b/src/mongo/db/catalog/catalog_control_test.cpp
@@ -79,10 +79,7 @@ public:
                       "The current storage engine doesn't support backup mode");
     }
     StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) final {
+        OperationContext* opCtx, const StorageEngine::BackupOptions& options) final {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
     }
diff --git a/src/mongo/db/pipeline/mongo_process_interface.h b/src/mongo/db/pipeline/mongo_process_interface.h
index 4d2a6062bc8..dc9b6ad507c 100644
--- a/src/mongo/db/pipeline/mongo_process_interface.h
+++ b/src/mongo/db/pipeline/mongo_process_interface.h
@@ -352,7 +352,7 @@ public:
      * The following methods forward to the BackupCursorHooks decorating the ServiceContext.
      */
     virtual BackupCursorState openBackupCursor(OperationContext* opCtx,
-                                               const BackupOptions& options) = 0;
+                                               const StorageEngine::BackupOptions& options) = 0;
 
     virtual void closeBackupCursor(OperationContext* opCtx, const UUID& backupId) = 0;
 
diff --git a/src/mongo/db/pipeline/mongos_process_interface.h b/src/mongo/db/pipeline/mongos_process_interface.h
index 6cfe9067494..94977c44d35 100644
--- a/src/mongo/db/pipeline/mongos_process_interface.h
+++ b/src/mongo/db/pipeline/mongos_process_interface.h
@@ -191,7 +191,7 @@ public:
      * a mongos.
      */
     BackupCursorState openBackupCursor(OperationContext* opCtx,
-                                       const BackupOptions& options) final {
+                                       const StorageEngine::BackupOptions& options) final {
         MONGO_UNREACHABLE;
     }
 
diff --git a/src/mongo/db/pipeline/process_interface_standalone.cpp b/src/mongo/db/pipeline/process_interface_standalone.cpp
index 8e60bbe57de..125dc53902c 100644
--- a/src/mongo/db/pipeline/process_interface_standalone.cpp
+++ b/src/mongo/db/pipeline/process_interface_standalone.cpp
@@ -496,8 +496,8 @@ boost::optional<Document> MongoInterfaceStandalone::lookupSingleDocument(
     return lookedUpDocument;
 }
 
-BackupCursorState MongoInterfaceStandalone::openBackupCursor(OperationContext* opCtx,
-                                                             const BackupOptions& options) {
+BackupCursorState MongoInterfaceStandalone::openBackupCursor(
+    OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
     auto backupCursorHooks = BackupCursorHooks::get(opCtx->getServiceContext());
     if (backupCursorHooks->enabled()) {
         return backupCursorHooks->openBackupCursor(opCtx, options);
diff --git a/src/mongo/db/pipeline/process_interface_standalone.h b/src/mongo/db/pipeline/process_interface_standalone.h
index f25650cc247..80df404ff48 100644
--- a/src/mongo/db/pipeline/process_interface_standalone.h
+++ b/src/mongo/db/pipeline/process_interface_standalone.h
@@ -125,7 +125,8 @@ public:
         bool allowSpeculativeMajorityRead = false) final;
     std::vector<GenericCursor> getIdleCursors(const boost::intrusive_ptr<ExpressionContext>& expCtx,
                                               CurrentOpUserMode userMode) const final;
-    BackupCursorState openBackupCursor(OperationContext* opCtx, const BackupOptions& options) final;
+    BackupCursorState openBackupCursor(OperationContext* opCtx,
+                                       const StorageEngine::BackupOptions& options) final;
     void closeBackupCursor(OperationContext* opCtx, const UUID& backupId) final;
     BackupCursorExtendState extendBackupCursor(OperationContext* opCtx,
                                                const UUID& backupId,
diff --git a/src/mongo/db/pipeline/stub_mongo_process_interface.h b/src/mongo/db/pipeline/stub_mongo_process_interface.h
index 389524d1151..e63d066a35f 100644
--- a/src/mongo/db/pipeline/stub_mongo_process_interface.h
+++ b/src/mongo/db/pipeline/stub_mongo_process_interface.h
@@ -183,7 +183,7 @@ public:
     }
 
     BackupCursorState openBackupCursor(OperationContext* opCtx,
-                                       const BackupOptions& options) final {
+                                       const StorageEngine::BackupOptions& options) final {
         return BackupCursorState{UUID::gen(), boost::none, {}};
     }
 
diff --git a/src/mongo/db/storage/backup_cursor_hooks.cpp b/src/mongo/db/storage/backup_cursor_hooks.cpp
index 6f5bd8ee830..de4c5098278 100644
--- a/src/mongo/db/storage/backup_cursor_hooks.cpp
+++ b/src/mongo/db/storage/backup_cursor_hooks.cpp
@@ -78,7 +78,7 @@ void BackupCursorHooks::fsyncUnlock(OperationContext* opCtx) {
 }
 
 BackupCursorState BackupCursorHooks::openBackupCursor(OperationContext* opCtx,
-                                                      const BackupOptions& options) {
+                                                      const StorageEngine::BackupOptions& options) {
     MONGO_UNREACHABLE;
 }
 
diff --git a/src/mongo/db/storage/backup_cursor_hooks.h b/src/mongo/db/storage/backup_cursor_hooks.h
index ef40a9a23b9..6852d544f25 100644
--- a/src/mongo/db/storage/backup_cursor_hooks.h
+++ b/src/mongo/db/storage/backup_cursor_hooks.h
@@ -39,13 +39,6 @@ class OperationContext;
 class ServiceContext;
 class StorageEngine;
 
-struct BackupOptions {
-    bool disableIncrementalBackup = false;
-    bool incrementalBackup = false;
-    boost::optional<std::string> thisBackupName;
-    boost::optional<std::string> srcBackupName;
-};
-
 class BackupCursorHooks {
 public:
     using InitializerFunction = std::function<std::unique_ptr<BackupCursorHooks>(StorageEngine*)>;
@@ -69,7 +62,7 @@ public:
     virtual void fsyncUnlock(OperationContext* opCtx);
 
     virtual BackupCursorState openBackupCursor(OperationContext* opCtx,
-                                               const BackupOptions& options);
+                                               const StorageEngine::BackupOptions& options);
 
     virtual void closeBackupCursor(OperationContext* opCtx, const UUID& backupId);
 
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
index db0939d854f..9b3bc072b2b 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp
@@ -271,10 +271,7 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup(
-    OperationContext* opCtx,
-    bool incrementalBackup,
-    boost::optional<std::string> thisBackupName,
-    boost::optional<std::string> srcBackupName) {
+    OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
     std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}};
     return blocksToCopy;
 }
diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h
index c335a5b53ea..994957684e4 100644
--- a/src/mongo/db/storage/devnull/devnull_kv_engine.h
+++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h
@@ -141,10 +141,7 @@ public:
     virtual void endBackup(OperationContext* opCtx) {}
 
     virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) override;
+        OperationContext* opCtx, const StorageEngine::BackupOptions& options) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override {}
 
diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h
index abce2f51311..3770da212bb 100644
--- a/src/mongo/db/storage/kv/kv_engine.h
+++ b/src/mongo/db/storage/kv/kv_engine.h
@@ -228,10 +228,7 @@ public:
     }
 
     virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) {
+        OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
         return Status(ErrorCodes::CommandNotSupported,
                       "The current storage engine doesn't support backup mode");
     }
diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h
index 01ea4aaa5e5..a5fe2b5d8ed 100644
--- a/src/mongo/db/storage/storage_engine.h
+++ b/src/mongo/db/storage/storage_engine.h
@@ -68,6 +68,14 @@ public:
     using OldestActiveTransactionTimestampCallback =
         std::function<OldestActiveTransactionTimestampResult(Timestamp stableTimestamp)>;
 
+    struct BackupOptions {
+        bool disableIncrementalBackup = false;
+        bool incrementalBackup = false;
+        int blockSizeMB = 16;
+        boost::optional<std::string> thisBackupName;
+        boost::optional<std::string> srcBackupName;
+    };
+
     struct BackupBlock {
         std::string filename;
         std::uint64_t offset;
@@ -287,6 +295,9 @@ public:
      * storage engine will take a backup called 'thisBackupName' which will contain the changes made
      * to data files since the backup named 'srcBackupName'.
      *
+     * The storage engine must use an upper bound limit of 'blockSizeMB' when returning changed
+     * file blocks.
+     *
      * The first full backup meant for incremental and future incremental backups must pass
      * 'incrementalBackup' as true.
      * 'thisBackupName' must exist only if 'incrementalBackup' is true.
@@ -294,10 +305,7 @@ public:
      * when 'incrementalBackup' is true.
      */
     virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) = 0;
+        OperationContext* opCtx, const BackupOptions& options) = 0;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) {
         return;
diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp
index 0ec1f6f3392..49a28591f93 100644
--- a/src/mongo/db/storage/storage_engine_impl.cpp
+++ b/src/mongo/db/storage/storage_engine_impl.cpp
@@ -652,11 +652,8 @@ Status StorageEngineImpl::disableIncrementalBackup(OperationContext* opCtx) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup(
-    OperationContext* opCtx,
-    bool incrementalBackup,
-    boost::optional<std::string> thisBackupName,
-    boost::optional<std::string> srcBackupName) {
-    return _engine->beginNonBlockingBackup(opCtx, incrementalBackup, thisBackupName, srcBackupName);
+    OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
+    return _engine->beginNonBlockingBackup(opCtx, options);
 }
 
 void StorageEngineImpl::endNonBlockingBackup(OperationContext* opCtx) {
diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h
index 53a807ae04c..c31aa8aa2ab 100644
--- a/src/mongo/db/storage/storage_engine_impl.h
+++ b/src/mongo/db/storage/storage_engine_impl.h
@@ -100,10 +100,7 @@ public:
     virtual Status disableIncrementalBackup(OperationContext* opCtx) override;
 
     virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) override;
+        OperationContext* opCtx, const BackupOptions& options) override;
 
     virtual void endNonBlockingBackup(OperationContext* opCtx) override;
 
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index e08cd46dca9..c71795a5d35 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -1068,20 +1068,18 @@ Status WiredTigerKVEngine::disableIncrementalBackup(OperationContext* opCtx) {
 }
 
 StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup(
-    OperationContext* opCtx,
-    bool incrementalBackup,
-    boost::optional<std::string> thisBackupName,
-    boost::optional<std::string> srcBackupName) {
+    OperationContext* opCtx, const StorageEngine::BackupOptions& options) {
     uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral());
 
     std::stringstream ss;
-    if (incrementalBackup) {
-        invariant(thisBackupName);
+    if (options.incrementalBackup) {
+        invariant(options.thisBackupName);
         ss << "incremental=(enabled=true,force_stop=false,";
-        ss << "this_id=" << std::quoted(str::escape(*thisBackupName)) << ",";
+        ss << "granularity=" << options.blockSizeMB << "MB,";
+        ss << "this_id=" << std::quoted(str::escape(*options.thisBackupName)) << ",";
 
-        if (srcBackupName) {
-            ss << "src_id=" << std::quoted(str::escape(*srcBackupName)) << ",";
+        if (options.srcBackupName) {
+            ss << "src_id=" << std::quoted(str::escape(*options.srcBackupName)) << ",";
         }
 
         ss << ")";
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
index 81a1cead276..a260b2a2684 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h
@@ -177,10 +177,7 @@ public:
     Status disableIncrementalBackup(OperationContext* opCtx) override;
 
     StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup(
-        OperationContext* opCtx,
-        bool incrementalBackup,
-        boost::optional<std::string> thisBackupName,
-        boost::optional<std::string> srcBackupName) override;
+        OperationContext* opCtx, const StorageEngine::BackupOptions& options) override;
 
     void endNonBlockingBackup(OperationContext* opCtx) override;