diff options
author | Gregory Wlodarek <gregory.wlodarek@mongodb.com> | 2020-01-21 13:27:45 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-25 22:26:41 +0000 |
commit | 1190ea2c3e23b1b0b9cbc98517b8f08b1acba8cc (patch) | |
tree | fc8acf9e2857aaee9e53e2615fc268428f3212b1 | |
parent | 8cbd99de29d6c610dec8540f4ddbda36fc9990e3 (diff) | |
download | mongo-1190ea2c3e23b1b0b9cbc98517b8f08b1acba8cc.tar.gz |
SERVER-45581 Configure the incremental backup API to allow block size granularity to be specified
(cherry picked from commit a6f0b19ab9c886d15ff691cfcd73685d621b5ef5)
-rw-r--r-- | src/mongo/db/catalog/catalog_control_test.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/pipeline/mongo_process_interface.h | 2 | ||||
-rw-r--r-- | src/mongo/db/pipeline/mongos_process_interface.h | 2 | ||||
-rw-r--r-- | src/mongo/db/pipeline/process_interface_standalone.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/pipeline/process_interface_standalone.h | 3 | ||||
-rw-r--r-- | src/mongo/db/pipeline/stub_mongo_process_interface.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/backup_cursor_hooks.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/backup_cursor_hooks.h | 9 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.h | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_engine.h | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine.h | 16 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine_impl.cpp | 7 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine_impl.h | 5 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 16 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h | 5 |
16 files changed, 36 insertions, 57 deletions
diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp index 2c25e41b047..0ead1ab40f0 100644 --- a/src/mongo/db/catalog/catalog_control_test.cpp +++ b/src/mongo/db/catalog/catalog_control_test.cpp @@ -79,10 +79,7 @@ public: "The current storage engine doesn't support backup mode"); } StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) final { + OperationContext* opCtx, const StorageEngine::BackupOptions& options) final { return Status(ErrorCodes::CommandNotSupported, "The current storage engine doesn't support backup mode"); } diff --git a/src/mongo/db/pipeline/mongo_process_interface.h b/src/mongo/db/pipeline/mongo_process_interface.h index 4d2a6062bc8..dc9b6ad507c 100644 --- a/src/mongo/db/pipeline/mongo_process_interface.h +++ b/src/mongo/db/pipeline/mongo_process_interface.h @@ -352,7 +352,7 @@ public: * The following methods forward to the BackupCursorHooks decorating the ServiceContext. */ virtual BackupCursorState openBackupCursor(OperationContext* opCtx, - const BackupOptions& options) = 0; + const StorageEngine::BackupOptions& options) = 0; virtual void closeBackupCursor(OperationContext* opCtx, const UUID& backupId) = 0; diff --git a/src/mongo/db/pipeline/mongos_process_interface.h b/src/mongo/db/pipeline/mongos_process_interface.h index 6cfe9067494..94977c44d35 100644 --- a/src/mongo/db/pipeline/mongos_process_interface.h +++ b/src/mongo/db/pipeline/mongos_process_interface.h @@ -191,7 +191,7 @@ public: * a mongos. */ BackupCursorState openBackupCursor(OperationContext* opCtx, - const BackupOptions& options) final { + const StorageEngine::BackupOptions& options) final { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/pipeline/process_interface_standalone.cpp b/src/mongo/db/pipeline/process_interface_standalone.cpp index 8e60bbe57de..125dc53902c 100644 --- a/src/mongo/db/pipeline/process_interface_standalone.cpp +++ b/src/mongo/db/pipeline/process_interface_standalone.cpp @@ -496,8 +496,8 @@ boost::optional<Document> MongoInterfaceStandalone::lookupSingleDocument( return lookedUpDocument; } -BackupCursorState MongoInterfaceStandalone::openBackupCursor(OperationContext* opCtx, - const BackupOptions& options) { +BackupCursorState MongoInterfaceStandalone::openBackupCursor( + OperationContext* opCtx, const StorageEngine::BackupOptions& options) { auto backupCursorHooks = BackupCursorHooks::get(opCtx->getServiceContext()); if (backupCursorHooks->enabled()) { return backupCursorHooks->openBackupCursor(opCtx, options); diff --git a/src/mongo/db/pipeline/process_interface_standalone.h b/src/mongo/db/pipeline/process_interface_standalone.h index f25650cc247..80df404ff48 100644 --- a/src/mongo/db/pipeline/process_interface_standalone.h +++ b/src/mongo/db/pipeline/process_interface_standalone.h @@ -125,7 +125,8 @@ public: bool allowSpeculativeMajorityRead = false) final; std::vector<GenericCursor> getIdleCursors(const boost::intrusive_ptr<ExpressionContext>& expCtx, CurrentOpUserMode userMode) const final; - BackupCursorState openBackupCursor(OperationContext* opCtx, const BackupOptions& options) final; + BackupCursorState openBackupCursor(OperationContext* opCtx, + const StorageEngine::BackupOptions& options) final; void closeBackupCursor(OperationContext* opCtx, const UUID& backupId) final; BackupCursorExtendState extendBackupCursor(OperationContext* opCtx, const UUID& backupId, diff --git a/src/mongo/db/pipeline/stub_mongo_process_interface.h b/src/mongo/db/pipeline/stub_mongo_process_interface.h index 389524d1151..e63d066a35f 100644 --- a/src/mongo/db/pipeline/stub_mongo_process_interface.h +++ b/src/mongo/db/pipeline/stub_mongo_process_interface.h @@ -183,7 +183,7 @@ public: } BackupCursorState openBackupCursor(OperationContext* opCtx, - const BackupOptions& options) final { + const StorageEngine::BackupOptions& options) final { return BackupCursorState{UUID::gen(), boost::none, {}}; } diff --git a/src/mongo/db/storage/backup_cursor_hooks.cpp b/src/mongo/db/storage/backup_cursor_hooks.cpp index 6f5bd8ee830..de4c5098278 100644 --- a/src/mongo/db/storage/backup_cursor_hooks.cpp +++ b/src/mongo/db/storage/backup_cursor_hooks.cpp @@ -78,7 +78,7 @@ void BackupCursorHooks::fsyncUnlock(OperationContext* opCtx) { } BackupCursorState BackupCursorHooks::openBackupCursor(OperationContext* opCtx, - const BackupOptions& options) { + const StorageEngine::BackupOptions& options) { MONGO_UNREACHABLE; } diff --git a/src/mongo/db/storage/backup_cursor_hooks.h b/src/mongo/db/storage/backup_cursor_hooks.h index ef40a9a23b9..6852d544f25 100644 --- a/src/mongo/db/storage/backup_cursor_hooks.h +++ b/src/mongo/db/storage/backup_cursor_hooks.h @@ -39,13 +39,6 @@ class OperationContext; class ServiceContext; class StorageEngine; -struct BackupOptions { - bool disableIncrementalBackup = false; - bool incrementalBackup = false; - boost::optional<std::string> thisBackupName; - boost::optional<std::string> srcBackupName; -}; - class BackupCursorHooks { public: using InitializerFunction = std::function<std::unique_ptr<BackupCursorHooks>(StorageEngine*)>; @@ -69,7 +62,7 @@ public: virtual void fsyncUnlock(OperationContext* opCtx); virtual BackupCursorState openBackupCursor(OperationContext* opCtx, - const BackupOptions& options); + const StorageEngine::BackupOptions& options); virtual void closeBackupCursor(OperationContext* opCtx, const UUID& backupId); diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index db0939d854f..9b3bc072b2b 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -271,10 +271,7 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) { } StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) { + OperationContext* opCtx, const StorageEngine::BackupOptions& options) { std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}}; return blocksToCopy; } diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index c335a5b53ea..994957684e4 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -141,10 +141,7 @@ public: virtual void endBackup(OperationContext* opCtx) {} virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) override; + OperationContext* opCtx, const StorageEngine::BackupOptions& options) override; virtual void endNonBlockingBackup(OperationContext* opCtx) override {} diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index abce2f51311..3770da212bb 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -228,10 +228,7 @@ public: } virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) { + OperationContext* opCtx, const StorageEngine::BackupOptions& options) { return Status(ErrorCodes::CommandNotSupported, "The current storage engine doesn't support backup mode"); } diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 01ea4aaa5e5..a5fe2b5d8ed 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -68,6 +68,14 @@ public: using OldestActiveTransactionTimestampCallback = std::function<OldestActiveTransactionTimestampResult(Timestamp stableTimestamp)>; + struct BackupOptions { + bool disableIncrementalBackup = false; + bool incrementalBackup = false; + int blockSizeMB = 16; + boost::optional<std::string> thisBackupName; + boost::optional<std::string> srcBackupName; + }; + struct BackupBlock { std::string filename; std::uint64_t offset; @@ -287,6 +295,9 @@ public: * storage engine will take a backup called 'thisBackupName' which will contain the changes made * to data files since the backup named 'srcBackupName'. * + * The storage engine must use an upper bound limit of 'blockSizeMB' when returning changed + * file blocks. + * * The first full backup meant for incremental and future incremental backups must pass * 'incrementalBackup' as true. * 'thisBackupName' must exist only if 'incrementalBackup' is true. @@ -294,10 +305,7 @@ public: * when 'incrementalBackup' is true. */ virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) = 0; + OperationContext* opCtx, const BackupOptions& options) = 0; virtual void endNonBlockingBackup(OperationContext* opCtx) { return; diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 0ec1f6f3392..49a28591f93 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -652,11 +652,8 @@ Status StorageEngineImpl::disableIncrementalBackup(OperationContext* opCtx) { } StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) { - return _engine->beginNonBlockingBackup(opCtx, incrementalBackup, thisBackupName, srcBackupName); + OperationContext* opCtx, const StorageEngine::BackupOptions& options) { + return _engine->beginNonBlockingBackup(opCtx, options); } void StorageEngineImpl::endNonBlockingBackup(OperationContext* opCtx) { diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h index 53a807ae04c..c31aa8aa2ab 100644 --- a/src/mongo/db/storage/storage_engine_impl.h +++ b/src/mongo/db/storage/storage_engine_impl.h @@ -100,10 +100,7 @@ public: virtual Status disableIncrementalBackup(OperationContext* opCtx) override; virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) override; + OperationContext* opCtx, const BackupOptions& options) override; virtual void endNonBlockingBackup(OperationContext* opCtx) override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index e08cd46dca9..c71795a5d35 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1068,20 +1068,18 @@ Status WiredTigerKVEngine::disableIncrementalBackup(OperationContext* opCtx) { } StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) { + OperationContext* opCtx, const StorageEngine::BackupOptions& options) { uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral()); std::stringstream ss; - if (incrementalBackup) { - invariant(thisBackupName); + if (options.incrementalBackup) { + invariant(options.thisBackupName); ss << "incremental=(enabled=true,force_stop=false,"; - ss << "this_id=" << std::quoted(str::escape(*thisBackupName)) << ","; + ss << "granularity=" << options.blockSizeMB << "MB,"; + ss << "this_id=" << std::quoted(str::escape(*options.thisBackupName)) << ","; - if (srcBackupName) { - ss << "src_id=" << std::quoted(str::escape(*srcBackupName)) << ","; + if (options.srcBackupName) { + ss << "src_id=" << std::quoted(str::escape(*options.srcBackupName)) << ","; } ss << ")"; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 81a1cead276..a260b2a2684 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -177,10 +177,7 @@ public: Status disableIncrementalBackup(OperationContext* opCtx) override; StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( - OperationContext* opCtx, - bool incrementalBackup, - boost::optional<std::string> thisBackupName, - boost::optional<std::string> srcBackupName) override; + OperationContext* opCtx, const StorageEngine::BackupOptions& options) override; void endNonBlockingBackup(OperationContext* opCtx) override; |