diff options
-rw-r--r-- | src/mongo/db/catalog/catalog_control_test.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/backup_cursor_state.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.cpp | 8 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/kv/kv_engine.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine.h | 23 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine_impl.cpp | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/storage_engine_impl.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 95 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h | 2 |
10 files changed, 98 insertions, 42 deletions
diff --git a/src/mongo/db/catalog/catalog_control_test.cpp b/src/mongo/db/catalog/catalog_control_test.cpp index 7b85e350cfa..6b15acb840f 100644 --- a/src/mongo/db/catalog/catalog_control_test.cpp +++ b/src/mongo/db/catalog/catalog_control_test.cpp @@ -90,7 +90,7 @@ public: return Status(ErrorCodes::CommandNotSupported, "The current storage engine doesn't support backup mode"); } - StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( + StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) final { return Status(ErrorCodes::CommandNotSupported, "The current storage engine doesn't support backup mode"); diff --git a/src/mongo/db/storage/backup_cursor_state.h b/src/mongo/db/storage/backup_cursor_state.h index d0af9c45b2b..814e81c4765 100644 --- a/src/mongo/db/storage/backup_cursor_state.h +++ b/src/mongo/db/storage/backup_cursor_state.h @@ -41,7 +41,7 @@ namespace mongo { struct BackupCursorState { UUID backupId; boost::optional<Document> preamble; - std::vector<StorageEngine::BackupBlock> blocksToCopy; + StorageEngine::BackupInformation backupInformation; }; struct BackupCursorExtendState { diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index f1982f314d8..b73bb4a9d4d 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -253,10 +253,12 @@ void DevNullKVEngine::setCachePressureForTest(int pressure) { _cachePressureForTest = pressure; } -StatusWith<std::vector<StorageEngine::BackupBlock>> DevNullKVEngine::beginNonBlockingBackup( +StatusWith<StorageEngine::BackupInformation> DevNullKVEngine::beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) { - std::vector<StorageEngine::BackupBlock> blocksToCopy = {{"filename.wt", 0, 0}}; - return blocksToCopy; + StorageEngine::BackupInformation backupInformation; + StorageEngine::BackupFile backupFile(0); + backupInformation.insert({"filename.wt", backupFile}); + return backupInformation; } StatusWith<std::vector<std::string>> DevNullKVEngine::extendBackupCursor(OperationContext* opCtx) { diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index 67989a4774f..3f73b4e4ade 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -139,7 +139,7 @@ public: virtual void endBackup(OperationContext* opCtx) {} - virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( + virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) override; virtual void endNonBlockingBackup(OperationContext* opCtx) override {} diff --git a/src/mongo/db/storage/kv/kv_engine.h b/src/mongo/db/storage/kv/kv_engine.h index 337a40d13b9..7c3316cb6a5 100644 --- a/src/mongo/db/storage/kv/kv_engine.h +++ b/src/mongo/db/storage/kv/kv_engine.h @@ -237,7 +237,7 @@ public: MONGO_UNREACHABLE; } - virtual StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( + virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) { return Status(ErrorCodes::CommandNotSupported, "The current storage engine doesn't support backup mode"); diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 5f609793bfa..45e2c444228 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -78,12 +78,27 @@ public: }; struct BackupBlock { - std::string filename; - std::uint64_t offset; - std::uint64_t length; + std::uint64_t offset = 0; + std::uint64_t length = 0; }; /** + * Contains the size of the file to be backed up. This allows the backup application to safely + * truncate the file for incremental backups. Files that have had changes since the last + * incremental backup will have their changed file blocks listed. + */ + struct BackupFile { + BackupFile() = delete; + explicit BackupFile(std::uint64_t fileSize) : fileSize(fileSize){}; + + std::uint64_t fileSize; + std::vector<BackupBlock> blocksToCopy; + }; + + // Map of filenames to backup file information. + using BackupInformation = stdx::unordered_map<std::string, BackupFile>; + + /** * The interface for creating new instances of storage engines. * * A storage engine provides an instance of this class (along with an associated @@ -331,7 +346,7 @@ public: * 'srcBackupName' must not exist when 'incrementalBackup' is false but may or may not exist * when 'incrementalBackup' is true. */ - virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup( + virtual StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const BackupOptions& options) = 0; virtual void endNonBlockingBackup(OperationContext* opCtx) = 0; diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index dbe69146dd3..15dc613ebe3 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -670,7 +670,7 @@ Status StorageEngineImpl::disableIncrementalBackup(OperationContext* opCtx) { return _engine->disableIncrementalBackup(opCtx); } -StatusWith<std::vector<StorageEngine::BackupBlock>> StorageEngineImpl::beginNonBlockingBackup( +StatusWith<StorageEngine::BackupInformation> StorageEngineImpl::beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) { return _engine->beginNonBlockingBackup(opCtx, options); } diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h index 44c08054f1d..f34c5bbac29 100644 --- a/src/mongo/db/storage/storage_engine_impl.h +++ b/src/mongo/db/storage/storage_engine_impl.h @@ -99,7 +99,7 @@ public: virtual Status disableIncrementalBackup(OperationContext* opCtx) override; - virtual StatusWith<std::vector<BackupBlock>> beginNonBlockingBackup( + virtual StatusWith<BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const BackupOptions& options) override; virtual void endNonBlockingBackup(OperationContext* opCtx) override; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index f85299d0422..1aa462b67a8 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -596,10 +596,14 @@ Status OpenReadTransactionParam::setFromString(const std::string& str) { namespace { -StatusWith<std::vector<StorageEngine::BackupBlock>> getDataBlocksFromBackupCursor( - WT_CURSOR* cursor, std::string dbPath, const char* statusPrefix) { +StatusWith<StorageEngine::BackupInformation> getBackupInformationFromBackupCursor( + WT_SESSION* session, + WT_CURSOR* cursor, + bool incrementalBackup, + std::string dbPath, + const char* statusPrefix) { int wtRet; - std::vector<StorageEngine::BackupBlock> blocks; + StorageEngine::BackupInformation backupInformation; const char* filename; const auto directoryPath = boost::filesystem::path(dbPath); const auto wiredTigerLogFilePrefix = "WiredTigerLog"; @@ -608,26 +612,57 @@ StatusWith<std::vector<StorageEngine::BackupBlock>> getDataBlocksFromBackupCurso std::string name(filename); - auto filePath = directoryPath; + boost::filesystem::path filePath = directoryPath; + boost::filesystem::path relativePath; if (name.find(wiredTigerLogFilePrefix) == 0) { // TODO SERVER-13455:replace `journal/` with the configurable journal path. filePath /= boost::filesystem::path("journal"); + relativePath /= boost::filesystem::path("journal"); } filePath /= name; + relativePath /= name; - boost::system::error_code errorCode; - const std::uint64_t filesize = boost::filesystem::file_size(filePath, errorCode); - uassert(31317, - "Failed to get a file's size. Filename: {} Error: {}"_format(filePath.string(), - errorCode.message()), - !errorCode); - blocks.push_back({filePath.string(), 0, filesize}); + // TODO: SERVER-44410 Implement fileSize. + std::uint64_t fileSize = 0; + StorageEngine::BackupFile backupFile(fileSize); + backupInformation.insert({filePath.string(), backupFile}); + + if (!incrementalBackup) { + continue; + } + + // For each file listed, open a duplicate backup cursor and get the blocks to copy. + std::stringstream ss; + ss << "incremental=(file=\"" << str::escape(relativePath.string()) << "\")"; + const std::string config = ss.str(); + WT_CURSOR* dupCursor; + wtRet = session->open_cursor(session, nullptr, cursor, config.c_str(), &dupCursor); + if (wtRet != 0) { + return wtRCToStatus(wtRet); + } + + while ((wtRet = dupCursor->next(dupCursor)) == 0) { + uint64_t offset, size, type; + invariantWTOK(dupCursor->get_key(dupCursor, &offset, &size, &type)); + LOG(2) << "Block to copy for incremental backup: filename: " << filePath.string() + << ", offset: " << offset << ", size: " << size << ", type: " << type; + backupInformation.at(filePath.string()).blocksToCopy.push_back({offset, size}); + } + + if (wtRet != WT_NOTFOUND) { + return wtRCToStatus(wtRet); + } + + wtRet = dupCursor->close(dupCursor); + if (wtRet != 0) { + return wtRCToStatus(wtRet); + } } if (wtRet != WT_NOTFOUND) { return wtRCToStatus(wtRet, statusPrefix); } - return blocks; + return backupInformation; } } // namespace @@ -1167,7 +1202,7 @@ Status WiredTigerKVEngine::disableIncrementalBackup(OperationContext* opCtx) { return Status::OK(); } -StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNonBlockingBackup( +StatusWith<StorageEngine::BackupInformation> WiredTigerKVEngine::beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) { uassert(51034, "Cannot open backup cursor with in-memory mode.", !isEphemeral()); @@ -1205,18 +1240,18 @@ StatusWith<std::vector<StorageEngine::BackupBlock>> WiredTigerKVEngine::beginNon return wtRCToStatus(wtRet); } - auto swBlocksToCopy = - getDataBlocksFromBackupCursor(cursor, _path, "Error opening backup cursor."); + auto swBackupInfo = getBackupInformationFromBackupCursor( + session, cursor, options.incrementalBackup, _path, "Error opening backup cursor."); - if (!swBlocksToCopy.isOK()) { - return swBlocksToCopy; + if (!swBackupInfo.isOK()) { + return swBackupInfo; } pinOplogGuard.dismiss(); _backupSession = std::move(sessionRaii); _backupCursor = cursor; - return swBlocksToCopy; + return swBackupInfo; } void WiredTigerKVEngine::endNonBlockingBackup(OperationContext* opCtx) { @@ -1232,6 +1267,8 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::extendBackupCursor( uassert(51033, "Cannot extend backup cursor with in-memory mode.", !isEphemeral()); invariant(_backupCursor); + // The "target=(\"log:\")" configuration string for the cursor will ensure that we only see the + // log files when iterating on the cursor. WT_CURSOR* cursor = nullptr; WT_SESSION* session = _backupSession->getSession(); int wtRet = session->open_cursor(session, nullptr, _backupCursor, "target=(\"log:\")", &cursor); @@ -1239,25 +1276,27 @@ StatusWith<std::vector<std::string>> WiredTigerKVEngine::extendBackupCursor( return wtRCToStatus(wtRet); } - StatusWith<std::vector<StorageEngine::BackupBlock>> swBlocksToCopy = - getDataBlocksFromBackupCursor(cursor, _path, "Error extending backup cursor."); + StatusWith<StorageEngine::BackupInformation> swBackupInfo = + getBackupInformationFromBackupCursor( + session, cursor, /*incrementalBackup=*/false, _path, "Error extending backup cursor."); wtRet = cursor->close(cursor); if (wtRet != 0) { return wtRCToStatus(wtRet); } - if (!swBlocksToCopy.isOK()) { - return swBlocksToCopy.getStatus(); + if (!swBackupInfo.isOK()) { + return swBackupInfo.getStatus(); } - // Journal files returned from the future backup cursor are not intended to have block level - // information. For now, this is being explicitly codified by transforming the result into a - // vector of filename strings. The lack of block/filesize information instructs the client to - // copy the whole file. + // Once all the backup cursors have been opened on a sharded cluster, we need to ensure that the + // data being copied from each shard is at the same point-in-time across the entire cluster to + // have a consistent view of the data. For shards that opened their backup cursor before the + // established point-in-time for backup, they will need to create a full copy of the additional + // journal files returned by this method to ensure a consistent backup of the data is taken. std::vector<std::string> filenames; - for (const auto& block : swBlocksToCopy.getValue()) { - filenames.push_back(std::move(block.filename)); + for (const auto& entry : swBackupInfo.getValue()) { + filenames.push_back(entry.first); } return {filenames}; diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h index 4dccf63f627..40295cb4670 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.h @@ -181,7 +181,7 @@ public: Status disableIncrementalBackup(OperationContext* opCtx) override; - StatusWith<std::vector<StorageEngine::BackupBlock>> beginNonBlockingBackup( + StatusWith<StorageEngine::BackupInformation> beginNonBlockingBackup( OperationContext* opCtx, const StorageEngine::BackupOptions& options) override; void endNonBlockingBackup(OperationContext* opCtx) override; |