diff options
author | Matt Kneiser <matt.kneiser@mongodb.com> | 2022-02-02 21:33:54 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-02-11 06:17:14 +0000 |
commit | 98171115a000d2a9174a9289f4db5713c1c6d162 (patch) | |
tree | d9b8f5c5d5208e7370ef8cae0e06a3e3cf05fdc0 /src/mongo/db/storage | |
parent | 6fe9f677add123bfb1a0fb62f3d95f9fd61794f7 (diff) | |
download | mongo-98171115a000d2a9174a9289f4db5713c1c6d162.tar.gz |
SERVER-62427 Return namespace with backupCursor
Diffstat (limited to 'src/mongo/db/storage')
-rw-r--r-- | src/mongo/db/storage/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/backup_block.cpp | 52 | ||||
-rw-r--r-- | src/mongo/db/storage/backup_block.h | 37 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/SConscript | 3 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.cpp | 3 | ||||
-rw-r--r-- | src/mongo/db/storage/devnull/devnull_kv_engine.h | 6 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog.h | 4 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog_impl.cpp | 21 | ||||
-rw-r--r-- | src/mongo/db/storage/durable_catalog_impl.h | 2 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/SConscript | 1 | ||||
-rw-r--r-- | src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp | 13 |
11 files changed, 123 insertions, 20 deletions
diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index b9fd8b0ed9b..859f2369a4d 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -563,6 +563,7 @@ env.Library( ], LIBDEPS_PRIVATE=[ '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/namespace_string', 'storage_options', ], ) diff --git a/src/mongo/db/storage/backup_block.cpp b/src/mongo/db/storage/backup_block.cpp index b872d203da2..78385dccece 100644 --- a/src/mongo/db/storage/backup_block.cpp +++ b/src/mongo/db/storage/backup_block.cpp @@ -27,11 +27,13 @@ * it in the license file. */ +#include "mongo/db/storage/backup_block.h" + #include <boost/filesystem.hpp> #include <set> #include "mongo/base/string_data.h" -#include "mongo/db/storage/backup_block.h" +#include "mongo/db/storage/durable_catalog.h" #include "mongo/db/storage/storage_options.h" namespace mongo { @@ -45,9 +47,20 @@ const std::set<std::string> kRequiredMDBFiles = {"_mdb_catalog.wt", "sizeStorer. } // namespace +BackupBlock::BackupBlock(OperationContext* opCtx, + std::string filePath, + std::uint64_t offset, + std::uint64_t length, + std::uint64_t fileSize) + : _filePath(filePath), _offset(offset), _length(length), _fileSize(fileSize) { + boost::filesystem::path path(filePath); + _filenameStem = path.stem().string(); + _setNamespaceString(opCtx); +} + bool BackupBlock::isRequired() const { // Extract the filename from the path. - boost::filesystem::path path(_filename); + boost::filesystem::path path(_filePath); const std::string filename = path.filename().string(); // Check whether this is a required WiredTiger file. @@ -72,11 +85,38 @@ bool BackupBlock::isRequired() const { return true; } - // TODO SERVER-62427: mark the following namespaces as required: - // - Any collection residing in an internal database (admin, local or config). - // - Each databases 'system.views' collection. - // - Collections with table logging enabled. See WiredTigerUtil::useTableLogging(). + // Check if collection resides in an internal database (admin, local, or config). + if (_nss.isOnInternalDb()) { + return true; + } + + // Check if collection is 'system.views'. + if (_nss.isSystemDotViews()) { + return true; + } + return false; } +void BackupBlock::_setNamespaceString(OperationContext* opCtx) { + if (!opCtx) { + return; + } + + DurableCatalog* catalog = DurableCatalog::get(opCtx); + std::vector<DurableCatalog::Entry> catalogEntries = catalog->getAllCatalogEntries(opCtx); + for (const DurableCatalog::Entry& e : catalogEntries) { + if (StringData(_filenameStem).startsWith("index-"_sd) && + catalog->isIndexInEntry(opCtx, e.catalogId, _filenameStem)) { + _nss = e.tenantNs.getNss(); + return; + } + + if (e.ident == _filenameStem) { + _nss = e.tenantNs.getNss(); + return; + } + } +} + } // namespace mongo diff --git a/src/mongo/db/storage/backup_block.h b/src/mongo/db/storage/backup_block.h index 9578fd19a39..d084c981d34 100644 --- a/src/mongo/db/storage/backup_block.h +++ b/src/mongo/db/storage/backup_block.h @@ -29,8 +29,12 @@ #pragma once +#include <boost/filesystem.hpp> #include <string> +#include "mongo/db/namespace_string.h" +#include "mongo/db/operation_context.h" + namespace mongo { /** @@ -47,16 +51,26 @@ namespace mongo { */ class BackupBlock final { public: - explicit BackupBlock(std::string filename, + explicit BackupBlock(OperationContext* opCtx, + std::string filePath, std::uint64_t offset = 0, std::uint64_t length = 0, - std::uint64_t fileSize = 0) - : _filename(filename), _offset(offset), _length(length), _fileSize(fileSize) {} + std::uint64_t fileSize = 0); ~BackupBlock() = default; - std::string filename() const { - return _filename; + std::string filePath() const { + return _filePath; + } + + std::string ns() const { + // Remove "system.buckets." from time-series collection namespaces since it is an + // internal detail that is not intended to be visible externally. + if (_nss.isTimeseriesBucketsCollection()) { + return _nss.getTimeseriesViewNamespace().toString(); + } + + return _nss.toString(); } std::uint64_t offset() const { @@ -77,9 +91,20 @@ public: bool isRequired() const; private: - const std::string _filename; + /** + * Sets '_nss' for: + * - collections + * - indexes, to the NSS of their respective collection + * A null opCtx is ignored. A null opCtx is exercised by FCBIS unit tests. + */ + void _setNamespaceString(OperationContext* opCtx); + + const std::string _filePath; const std::uint64_t _offset; const std::uint64_t _length; const std::uint64_t _fileSize; + + std::string _filenameStem; + NamespaceString _nss; }; } // namespace mongo diff --git a/src/mongo/db/storage/devnull/SConscript b/src/mongo/db/storage/devnull/SConscript index b13b49a5f09..c9a4cee0b75 100644 --- a/src/mongo/db/storage/devnull/SConscript +++ b/src/mongo/db/storage/devnull/SConscript @@ -11,7 +11,10 @@ env.Library( 'ephemeral_catalog_record_store.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/base', + '$BUILD_DIR/mongo/db/namespace_string', '$BUILD_DIR/mongo/db/record_id_helpers', + '$BUILD_DIR/mongo/db/storage/backup_block', '$BUILD_DIR/mongo/db/storage/record_store_base', '$BUILD_DIR/mongo/db/storage/recovery_unit_base', ], diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp index 93b2c958d02..88cd7211441 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.cpp +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.cpp @@ -246,6 +246,9 @@ public: } }; +DevNullKVEngine::DevNullKVEngine() { + _mockBackupBlocks.push_back(BackupBlock(/*opCtx=*/nullptr, "filename.wt")); +} std::unique_ptr<RecordStore> DevNullKVEngine::getRecordStore(OperationContext* opCtx, StringData ns, diff --git a/src/mongo/db/storage/devnull/devnull_kv_engine.h b/src/mongo/db/storage/devnull/devnull_kv_engine.h index 897b10e1420..17e82e5cc0a 100644 --- a/src/mongo/db/storage/devnull/devnull_kv_engine.h +++ b/src/mongo/db/storage/devnull/devnull_kv_engine.h @@ -44,6 +44,8 @@ class JournalListener; */ class DevNullKVEngine : public KVEngine { public: + DevNullKVEngine(); + virtual ~DevNullKVEngine() {} virtual RecoveryUnit* newRecoveryUnit() { @@ -172,9 +174,7 @@ public: private: std::shared_ptr<void> _catalogInfo; - int _cachePressureForTest; - - std::vector<BackupBlock> _mockBackupBlocks = {BackupBlock("filename.wt")}; + std::vector<BackupBlock> _mockBackupBlocks; }; } // namespace mongo diff --git a/src/mongo/db/storage/durable_catalog.h b/src/mongo/db/storage/durable_catalog.h index 1f3c25e145d..4a5c4e893c7 100644 --- a/src/mongo/db/storage/durable_catalog.h +++ b/src/mongo/db/storage/durable_catalog.h @@ -83,6 +83,10 @@ public: virtual std::vector<std::string> getIndexIdents(OperationContext* opCtx, RecordId id) const = 0; + virtual bool isIndexInEntry(OperationContext* opCtx, + RecordId id, + StringData idxIdent) const = 0; + virtual BSONObj getCatalogEntry(OperationContext* opCtx, RecordId catalogId) const = 0; virtual std::shared_ptr<BSONCollectionCatalogEntry::MetaData> getMetaData( diff --git a/src/mongo/db/storage/durable_catalog_impl.cpp b/src/mongo/db/storage/durable_catalog_impl.cpp index 8594880f8e2..020374a67c5 100644 --- a/src/mongo/db/storage/durable_catalog_impl.cpp +++ b/src/mongo/db/storage/durable_catalog_impl.cpp @@ -423,6 +423,27 @@ std::vector<std::string> DurableCatalogImpl::getIndexIdents(OperationContext* op return idents; } +bool DurableCatalogImpl::isIndexInEntry(OperationContext* opCtx, + RecordId catalogId, + StringData idxIdent) const { + BSONObj obj = _findEntry(opCtx, catalogId); + if (obj["idxIdent"].eoo()) { + return false; + } + + BSONObj idxIdentObj = obj["idxIdent"].Obj(); + + BSONObjIterator it(idxIdentObj); + while (it.more()) { + BSONElement elem = it.next(); + if (elem.valueStringData() == idxIdent) { + return true; + } + } + + return false; +} + BSONObj DurableCatalogImpl::_findEntry(OperationContext* opCtx, RecordId catalogId) const { LOGV2_DEBUG(22208, 3, "looking up metadata for: {catalogId}", "catalogId"_attr = catalogId); RecordData data; diff --git a/src/mongo/db/storage/durable_catalog_impl.h b/src/mongo/db/storage/durable_catalog_impl.h index 63732d5846f..19a0acf0cd1 100644 --- a/src/mongo/db/storage/durable_catalog_impl.h +++ b/src/mongo/db/storage/durable_catalog_impl.h @@ -74,6 +74,8 @@ public: std::vector<std::string> getIndexIdents(OperationContext* opCtx, RecordId catalogId) const; + bool isIndexInEntry(OperationContext* opCtx, RecordId catalogId, StringData idxIdent) const; + BSONObj getCatalogEntry(OperationContext* opCtx, RecordId catalogId) const { return _findEntry(opCtx, catalogId); } diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript index 44467981934..4f10c052cc9 100644 --- a/src/mongo/db/storage/wiredtiger/SConscript +++ b/src/mongo/db/storage/wiredtiger/SConscript @@ -63,6 +63,7 @@ wtEnv.Library( '$BUILD_DIR/mongo/db/repl/repl_settings', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/service_context', + '$BUILD_DIR/mongo/db/storage/backup_block', '$BUILD_DIR/mongo/db/storage/index_entry_comparison', '$BUILD_DIR/mongo/db/storage/key_string', '$BUILD_DIR/mongo/db/storage/record_store_base', diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp index 20d3e03241f..6be8becd96f 100644 --- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp +++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp @@ -1082,11 +1082,13 @@ std::vector<std::string> getUniqueFiles(const std::vector<std::string>& files, class StreamingCursorImpl : public StorageEngine::StreamingCursor { public: StreamingCursorImpl() = delete; - explicit StreamingCursorImpl(WT_SESSION* session, + explicit StreamingCursorImpl(OperationContext* opCtx, + WT_SESSION* session, std::string path, StorageEngine::BackupOptions options, WiredTigerBackup* wtBackup) : StorageEngine::StreamingCursor(options), + _opCtx(opCtx), _session(session), _path(path), _wtBackup(wtBackup){}; @@ -1154,7 +1156,7 @@ public: // are the initial incremental backup. const std::uint64_t length = options.incrementalBackup ? fileSize : 0; backupBlocks.push_back( - BackupBlock(filePath.string(), 0 /* offset */, length, fileSize)); + BackupBlock(_opCtx, filePath.string(), 0 /* offset */, length, fileSize)); } } @@ -1207,14 +1209,14 @@ private: "offset"_attr = offset, "size"_attr = size, "type"_attr = type); - backupBlocks->push_back(BackupBlock(filePath.string(), offset, size, fileSize)); + backupBlocks->push_back(BackupBlock(_opCtx, filePath.string(), offset, size, fileSize)); } // If the file is unchanged, push a BackupBlock with offset=0 and length=0. This allows us // to distinguish between an unchanged file and a deleted file in an incremental backup. if (fileUnchangedFlag) { backupBlocks->push_back( - BackupBlock(filePath.string(), 0 /* offset */, 0 /* length */, fileSize)); + BackupBlock(_opCtx, filePath.string(), 0 /* offset */, 0 /* length */, fileSize)); } // If the duplicate backup cursor has been exhausted, close it and set @@ -1231,6 +1233,7 @@ private: return Status::OK(); } + OperationContext* _opCtx; WT_SESSION* _session; std::string _path; WiredTigerBackup* _wtBackup; // '_wtBackup' is an out parameter. @@ -1286,7 +1289,7 @@ WiredTigerKVEngine::beginNonBlockingBackup(OperationContext* opCtx, invariant(_wtBackup.logFilePathsSeenByExtendBackupCursor.empty()); invariant(_wtBackup.logFilePathsSeenByGetNextBatch.empty()); auto streamingCursor = - std::make_unique<StreamingCursorImpl>(session, _path, options, &_wtBackup); + std::make_unique<StreamingCursorImpl>(opCtx, session, _path, options, &_wtBackup); pinOplogGuard.dismiss(); _backupSession = std::move(sessionRaii); |