From 1b18cd9b5dcb352c6fdd6b15d4bd0a40c0da35c3 Mon Sep 17 00:00:00 2001 From: Sophia Tan Date: Fri, 28 Jan 2022 17:25:40 +0000 Subject: SERVER-63058 Change StorageEngineImpl::listDatabases() to return a list of TenantDatabaseName instead of string --- src/mongo/db/SConscript | 1 + src/mongo/db/catalog/catalog_control.cpp | 20 ++++++++++-------- .../db/commands/feature_compatibility_version.cpp | 17 +++++++-------- src/mongo/db/commands/list_databases.cpp | 20 +++++++++--------- src/mongo/db/repl/repl_set_commands.cpp | 8 ++++---- ...replication_coordinator_external_state_impl.cpp | 13 +++++++----- src/mongo/db/repl/rollback_impl.cpp | 21 ++++++++++--------- src/mongo/db/repl/storage_interface_impl.cpp | 18 ++++++++-------- src/mongo/db/startup_recovery.cpp | 24 ++++++++++++---------- src/mongo/db/storage/SConscript | 3 ++- src/mongo/db/storage/storage_engine.h | 3 ++- src/mongo/db/storage/storage_engine_impl.cpp | 11 ++++++++-- src/mongo/db/storage/storage_engine_impl.h | 2 +- src/mongo/db/storage/storage_engine_mock.h | 2 +- 14 files changed, 91 insertions(+), 72 deletions(-) (limited to 'src/mongo/db') diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index cc673aa8f03..d4a0114c395 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -1465,6 +1465,7 @@ env.Library( 'index_builds_coordinator_interface', 'rebuild_indexes', 'repair', + 'server_feature_flags', ], ) diff --git a/src/mongo/db/catalog/catalog_control.cpp b/src/mongo/db/catalog/catalog_control.cpp index 10bf24a5515..4e73183ca3a 100644 --- a/src/mongo/db/catalog/catalog_control.cpp +++ b/src/mongo/db/catalog/catalog_control.cpp @@ -60,13 +60,13 @@ void reopenAllDatabasesAndReloadCollectionCatalog( catalogBatchWriter.emplace(opCtx); auto databaseHolder = DatabaseHolder::get(opCtx); - std::vector databasesToOpen = storageEngine->listDatabases(); - for (auto&& dbName : databasesToOpen) { + std::vector databasesToOpen = storageEngine->listDatabases(); + for (auto&& tenantDbName : databasesToOpen) { LOGV2_FOR_RECOVERY( - 23992, 1, "openCatalog: dbholder reopening database", "db"_attr = dbName); - auto db = databaseHolder->openDb(opCtx, dbName); - invariant(db, str::stream() << "failed to reopen database " << dbName); - for (auto&& collNss : catalog->getAllCollectionNamesFromDb(opCtx, dbName)) { + 23992, 1, "openCatalog: dbholder reopening database", "db"_attr = tenantDbName); + auto db = databaseHolder->openDb(opCtx, tenantDbName.dbName()); + invariant(db, str::stream() << "failed to reopen database " << tenantDbName.toString()); + for (auto&& collNss : catalog->getAllCollectionNamesFromDb(opCtx, tenantDbName.dbName())) { // Note that the collection name already includes the database component. auto collection = catalog->lookupCollectionByNamespaceForMetadataWrite( opCtx, CollectionCatalog::LifetimeMode::kInplace, collNss); @@ -118,13 +118,15 @@ MinVisibleTimestampMap closeCatalog(OperationContext* opCtx) { IndexBuildsCoordinator::get(opCtx)->assertNoIndexBuildInProgress(); MinVisibleTimestampMap minVisibleTimestampMap; - std::vector allDbs = + std::vector allDbs = opCtx->getServiceContext()->getStorageEngine()->listDatabases(); auto databaseHolder = DatabaseHolder::get(opCtx); auto catalog = CollectionCatalog::get(opCtx); - for (auto&& dbName : allDbs) { - for (auto collIt = catalog->begin(opCtx, dbName); collIt != catalog->end(opCtx); ++collIt) { + for (auto&& tenantDbName : allDbs) { + for (auto collIt = catalog->begin(opCtx, tenantDbName.dbName()); + collIt != catalog->end(opCtx); + ++collIt) { auto coll = *collIt; if (!coll) { break; diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp index f641012f28d..09751b99db6 100644 --- a/src/mongo/db/commands/feature_compatibility_version.cpp +++ b/src/mongo/db/commands/feature_compatibility_version.cpp @@ -379,11 +379,11 @@ void FeatureCompatibilityVersion::setIfCleanStartup(OperationContext* opCtx, bool FeatureCompatibilityVersion::hasNoReplicatedCollections(OperationContext* opCtx) { StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); - std::vector dbNames = storageEngine->listDatabases(); + std::vector tenantDbNames = storageEngine->listDatabases(); auto catalog = CollectionCatalog::get(opCtx); - for (auto&& dbName : dbNames) { - Lock::DBLock dbLock(opCtx, dbName, MODE_S); - for (auto&& collNss : catalog->getAllCollectionNamesFromDb(opCtx, dbName)) { + for (auto&& tenantDbName : tenantDbNames) { + Lock::DBLock dbLock(opCtx, tenantDbName.dbName(), MODE_S); + for (auto&& collNss : catalog->getAllCollectionNamesFromDb(opCtx, tenantDbName.dbName())) { if (collNss.isReplicated()) { return false; } @@ -482,10 +482,11 @@ void FeatureCompatibilityVersion::fassertInitializedAfterStartup(OperationContex auto fcvDocument = findFeatureCompatibilityVersionDocument(opCtx); auto const storageEngine = opCtx->getServiceContext()->getStorageEngine(); - auto dbNames = storageEngine->listDatabases(); - bool nonLocalDatabases = std::any_of(dbNames.begin(), dbNames.end(), [](auto name) { - return name != NamespaceString::kLocalDb; - }); + auto tenantDbNames = storageEngine->listDatabases(); + bool nonLocalDatabases = + std::any_of(tenantDbNames.begin(), tenantDbNames.end(), [](auto tenantDbName) { + return tenantDbName.dbName() != NamespaceString::kLocalDb; + }); // Fail to start up if there is no featureCompatibilityVersion document and there are non-local // databases present. diff --git a/src/mongo/db/commands/list_databases.cpp b/src/mongo/db/commands/list_databases.cpp index 8196241368f..6111555e8ce 100644 --- a/src/mongo/db/commands/list_databases.cpp +++ b/src/mongo/db/commands/list_databases.cpp @@ -122,13 +122,13 @@ public: filter = std::move(matcher); } - std::vector dbNames; + std::vector tenantDbNames; StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); { Lock::GlobalLock lk(opCtx, MODE_IS); CurOpFailpointHelpers::waitWhileFailPointEnabled( &hangBeforeListDatabases, opCtx, "hangBeforeListDatabases", []() {}); - dbNames = storageEngine->listDatabases(); + tenantDbNames = storageEngine->listDatabases(); } std::vector items; @@ -137,14 +137,14 @@ public: filter->getCategory() == MatchExpression::MatchCategory::kLeaf && filter->path() == kName; long long totalSize = 0; - for (const auto& itemName : dbNames) { + for (const auto& tenantDbName : tenantDbNames) { if (authorizedDatabases && - !as->isAuthorizedForAnyActionOnAnyResourceInDB(itemName)) { + !as->isAuthorizedForAnyActionOnAnyResourceInDB(tenantDbName.dbName())) { // We don't have listDatabases on the cluster or find on this database. continue; } - ListDatabasesReplyItem item(itemName); + ListDatabasesReplyItem item(tenantDbName.dbName()); long long size = 0; if (!nameOnly) { @@ -153,18 +153,18 @@ public: continue; } - AutoGetDbForReadMaybeLockFree lockFreeReadBlock(opCtx, itemName); + AutoGetDbForReadMaybeLockFree lockFreeReadBlock(opCtx, tenantDbName.dbName()); // The database could have been dropped since we called 'listDatabases()' above. - if (!DatabaseHolder::get(opCtx)->dbExists(opCtx, itemName)) { + if (!DatabaseHolder::get(opCtx)->dbExists(opCtx, tenantDbName.dbName())) { continue; } - writeConflictRetry(opCtx, "sizeOnDisk", itemName, [&] { - size = storageEngine->sizeOnDiskForDb(opCtx, itemName); + writeConflictRetry(opCtx, "sizeOnDisk", tenantDbName.dbName(), [&] { + size = storageEngine->sizeOnDiskForDb(opCtx, tenantDbName.dbName()); }); item.setSizeOnDisk(size); item.setEmpty(CollectionCatalog::get(opCtx) - ->getAllCollectionUUIDsFromDb(itemName) + ->getAllCollectionUUIDsFromDb(tenantDbName.dbName()) .empty()); } if (!filter || filter->matchesBSON(item.toBSON())) { diff --git a/src/mongo/db/repl/repl_set_commands.cpp b/src/mongo/db/repl/repl_set_commands.cpp index fcaedea011c..0d5eeced642 100644 --- a/src/mongo/db/repl/repl_set_commands.cpp +++ b/src/mongo/db/repl/repl_set_commands.cpp @@ -704,12 +704,12 @@ namespace { */ bool replHasDatabases(OperationContext* opCtx) { StorageEngine* storageEngine = getGlobalServiceContext()->getStorageEngine(); - std::vector names = storageEngine->listDatabases(); + std::vector tenantDbNames = storageEngine->listDatabases(); - if (names.size() >= 2) + if (tenantDbNames.size() >= 2) return true; - if (names.size() == 1) { - if (names[0] != "local") + if (tenantDbNames.size() == 1) { + if (tenantDbNames[0].dbName() != "local") return true; // we have a local database. return true if oplog isn't empty diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp index d2ea082a4cd..b3faa85859f 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp @@ -999,20 +999,23 @@ void ReplicationCoordinatorExternalStateImpl::_dropAllTempCollections(OperationC Lock::GlobalLock lk(opCtx, MODE_IS); StorageEngine* storageEngine = _service->getStorageEngine(); - std::vector dbNames = storageEngine->listDatabases(); + std::vector tenantDbNames = storageEngine->listDatabases(); - for (std::vector::iterator it = dbNames.begin(); it != dbNames.end(); ++it) { + for (std::vector::iterator it = tenantDbNames.begin(); + it != tenantDbNames.end(); + ++it) { // The local db is special because it isn't replicated. It is cleared at startup even on // replica set members. - if (*it == "local") + if (it->dbName() == "local") continue; LOGV2_DEBUG(21309, 2, "Removing temporary collections from {db}", "Removing temporary collections", "db"_attr = *it); - AutoGetDb autoDb(opCtx, *it, MODE_IX); - invariant(autoDb.getDb(), str::stream() << "Unable to get reference to database " << *it); + AutoGetDb autoDb(opCtx, it->dbName(), MODE_IX); + invariant(autoDb.getDb(), + str::stream() << "Unable to get reference to database " << it->dbName()); autoDb.getDb()->clearTmpCollections(opCtx); } } diff --git a/src/mongo/db/repl/rollback_impl.cpp b/src/mongo/db/repl/rollback_impl.cpp index b4dad5b54ba..cc543b5b5ce 100644 --- a/src/mongo/db/repl/rollback_impl.cpp +++ b/src/mongo/db/repl/rollback_impl.cpp @@ -355,7 +355,7 @@ void RollbackImpl::_stopAndWaitForIndexBuilds(OperationContext* opCtx) { // Get a list of all databases. StorageEngine* storageEngine = opCtx->getServiceContext()->getStorageEngine(); - std::vector dbs; + std::vector dbs; { Lock::GlobalLock lk(opCtx, MODE_IS); dbs = storageEngine->listDatabases(); @@ -364,10 +364,10 @@ void RollbackImpl::_stopAndWaitForIndexBuilds(OperationContext* opCtx) { // Wait for all background operations to complete by waiting on each database. Single-phase // index builds are not stopped before rollback, so we must wait for these index builds to // complete. - std::vector dbNames(dbs.begin(), dbs.end()); + std::vector tenantDbNames(dbs.begin(), dbs.end()); LOGV2(21595, "Waiting for all background operations to complete before starting rollback"); - for (auto db : dbNames) { - auto numInProg = IndexBuildsCoordinator::get(opCtx)->numInProgForDb(db); + for (auto tenantDbName : tenantDbNames) { + auto numInProg = IndexBuildsCoordinator::get(opCtx)->numInProgForDb(tenantDbName.dbName()); if (numInProg > 0) { LOGV2_DEBUG(21596, 1, @@ -375,8 +375,9 @@ void RollbackImpl::_stopAndWaitForIndexBuilds(OperationContext* opCtx) { "background operations to complete on database '{db}'", "Waiting for background operations to complete", "numBackgroundOperationsInProgress"_attr = numInProg, - "db"_attr = db); - IndexBuildsCoordinator::get(opCtx)->awaitNoBgOpInProgForDb(opCtx, db); + "db"_attr = tenantDbName); + IndexBuildsCoordinator::get(opCtx)->awaitNoBgOpInProgForDb(opCtx, + tenantDbName.dbName()); } } @@ -1360,11 +1361,11 @@ void RollbackImpl::_resetDropPendingState(OperationContext* opCtx) { auto storageEngine = opCtx->getServiceContext()->getStorageEngine(); storageEngine->clearDropPendingState(); - std::vector dbNames = storageEngine->listDatabases(); + std::vector tenantDbNames = storageEngine->listDatabases(); auto databaseHolder = DatabaseHolder::get(opCtx); - for (const auto& dbName : dbNames) { - Lock::DBLock dbLock(opCtx, dbName, MODE_X); - auto db = databaseHolder->openDb(opCtx, dbName); + for (const auto& tenantDbName : tenantDbNames) { + Lock::DBLock dbLock(opCtx, tenantDbName.dbName(), MODE_X); + auto db = databaseHolder->openDb(opCtx, tenantDbName.dbName()); db->checkForIdIndexesAndDropPendingCollections(opCtx); } } diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp index 3c61a2de652..716743743a3 100644 --- a/src/mongo/db/repl/storage_interface_impl.cpp +++ b/src/mongo/db/repl/storage_interface_impl.cpp @@ -411,25 +411,25 @@ Status StorageInterfaceImpl::insertDocuments(OperationContext* opCtx, Status StorageInterfaceImpl::dropReplicatedDatabases(OperationContext* opCtx) { Lock::GlobalWrite globalWriteLock(opCtx); - std::vector dbNames = + std::vector tenantDbNames = opCtx->getServiceContext()->getStorageEngine()->listDatabases(); - invariant(!dbNames.empty()); + invariant(!tenantDbNames.empty()); LOGV2(21754, "dropReplicatedDatabases - dropping {numDatabases} databases", "dropReplicatedDatabases - dropping databases", - "numDatabases"_attr = dbNames.size()); + "numDatabases"_attr = tenantDbNames.size()); ReplicationCoordinator::get(opCtx)->clearCommittedSnapshot(); auto databaseHolder = DatabaseHolder::get(opCtx); auto hasLocalDatabase = false; - for (const auto& dbName : dbNames) { - if (dbName == "local") { + for (const auto& tenantDbName : tenantDbNames) { + if (tenantDbName.dbName() == "local") { hasLocalDatabase = true; continue; } - writeConflictRetry(opCtx, "dropReplicatedDatabases", dbName, [&] { - if (auto db = databaseHolder->getDb(opCtx, dbName)) { + writeConflictRetry(opCtx, "dropReplicatedDatabases", tenantDbName.dbName(), [&] { + if (auto db = databaseHolder->getDb(opCtx, tenantDbName.dbName())) { databaseHolder->dropDb(opCtx, db); } else { // This is needed since dropDatabase can't be rolled back. @@ -439,7 +439,7 @@ Status StorageInterfaceImpl::dropReplicatedDatabases(OperationContext* opCtx) { "database names but before drop: {dbName}", "dropReplicatedDatabases - database disappeared after retrieving list of " "database names but before drop", - "dbName"_attr = dbName); + "dbName"_attr = tenantDbName); } }); } @@ -447,7 +447,7 @@ Status StorageInterfaceImpl::dropReplicatedDatabases(OperationContext* opCtx) { LOGV2(21756, "dropReplicatedDatabases - dropped {numDatabases} databases", "dropReplicatedDatabases - dropped databases", - "numDatabases"_attr = dbNames.size()); + "numDatabases"_attr = tenantDbNames.size()); return Status::OK(); } diff --git a/src/mongo/db/startup_recovery.cpp b/src/mongo/db/startup_recovery.cpp index 179d4f35be3..0c2aed85986 100644 --- a/src/mongo/db/startup_recovery.cpp +++ b/src/mongo/db/startup_recovery.cpp @@ -257,10 +257,10 @@ void openDatabases(OperationContext* opCtx, const StorageEngine* storageEngine, invariant(opCtx->lockState()->isW()); auto databaseHolder = DatabaseHolder::get(opCtx); - auto dbNames = storageEngine->listDatabases(); - for (const auto& dbName : dbNames) { - LOGV2_DEBUG(21010, 1, " Opening database: {dbName}", "dbName"_attr = dbName); - auto db = databaseHolder->openDb(opCtx, dbName); + auto tenantDbNames = storageEngine->listDatabases(); + for (const auto& tenantDbName : tenantDbNames) { + LOGV2_DEBUG(21010, 1, " Opening database: {dbName}", "dbName"_attr = tenantDbName); + auto db = databaseHolder->openDb(opCtx, tenantDbName.dbName()); invariant(db); onDatabase(db); @@ -482,19 +482,21 @@ void startupRepair(OperationContext* opCtx, StorageEngine* storageEngine) { // The local database should be repaired before any other replicated collections so we know // whether not to rebuild unfinished two-phase index builds if this is a replica set node // running in standalone mode. - auto dbNames = storageEngine->listDatabases(); - if (auto it = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb); - it != dbNames.end()) { - fassertNoTrace(4805001, repair::repairDatabase(opCtx, storageEngine, *it)); + auto tenantDbNames = storageEngine->listDatabases(); + if (auto it = std::find(tenantDbNames.begin(), + tenantDbNames.end(), + TenantDatabaseName(boost::none, NamespaceString::kLocalDb)); + it != tenantDbNames.end()) { + fassertNoTrace(4805001, repair::repairDatabase(opCtx, storageEngine, it->dbName())); // This must be set before rebuilding index builds on replicated collections. setReplSetMemberInStandaloneMode(opCtx, StartupRecoveryMode::kAuto); - dbNames.erase(it); + tenantDbNames.erase(it); } // Repair the remaining databases. - for (const auto& dbName : dbNames) { - fassertNoTrace(18506, repair::repairDatabase(opCtx, storageEngine, dbName)); + for (const auto& tenantDbName : tenantDbNames) { + fassertNoTrace(18506, repair::repairDatabase(opCtx, storageEngine, tenantDbName.dbName())); } openDatabases(opCtx, storageEngine, [&](auto db) { diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript index f44ddf8a9d8..cc16a8a4602 100644 --- a/src/mongo/db/storage/SConscript +++ b/src/mongo/db/storage/SConscript @@ -575,6 +575,7 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog/catalog_control', '$BUILD_DIR/mongo/db/catalog_raii', + '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/server_options_core', '$BUILD_DIR/mongo/db/storage/durable_catalog_impl', '$BUILD_DIR/mongo/db/storage/kv/kv_drop_pending_ident_reaper', @@ -584,8 +585,8 @@ env.Library( '$BUILD_DIR/mongo/db/audit', '$BUILD_DIR/mongo/db/catalog/collection_catalog_helper', '$BUILD_DIR/mongo/db/catalog/index_catalog', - '$BUILD_DIR/mongo/db/multitenancy', '$BUILD_DIR/mongo/db/resumable_index_builds_idl', + '$BUILD_DIR/mongo/db/server_feature_flags', '$BUILD_DIR/mongo/db/storage/storage_repair_observer', '$BUILD_DIR/mongo/db/vector_clock', 'backup_block', diff --git a/src/mongo/db/storage/storage_engine.h b/src/mongo/db/storage/storage_engine.h index 064508add80..ed49cf62ca4 100644 --- a/src/mongo/db/storage/storage_engine.h +++ b/src/mongo/db/storage/storage_engine.h @@ -39,6 +39,7 @@ #include "mongo/db/catalog/index_builds.h" #include "mongo/db/resumable_index_builds_gen.h" #include "mongo/db/storage/temporary_record_store.h" +#include "mongo/db/tenant_database_name.h" #include "mongo/util/functional.h" #include "mongo/util/str.h" @@ -201,7 +202,7 @@ public: /** * List the databases stored in this storage engine. */ - virtual std::vector listDatabases() const = 0; + virtual std::vector listDatabases() const = 0; /** * Returns whether the storage engine supports capped collections. diff --git a/src/mongo/db/storage/storage_engine_impl.cpp b/src/mongo/db/storage/storage_engine_impl.cpp index 9c6dd3f0423..16435c9cac8 100644 --- a/src/mongo/db/storage/storage_engine_impl.cpp +++ b/src/mongo/db/storage/storage_engine_impl.cpp @@ -773,8 +773,15 @@ RecoveryUnit* StorageEngineImpl::newRecoveryUnit() { return _engine->newRecoveryUnit(); } -std::vector StorageEngineImpl::listDatabases() const { - return CollectionCatalog::get(getGlobalServiceContext())->getAllDbNames(); +std::vector StorageEngineImpl::listDatabases() const { + // TODO SERVER-61988 Return instead the result of CollectionCatalog::getAllDbNames() directly. + std::vector tenantDbNames; + std::vector dbNames = + CollectionCatalog::get(getGlobalServiceContext())->getAllDbNames(); + for (auto dbName : dbNames) { + tenantDbNames.push_back(TenantDatabaseName(boost::none, dbName)); + } + return tenantDbNames; } Status StorageEngineImpl::closeDatabase(OperationContext* opCtx, StringData db) { diff --git a/src/mongo/db/storage/storage_engine_impl.h b/src/mongo/db/storage/storage_engine_impl.h index 4f25de8f7d2..b2e60391fcc 100644 --- a/src/mongo/db/storage/storage_engine_impl.h +++ b/src/mongo/db/storage/storage_engine_impl.h @@ -72,7 +72,7 @@ public: virtual RecoveryUnit* newRecoveryUnit() override; - virtual std::vector listDatabases() const override; + virtual std::vector listDatabases() const override; virtual bool supportsCappedCollections() const override { return _supportsCappedCollections; diff --git a/src/mongo/db/storage/storage_engine_mock.h b/src/mongo/db/storage/storage_engine_mock.h index 93a187c7615..974393c5a5c 100644 --- a/src/mongo/db/storage/storage_engine_mock.h +++ b/src/mongo/db/storage/storage_engine_mock.h @@ -41,7 +41,7 @@ public: RecoveryUnit* newRecoveryUnit() final { return nullptr; } - std::vector listDatabases() const final { + std::vector listDatabases() const final { return {}; } bool supportsCappedCollections() const final { -- cgit v1.2.1