From d76c1d8a1061cb4fe8111ded7db18d1ea9e6275e Mon Sep 17 00:00:00 2001 From: Gregory Wlodarek Date: Fri, 25 Jan 2019 18:26:12 -0500 Subject: SERVER-39037 Refactor check whether a replica set node is running in standalone mode --- src/mongo/db/db.cpp | 26 ++------- src/mongo/db/namespace_string.cpp | 2 + src/mongo/db/namespace_string.h | 3 + src/mongo/db/repair_database.cpp | 21 +++++-- src/mongo/db/repair_database.h | 13 ++++- src/mongo/db/repair_database_and_check_version.cpp | 65 ++++++++++++++++++++-- src/mongo/db/repair_database_and_check_version.h | 9 +-- 7 files changed, 100 insertions(+), 39 deletions(-) (limited to 'src/mongo') diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp index 07b54f4ab9b..f25199d6523 100644 --- a/src/mongo/db/db.cpp +++ b/src/mongo/db/db.cpp @@ -95,6 +95,7 @@ #include "mongo/db/logical_time_metadata_hook.h" #include "mongo/db/logical_time_validator.h" #include "mongo/db/mongod_options.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/op_observer_registry.h" #include "mongo/db/operation_context.h" #include "mongo/db/periodic_runner_job_abort_expired_transactions.h" @@ -112,6 +113,7 @@ #include "mongo/db/repl/replication_recovery.h" #include "mongo/db/repl/storage_interface_impl.h" #include "mongo/db/repl/topology_coordinator.h" +#include "mongo/db/repl_set_member_in_standalone_mode.h" #include "mongo/db/s/balancer/balancer.h" #include "mongo/db/s/config/sharding_catalog_manager.h" #include "mongo/db/s/config_server_op_observer.h" @@ -193,7 +195,6 @@ using std::endl; namespace { const NamespaceString startupLogCollectionName("local.startup_log"); -const NamespaceString kSystemReplSetCollection("local.system.replset"); #ifdef _WIN32 const ntservice::NtServiceDefaultStrings defaultServiceStrings = { @@ -242,21 +243,6 @@ void logStartup(OperationContext* opCtx) { wunit.commit(); } -/** - * Checks if this server was started without --replset but has a config in local.system.replset - * (meaning that this is probably a replica set member started in stand-alone mode). - * - * @returns the number of documents in local.system.replset or 0 if this was started with - * --replset. - */ -unsigned long long checkIfReplMissingFromCommandLine(OperationContext* opCtx) { - if (!repl::ReplicationCoordinator::get(opCtx)->getSettings().usingReplSets()) { - DBDirectClient c(opCtx); - return c.count(kSystemReplSetCollection.ns()); - } - return 0; -} - void initWireSpec() { WireSpec& spec = WireSpec::instance(); @@ -581,12 +567,10 @@ ExitCode _initAndListen(int listenPort) { } repl::ReplicationCoordinator::get(startupOpCtx.get())->startup(startupOpCtx.get()); - const unsigned long long missingRepl = - checkIfReplMissingFromCommandLine(startupOpCtx.get()); - if (missingRepl) { + if (getReplSetMemberInStandaloneMode(serviceContext)) { log() << startupWarningsLog; - log() << "** WARNING: mongod started without --replSet yet " << missingRepl - << " documents are present in local.system.replset." << startupWarningsLog; + log() << "** WARNING: mongod started without --replSet yet document(s) are present in " + << NamespaceString::kSystemReplSetNamespace << "." << startupWarningsLog; log() << "** Database contents may appear inconsistent with the oplog and may " "appear to not contain" << startupWarningsLog; diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp index 53e119e6f95..f13a31e3ecd 100644 --- a/src/mongo/db/namespace_string.cpp +++ b/src/mongo/db/namespace_string.cpp @@ -73,6 +73,8 @@ const NamespaceString NamespaceString::kShardConfigDatabasesNamespace(NamespaceS const NamespaceString NamespaceString::kSystemKeysNamespace(NamespaceString::kAdminDb, "system.keys"); const NamespaceString NamespaceString::kRsOplogNamespace(NamespaceString::kLocalDb, "oplog.rs"); +const NamespaceString NamespaceString::kSystemReplSetNamespace(NamespaceString::kLocalDb, + "system.replset"); bool NamespaceString::isListCollectionsCursorNS() const { return coll() == listCollectionsCursorCol; diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h index 69122a36229..80766e9564d 100644 --- a/src/mongo/db/namespace_string.h +++ b/src/mongo/db/namespace_string.h @@ -96,6 +96,9 @@ public: // Namespace for storing the persisted state of transaction coordinators. static const NamespaceString kTransactionCoordinatorsNamespace; + // Namespace for replica set configuration settings. + static const NamespaceString kSystemReplSetNamespace; + /** * Constructs an empty NamespaceString. */ diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp index 2e4007b90e1..20808127c4f 100644 --- a/src/mongo/db/repair_database.cpp +++ b/src/mongo/db/repair_database.cpp @@ -234,7 +234,8 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx, namespace { Status repairCollections(OperationContext* opCtx, StorageEngine* engine, - const std::string& dbName) { + const std::string& dbName, + stdx::function onRecordStoreRepair) { DatabaseCatalogEntry* dbce = engine->getDatabaseCatalogEntry(opCtx, dbName); @@ -242,8 +243,6 @@ Status repairCollections(OperationContext* opCtx, dbce->getCollectionNamespaces(&colls); for (std::list::const_iterator it = colls.begin(); it != colls.end(); ++it) { - // Don't check for interrupt after starting to repair a collection otherwise we can - // leave data in an inconsistent state. Interrupting between collections is ok, however. opCtx->checkForInterrupt(); log() << "Repairing collection " << *it; @@ -251,13 +250,19 @@ Status repairCollections(OperationContext* opCtx, Status status = engine->repairRecordStore(opCtx, *it); if (!status.isOK()) return status; + } + + onRecordStoreRepair(dbName); + + for (std::list::const_iterator it = colls.begin(); it != colls.end(); ++it) { + opCtx->checkForInterrupt(); CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry(*it); auto swIndexNameObjs = getIndexNameObjs(opCtx, dbce, cce); if (!swIndexNameObjs.isOK()) return swIndexNameObjs.getStatus(); - status = rebuildIndexesOnCollection(opCtx, dbce, cce, swIndexNameObjs.getValue()); + Status status = rebuildIndexesOnCollection(opCtx, dbce, cce, swIndexNameObjs.getValue()); if (!status.isOK()) return status; @@ -267,7 +272,10 @@ Status repairCollections(OperationContext* opCtx, } } // namespace -Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName) { +Status repairDatabase(OperationContext* opCtx, + StorageEngine* engine, + const std::string& dbName, + stdx::function onRecordStoreRepair) { DisableDocumentValidation validationDisabler(opCtx); // We must hold some form of lock here @@ -308,7 +316,7 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std: } }); - auto status = repairCollections(opCtx, engine, dbName); + auto status = repairCollections(opCtx, engine, dbName, onRecordStoreRepair); if (!status.isOK()) { severe() << "Failed to repair database " << dbName << ": " << status.reason(); return status; @@ -316,4 +324,5 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std: return Status::OK(); } + } // namespace mongo diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair_database.h index 2987834a03f..c3c5d0ef423 100644 --- a/src/mongo/db/repair_database.h +++ b/src/mongo/db/repair_database.h @@ -74,6 +74,15 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx, * Repairs a database using a storage engine-specific, best-effort process. * Some data may be lost or modified in the process but the output will * be structurally valid on successful return. + * + * Calls 'onRecordStoreRepair' after repairing all the collection record stores for each database + * before rebuilding the appropriate indexes. + * + * It is expected that the local database will be repaired first when running in repair mode. */ -Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName); -} +Status repairDatabase(OperationContext* opCtx, + StorageEngine* engine, + const std::string& dbName, + stdx::function onRecordStoreRepair); + +} // namespace mongo diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp index e45fb40ab5b..d2a1be11e38 100644 --- a/src/mongo/db/repair_database_and_check_version.cpp +++ b/src/mongo/db/repair_database_and_check_version.cpp @@ -42,12 +42,16 @@ #include "mongo/db/commands/feature_compatibility_version_documentation.h" #include "mongo/db/commands/feature_compatibility_version_parser.h" #include "mongo/db/concurrency/write_conflict_exception.h" +#include "mongo/db/db_raii.h" #include "mongo/db/dbhelpers.h" +#include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/repair_database.h" #include "mongo/db/repl/replication_coordinator.h" +#include "mongo/db/repl_set_member_in_standalone_mode.h" #include "mongo/db/server_options.h" #include "mongo/db/storage/storage_repair_observer.h" +#include "mongo/stdx/functional.h" #include "mongo/util/exit.h" #include "mongo/util/fail_point.h" #include "mongo/util/log.h" @@ -152,7 +156,6 @@ Status ensureAllCollectionsHaveUUIDs(OperationContext* opCtx, } const NamespaceString startupLogCollectionName("local.startup_log"); -const NamespaceString kSystemReplSetCollection("local.system.replset"); /** * Returns 'true' if this server has a configuration document in local.system.replset. @@ -160,7 +163,8 @@ const NamespaceString kSystemReplSetCollection("local.system.replset"); bool hasReplSetConfigDoc(OperationContext* opCtx) { Lock::GlobalWrite lk(opCtx); BSONObj config; - return Helpers::getSingleton(opCtx, kSystemReplSetCollection.ns().c_str(), config); + return Helpers::getSingleton( + opCtx, NamespaceString::kSystemReplSetNamespace.ns().c_str(), config); } /** @@ -232,6 +236,34 @@ void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) { } } +/** + * Sets the appropriate flag on the service context decorable 'replSetMemberInStandaloneMode' to + * 'true' if this is a replica set node running in standalone mode, otherwise 'false'. + */ +void setReplSetMemberInStandaloneMode(OperationContext* opCtx) { + const repl::ReplSettings& replSettings = + repl::ReplicationCoordinator::get(opCtx)->getSettings(); + + if (replSettings.usingReplSets()) { + // Not in standalone mode. + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); + return; + } + + Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X); + auto databaseHolder = DatabaseHolder::get(opCtx); + databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db()); + + AutoGetCollectionForRead autoCollection(opCtx, NamespaceString::kSystemReplSetNamespace); + Collection* collection = autoCollection.getCollection(); + if (collection && collection->numRecords(opCtx) > 0) { + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), true); + return; + } + + setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false); +} + } // namespace /** @@ -248,6 +280,10 @@ StatusWith repairDatabasesAndCheckVersion(OperationContext* opCtx) { // Rebuilding indexes must be done before a database can be opened, except when using repair, // which rebuilds all indexes when it is done. if (!storageGlobalParams.readOnly && !storageGlobalParams.repair) { + // Determine whether this is a replica set node running in standalone mode. If we're in + // repair mode, we cannot set the flag yet as it needs to open a database and look through a + // collection. Rebuild the necessary indexes after setting the flag. + setReplSetMemberInStandaloneMode(opCtx); rebuildIndexes(opCtx, storageEngine); } @@ -263,9 +299,26 @@ StatusWith repairDatabasesAndCheckVersion(OperationContext* opCtx) { quickExit(EXIT_ABRUPT); } + // Ensure that the local database is repaired first, if it exists, so that we can open it + // before any other database to be able to determine if this is a replica set node running + // in standalone mode before rebuilding any indexes. + auto dbNamesIt = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb); + if (dbNamesIt != dbNames.end()) { + std::swap(dbNames.front(), *dbNamesIt); + invariant(dbNames.front() == NamespaceString::kLocalDb); + } + + stdx::function onRecordStoreRepair = + [opCtx](const std::string& dbName) { + if (dbName == NamespaceString::kLocalDb) { + setReplSetMemberInStandaloneMode(opCtx); + } + }; + for (const auto& dbName : dbNames) { LOG(1) << " Repairing database: " << dbName; - fassertNoTrace(18506, repairDatabase(opCtx, storageEngine, dbName)); + fassertNoTrace(18506, + repairDatabase(opCtx, storageEngine, dbName, onRecordStoreRepair)); } // All collections must have UUIDs before restoring the FCV document to a version that @@ -304,13 +357,13 @@ StatusWith repairDatabasesAndCheckVersion(OperationContext* opCtx) { if (!storageGlobalParams.readOnly) { // We open the "local" database before calling hasReplSetConfigDoc() to ensure the in-memory - // catalog entries for the 'kSystemReplSetCollection' collection have been populated if the + // catalog entries for the 'kSystemReplSetNamespace' collection have been populated if the // collection exists. If the "local" database didn't exist at this point yet, then it will // be created. If the mongod is running in a read-only mode, then it is fine to not open the // "local" database and populate the catalog entries because we won't attempt to drop the // temporary collections anyway. - Lock::DBLock dbLock(opCtx, kSystemReplSetCollection.db(), MODE_X); - databaseHolder->openDb(opCtx, kSystemReplSetCollection.db()); + Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X); + databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db()); } if (storageGlobalParams.repair) { diff --git a/src/mongo/db/repair_database_and_check_version.h b/src/mongo/db/repair_database_and_check_version.h index d88f777e744..2999409cc6c 100644 --- a/src/mongo/db/repair_database_and_check_version.h +++ b/src/mongo/db/repair_database_and_check_version.h @@ -36,8 +36,9 @@ namespace mongo { class OperationContext; /** -* Return an error status if the wrong mongod version was used for these datafiles. The boolean -* represents whether there are non-local databases. -*/ + * Return an error status if the wrong mongod version was used for these datafiles. The boolean + * represents whether there are non-local databases. + */ StatusWith repairDatabasesAndCheckVersion(OperationContext* opCtx); -} + +} // namespace mongo -- cgit v1.2.1