SERVER-39037 Refactor check whether a replica set node is running in standalone mode

7 files changed, 100 insertions, 39 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 07b54f4ab9b..f25199d6523 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -95,6 +95,7 @@
 #include "mongo/db/logical_time_metadata_hook.h"
 #include "mongo/db/logical_time_validator.h"
 #include "mongo/db/mongod_options.h"
+#include "mongo/db/namespace_string.h"
 #include "mongo/db/op_observer_registry.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/periodic_runner_job_abort_expired_transactions.h"
@@ -112,6 +113,7 @@
 #include "mongo/db/repl/replication_recovery.h"
 #include "mongo/db/repl/storage_interface_impl.h"
 #include "mongo/db/repl/topology_coordinator.h"
+#include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/s/balancer/balancer.h"
 #include "mongo/db/s/config/sharding_catalog_manager.h"
 #include "mongo/db/s/config_server_op_observer.h"
@@ -193,7 +195,6 @@ using std::endl;
 namespace {
 
 const NamespaceString startupLogCollectionName("local.startup_log");
-const NamespaceString kSystemReplSetCollection("local.system.replset");
 
 #ifdef _WIN32
 const ntservice::NtServiceDefaultStrings defaultServiceStrings = {
@@ -242,21 +243,6 @@ void logStartup(OperationContext* opCtx) {
     wunit.commit();
 }
 
-/**
- * Checks if this server was started without --replset but has a config in local.system.replset
- * (meaning that this is probably a replica set member started in stand-alone mode).
- *
- * @returns the number of documents in local.system.replset or 0 if this was started with
- *          --replset.
- */
-unsigned long long checkIfReplMissingFromCommandLine(OperationContext* opCtx) {
-    if (!repl::ReplicationCoordinator::get(opCtx)->getSettings().usingReplSets()) {
-        DBDirectClient c(opCtx);
-        return c.count(kSystemReplSetCollection.ns());
-    }
-    return 0;
-}
-
 void initWireSpec() {
     WireSpec& spec = WireSpec::instance();
 
@@ -581,12 +567,10 @@ ExitCode _initAndListen(int listenPort) {
         }
 
         repl::ReplicationCoordinator::get(startupOpCtx.get())->startup(startupOpCtx.get());
-        const unsigned long long missingRepl =
-            checkIfReplMissingFromCommandLine(startupOpCtx.get());
-        if (missingRepl) {
+        if (getReplSetMemberInStandaloneMode(serviceContext)) {
             log() << startupWarningsLog;
-            log() << "** WARNING: mongod started without --replSet yet " << missingRepl
-                  << " documents are present in local.system.replset." << startupWarningsLog;
+            log() << "** WARNING: mongod started without --replSet yet document(s) are present in "
+                  << NamespaceString::kSystemReplSetNamespace << "." << startupWarningsLog;
             log() << "**          Database contents may appear inconsistent with the oplog and may "
                      "appear to not contain"
                   << startupWarningsLog;
diff --git a/src/mongo/db/namespace_string.cpp b/src/mongo/db/namespace_string.cpp
index 53e119e6f95..f13a31e3ecd 100644
--- a/src/mongo/db/namespace_string.cpp
+++ b/src/mongo/db/namespace_string.cpp
@@ -73,6 +73,8 @@ const NamespaceString NamespaceString::kShardConfigDatabasesNamespace(NamespaceS
 const NamespaceString NamespaceString::kSystemKeysNamespace(NamespaceString::kAdminDb,
                                                             "system.keys");
 const NamespaceString NamespaceString::kRsOplogNamespace(NamespaceString::kLocalDb, "oplog.rs");
+const NamespaceString NamespaceString::kSystemReplSetNamespace(NamespaceString::kLocalDb,
+                                                               "system.replset");
 
 bool NamespaceString::isListCollectionsCursorNS() const {
     return coll() == listCollectionsCursorCol;
diff --git a/src/mongo/db/namespace_string.h b/src/mongo/db/namespace_string.h
index 69122a36229..80766e9564d 100644
--- a/src/mongo/db/namespace_string.h
+++ b/src/mongo/db/namespace_string.h
@@ -96,6 +96,9 @@ public:
     // Namespace for storing the persisted state of transaction coordinators.
     static const NamespaceString kTransactionCoordinatorsNamespace;
 
+    // Namespace for replica set configuration settings.
+    static const NamespaceString kSystemReplSetNamespace;
+
     /**
      * Constructs an empty NamespaceString.
      */
diff --git a/src/mongo/db/repair_database.cpp b/src/mongo/db/repair_database.cpp
index 2e4007b90e1..20808127c4f 100644
--- a/src/mongo/db/repair_database.cpp
+++ b/src/mongo/db/repair_database.cpp
@@ -234,7 +234,8 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx,
 namespace {
 Status repairCollections(OperationContext* opCtx,
                          StorageEngine* engine,
-                         const std::string& dbName) {
+                         const std::string& dbName,
+                         stdx::function<void(const std::string& dbName)> onRecordStoreRepair) {
 
     DatabaseCatalogEntry* dbce = engine->getDatabaseCatalogEntry(opCtx, dbName);
 
@@ -242,8 +243,6 @@ Status repairCollections(OperationContext* opCtx,
     dbce->getCollectionNamespaces(&colls);
 
     for (std::list<std::string>::const_iterator it = colls.begin(); it != colls.end(); ++it) {
-        // Don't check for interrupt after starting to repair a collection otherwise we can
-        // leave data in an inconsistent state. Interrupting between collections is ok, however.
         opCtx->checkForInterrupt();
 
         log() << "Repairing collection " << *it;
@@ -251,13 +250,19 @@ Status repairCollections(OperationContext* opCtx,
         Status status = engine->repairRecordStore(opCtx, *it);
         if (!status.isOK())
             return status;
+    }
+
+    onRecordStoreRepair(dbName);
+
+    for (std::list<std::string>::const_iterator it = colls.begin(); it != colls.end(); ++it) {
+        opCtx->checkForInterrupt();
 
         CollectionCatalogEntry* cce = dbce->getCollectionCatalogEntry(*it);
         auto swIndexNameObjs = getIndexNameObjs(opCtx, dbce, cce);
         if (!swIndexNameObjs.isOK())
             return swIndexNameObjs.getStatus();
 
-        status = rebuildIndexesOnCollection(opCtx, dbce, cce, swIndexNameObjs.getValue());
+        Status status = rebuildIndexesOnCollection(opCtx, dbce, cce, swIndexNameObjs.getValue());
         if (!status.isOK())
             return status;
 
@@ -267,7 +272,10 @@ Status repairCollections(OperationContext* opCtx,
 }
 }  // namespace
 
-Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName) {
+Status repairDatabase(OperationContext* opCtx,
+                      StorageEngine* engine,
+                      const std::string& dbName,
+                      stdx::function<void(const std::string& dbName)> onRecordStoreRepair) {
     DisableDocumentValidation validationDisabler(opCtx);
 
     // We must hold some form of lock here
@@ -308,7 +316,7 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std:
         }
     });
 
-    auto status = repairCollections(opCtx, engine, dbName);
+    auto status = repairCollections(opCtx, engine, dbName, onRecordStoreRepair);
     if (!status.isOK()) {
         severe() << "Failed to repair database " << dbName << ": " << status.reason();
         return status;
@@ -316,4 +324,5 @@ Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std:
 
     return Status::OK();
 }
+
 }  // namespace mongo
diff --git a/src/mongo/db/repair_database.h b/src/mongo/db/repair_database.h
index 2987834a03f..c3c5d0ef423 100644
--- a/src/mongo/db/repair_database.h
+++ b/src/mongo/db/repair_database.h
@@ -74,6 +74,15 @@ Status rebuildIndexesOnCollection(OperationContext* opCtx,
  * Repairs a database using a storage engine-specific, best-effort process.
  * Some data may be lost or modified in the process but the output will
  * be structurally valid on successful return.
+ *
+ * Calls 'onRecordStoreRepair' after repairing all the collection record stores for each database
+ * before rebuilding the appropriate indexes.
+ *
+ * It is expected that the local database will be repaired first when running in repair mode.
  */
-Status repairDatabase(OperationContext* opCtx, StorageEngine* engine, const std::string& dbName);
-}
+Status repairDatabase(OperationContext* opCtx,
+                      StorageEngine* engine,
+                      const std::string& dbName,
+                      stdx::function<void(const std::string& dbName)> onRecordStoreRepair);
+
+}  // namespace mongo
diff --git a/src/mongo/db/repair_database_and_check_version.cpp b/src/mongo/db/repair_database_and_check_version.cpp
index e45fb40ab5b..d2a1be11e38 100644
--- a/src/mongo/db/repair_database_and_check_version.cpp
+++ b/src/mongo/db/repair_database_and_check_version.cpp
@@ -42,12 +42,16 @@
 #include "mongo/db/commands/feature_compatibility_version_documentation.h"
 #include "mongo/db/commands/feature_compatibility_version_parser.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
+#include "mongo/db/db_raii.h"
 #include "mongo/db/dbhelpers.h"
+#include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/repair_database.h"
 #include "mongo/db/repl/replication_coordinator.h"
+#include "mongo/db/repl_set_member_in_standalone_mode.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/storage/storage_repair_observer.h"
+#include "mongo/stdx/functional.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/log.h"
@@ -152,7 +156,6 @@ Status ensureAllCollectionsHaveUUIDs(OperationContext* opCtx,
 }
 
 const NamespaceString startupLogCollectionName("local.startup_log");
-const NamespaceString kSystemReplSetCollection("local.system.replset");
 
 /**
  * Returns 'true' if this server has a configuration document in local.system.replset.
@@ -160,7 +163,8 @@ const NamespaceString kSystemReplSetCollection("local.system.replset");
 bool hasReplSetConfigDoc(OperationContext* opCtx) {
     Lock::GlobalWrite lk(opCtx);
     BSONObj config;
-    return Helpers::getSingleton(opCtx, kSystemReplSetCollection.ns().c_str(), config);
+    return Helpers::getSingleton(
+        opCtx, NamespaceString::kSystemReplSetNamespace.ns().c_str(), config);
 }
 
 /**
@@ -232,6 +236,34 @@ void rebuildIndexes(OperationContext* opCtx, StorageEngine* storageEngine) {
     }
 }
 
+/**
+ * Sets the appropriate flag on the service context decorable 'replSetMemberInStandaloneMode' to
+ * 'true' if this is a replica set node running in standalone mode, otherwise 'false'.
+ */
+void setReplSetMemberInStandaloneMode(OperationContext* opCtx) {
+    const repl::ReplSettings& replSettings =
+        repl::ReplicationCoordinator::get(opCtx)->getSettings();
+
+    if (replSettings.usingReplSets()) {
+        // Not in standalone mode.
+        setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false);
+        return;
+    }
+
+    Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X);
+    auto databaseHolder = DatabaseHolder::get(opCtx);
+    databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db());
+
+    AutoGetCollectionForRead autoCollection(opCtx, NamespaceString::kSystemReplSetNamespace);
+    Collection* collection = autoCollection.getCollection();
+    if (collection && collection->numRecords(opCtx) > 0) {
+        setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), true);
+        return;
+    }
+
+    setReplSetMemberInStandaloneMode(opCtx->getServiceContext(), false);
+}
+
 }  // namespace
 
 /**
@@ -248,6 +280,10 @@ StatusWith<bool> repairDatabasesAndCheckVersion(OperationContext* opCtx) {
     // Rebuilding indexes must be done before a database can be opened, except when using repair,
     // which rebuilds all indexes when it is done.
     if (!storageGlobalParams.readOnly && !storageGlobalParams.repair) {
+        // Determine whether this is a replica set node running in standalone mode. If we're in
+        // repair mode, we cannot set the flag yet as it needs to open a database and look through a
+        // collection. Rebuild the necessary indexes after setting the flag.
+        setReplSetMemberInStandaloneMode(opCtx);
         rebuildIndexes(opCtx, storageEngine);
     }
 
@@ -263,9 +299,26 @@ StatusWith<bool> repairDatabasesAndCheckVersion(OperationContext* opCtx) {
             quickExit(EXIT_ABRUPT);
         }
 
+        // Ensure that the local database is repaired first, if it exists, so that we can open it
+        // before any other database to be able to determine if this is a replica set node running
+        // in standalone mode before rebuilding any indexes.
+        auto dbNamesIt = std::find(dbNames.begin(), dbNames.end(), NamespaceString::kLocalDb);
+        if (dbNamesIt != dbNames.end()) {
+            std::swap(dbNames.front(), *dbNamesIt);
+            invariant(dbNames.front() == NamespaceString::kLocalDb);
+        }
+
+        stdx::function<void(const std::string& dbName)> onRecordStoreRepair =
+            [opCtx](const std::string& dbName) {
+                if (dbName == NamespaceString::kLocalDb) {
+                    setReplSetMemberInStandaloneMode(opCtx);
+                }
+            };
+
         for (const auto& dbName : dbNames) {
             LOG(1) << "    Repairing database: " << dbName;
-            fassertNoTrace(18506, repairDatabase(opCtx, storageEngine, dbName));
+            fassertNoTrace(18506,
+                           repairDatabase(opCtx, storageEngine, dbName, onRecordStoreRepair));
         }
 
         // All collections must have UUIDs before restoring the FCV document to a version that
@@ -304,13 +357,13 @@ StatusWith<bool> repairDatabasesAndCheckVersion(OperationContext* opCtx) {
 
     if (!storageGlobalParams.readOnly) {
         // We open the "local" database before calling hasReplSetConfigDoc() to ensure the in-memory
-        // catalog entries for the 'kSystemReplSetCollection' collection have been populated if the
+        // catalog entries for the 'kSystemReplSetNamespace' collection have been populated if the
         // collection exists. If the "local" database didn't exist at this point yet, then it will
         // be created. If the mongod is running in a read-only mode, then it is fine to not open the
         // "local" database and populate the catalog entries because we won't attempt to drop the
         // temporary collections anyway.
-        Lock::DBLock dbLock(opCtx, kSystemReplSetCollection.db(), MODE_X);
-        databaseHolder->openDb(opCtx, kSystemReplSetCollection.db());
+        Lock::DBLock dbLock(opCtx, NamespaceString::kSystemReplSetNamespace.db(), MODE_X);
+        databaseHolder->openDb(opCtx, NamespaceString::kSystemReplSetNamespace.db());
     }
 
     if (storageGlobalParams.repair) {
diff --git a/src/mongo/db/repair_database_and_check_version.h b/src/mongo/db/repair_database_and_check_version.h
index d88f777e744..2999409cc6c 100644
--- a/src/mongo/db/repair_database_and_check_version.h
+++ b/src/mongo/db/repair_database_and_check_version.h
@@ -36,8 +36,9 @@ namespace mongo {
 class OperationContext;
 
 /**
-* Return an error status if the wrong mongod version was used for these datafiles. The boolean
-* represents whether there are non-local databases.
-*/
+ * Return an error status if the wrong mongod version was used for these datafiles. The boolean
+ * represents whether there are non-local databases.
+ */
 StatusWith<bool> repairDatabasesAndCheckVersion(OperationContext* opCtx);
-}
+
+}  // namespace mongo