SERVER-25475 Re-order mongod initialization to avoid segfault.

This ensures that the sharding system is initialized before the ReplicationCoordinator is started,
thus avoiding the possibility of the becoming primary before the ShardingCatalogManager has been
installed on the Grid object.

2 files changed, 71 insertions, 74 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 5e2f946c96f..7ff7adcc55a 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -669,86 +669,52 @@ static ExitCode _initAndListen(int listenPort) {
         web.detach();
     }
 
-    {
 #ifndef _WIN32
-        mongo::signalForkSuccess();
+    mongo::signalForkSuccess();
 #endif
-        AuthorizationManager* globalAuthzManager = getGlobalAuthorizationManager();
-        if (globalAuthzManager->shouldValidateAuthSchemaOnStartup()) {
-            Status status = authindex::verifySystemIndexes(startupOpCtx.get());
-            if (!status.isOK()) {
-                log() << status.reason();
-                exitCleanly(EXIT_NEED_UPGRADE);
-            }
-
-            // SERVER-14090: Verify that auth schema version is schemaVersion26Final.
-            int foundSchemaVersion;
-            status = globalAuthzManager->getAuthorizationVersion(startupOpCtx.get(),
-                                                                 &foundSchemaVersion);
-            if (!status.isOK()) {
-                log() << "Auth schema version is incompatible: "
-                      << "User and role management commands require auth data to have "
-                      << "at least schema version " << AuthorizationManager::schemaVersion26Final
-                      << " but startup could not verify schema version: " << status.toString()
-                      << endl;
-                exitCleanly(EXIT_NEED_UPGRADE);
-            }
-            if (foundSchemaVersion < AuthorizationManager::schemaVersion26Final) {
-                log() << "Auth schema version is incompatible: "
-                      << "User and role management commands require auth data to have "
-                      << "at least schema version " << AuthorizationManager::schemaVersion26Final
-                      << " but found " << foundSchemaVersion << ". In order to upgrade "
-                      << "the auth schema, first downgrade MongoDB binaries to version "
-                      << "2.6 and then run the authSchemaUpgrade command." << endl;
-                exitCleanly(EXIT_NEED_UPGRADE);
-            }
-        } else if (globalAuthzManager->isAuthEnabled()) {
-            error() << "Auth must be disabled when starting without auth schema validation";
-            exitCleanly(EXIT_BADOPTIONS);
-        } else {
-            // If authSchemaValidation is disabled and server is running without auth,
-            // warn the user and continue startup without authSchema metadata checks.
-            log() << startupWarningsLog;
-            log() << "** WARNING: Startup auth schema validation checks are disabled for the "
-                     "database."
-                  << startupWarningsLog;
-            log() << "**          This mode should only be used to manually repair corrupted auth "
-                     "data."
-                  << startupWarningsLog;
+    AuthorizationManager* globalAuthzManager = getGlobalAuthorizationManager();
+    if (globalAuthzManager->shouldValidateAuthSchemaOnStartup()) {
+        Status status = authindex::verifySystemIndexes(startupOpCtx.get());
+        if (!status.isOK()) {
+            log() << status.reason();
+            exitCleanly(EXIT_NEED_UPGRADE);
         }
 
-        if (!storageGlobalParams.readOnly) {
-            logStartup(startupOpCtx.get());
-
-            getDeleter()->startWorkers();
-
-            restartInProgressIndexesFromLastShutdown(startupOpCtx.get());
-
-            repl::getGlobalReplicationCoordinator()->startup(startupOpCtx.get());
-
-            const unsigned long long missingRepl =
-                checkIfReplMissingFromCommandLine(startupOpCtx.get());
-            if (missingRepl) {
-                log() << startupWarningsLog;
-                log() << "** WARNING: mongod started without --replSet yet " << missingRepl
-                      << " documents are present in local.system.replset" << startupWarningsLog;
-                log() << "**          Restart with --replSet unless you are doing maintenance and "
-                      << " no other clients are connected." << startupWarningsLog;
-                log() << "**          The TTL collection monitor will not start because of this."
-                      << startupWarningsLog;
-                log() << "**         ";
-                log() << " For more info see http://dochub.mongodb.org/core/ttlcollections";
-                log() << startupWarningsLog;
-            } else {
-                startTTLBackgroundJob();
-            }
+        // SERVER-14090: Verify that auth schema version is schemaVersion26Final.
+        int foundSchemaVersion;
+        status =
+            globalAuthzManager->getAuthorizationVersion(startupOpCtx.get(), &foundSchemaVersion);
+        if (!status.isOK()) {
+            log() << "Auth schema version is incompatible: "
+                  << "User and role management commands require auth data to have "
+                  << "at least schema version " << AuthorizationManager::schemaVersion26Final
+                  << " but startup could not verify schema version: " << status.toString() << endl;
+            exitCleanly(EXIT_NEED_UPGRADE);
+        }
+        if (foundSchemaVersion < AuthorizationManager::schemaVersion26Final) {
+            log() << "Auth schema version is incompatible: "
+                  << "User and role management commands require auth data to have "
+                  << "at least schema version " << AuthorizationManager::schemaVersion26Final
+                  << " but found " << foundSchemaVersion << ". In order to upgrade "
+                  << "the auth schema, first downgrade MongoDB binaries to version "
+                  << "2.6 and then run the authSchemaUpgrade command." << endl;
+            exitCleanly(EXIT_NEED_UPGRADE);
         }
+    } else if (globalAuthzManager->isAuthEnabled()) {
+        error() << "Auth must be disabled when starting without auth schema validation";
+        exitCleanly(EXIT_BADOPTIONS);
+    } else {
+        // If authSchemaValidation is disabled and server is running without auth,
+        // warn the user and continue startup without authSchema metadata checks.
+        log() << startupWarningsLog;
+        log() << "** WARNING: Startup auth schema validation checks are disabled for the "
+                 "database."
+              << startupWarningsLog;
+        log() << "**          This mode should only be used to manually repair corrupted auth "
+                 "data."
+              << startupWarningsLog;
     }
 
-    startClientCursorMonitor();
-
-    PeriodicTask::startRunningPeriodicTasks();
-
     HostnameCanonicalizationWorker::start(getGlobalServiceContext());
 
     if (!replSettings.usingReplSets() && !replSettings.isSlave() &&
@@ -762,7 +728,14 @@ static ExitCode _initAndListen(int listenPort) {
                         ->initializeShardingAwarenessIfNeeded(startupOpCtx.get()));
 
     if (!storageGlobalParams.readOnly) {
+        logStartup(startupOpCtx.get());
+
         startFTDC();
+
+        getDeleter()->startWorkers();
+
+        restartInProgressIndexesFromLastShutdown(startupOpCtx.get());
+
         if (serverGlobalParams.clusterRole == ClusterRole::ShardServer) {
             // Note: For replica sets, ShardingStateRecovery happens on transition to primary.
             if (!repl::getGlobalReplicationCoordinator()->isReplEnabled()) {
@@ -775,8 +748,32 @@ static ExitCode _initAndListen(int listenPort) {
                                                        kDistLockProcessIdForConfigServer));
             Balancer::create(startupOpCtx->getServiceContext());
         }
+
+        repl::getGlobalReplicationCoordinator()->startup(startupOpCtx.get());
+
+        const unsigned long long missingRepl =
+            checkIfReplMissingFromCommandLine(startupOpCtx.get());
+        if (missingRepl) {
+            log() << startupWarningsLog;
+            log() << "** WARNING: mongod started without --replSet yet " << missingRepl
+                  << " documents are present in local.system.replset" << startupWarningsLog;
+            log() << "**          Restart with --replSet unless you are doing maintenance and "
+                  << " no other clients are connected." << startupWarningsLog;
+            log() << "**          The TTL collection monitor will not start because of this."
+                  << startupWarningsLog;
+            log() << "**         ";
+            log() << " For more info see http://dochub.mongodb.org/core/ttlcollections";
+            log() << startupWarningsLog;
+        } else {
+            startTTLBackgroundJob();
+        }
     }
 
+
+    startClientCursorMonitor();
+
+    PeriodicTask::startRunningPeriodicTasks();
+
     // MessageServer::run will return when exit code closes its socket and we don't need the
     // operation context anymore
     startupOpCtx.reset();
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index ad89b160e80..3b0188bbad2 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -650,7 +650,7 @@ void ReplicationCoordinatorExternalStateImpl::shardingOnTransitionToPrimaryHook(
                                                << causedBy(status)));
         }
 
-        if (status != ErrorCodes::AlreadyInitialized) {
+        if (status.isOK()) {
             // Load the clusterId into memory. Use local readConcern, since we can't use majority
             // readConcern in drain mode because the global lock prevents replication. This is
             // safe, since if the clusterId write is rolled back, any writes that depend on it will