SERVER-19903 Dist lock pinger not running for config RS

17 files changed, 49 insertions, 21 deletions

diff --git a/src/mongo/db/s/merge_chunks_command.cpp b/src/mongo/db/s/merge_chunks_command.cpp
index 58fc9c71ac3..813be2a8aa7 100644
--- a/src/mongo/db/s/merge_chunks_command.cpp
+++ b/src/mongo/db/s/merge_chunks_command.cpp
@@ -152,7 +152,7 @@ public:
                 return false;
             }
 
-            shardingState->initialize(config);
+            shardingState->initialize(txn, config);
         }
 
         // ShardName is optional, but might not be set yet
diff --git a/src/mongo/db/s/set_shard_version_command.cpp b/src/mongo/db/s/set_shard_version_command.cpp
index 03b96373844..6febcbaffa2 100644
--- a/src/mongo/db/s/set_shard_version_command.cpp
+++ b/src/mongo/db/s/set_shard_version_command.cpp
@@ -328,7 +328,7 @@ private:
         }
 
         if (locked) {
-            ShardingState::get(txn)->initialize(configdb);
+            ShardingState::get(txn)->initialize(txn, configdb);
             return true;
         }
 
diff --git a/src/mongo/db/s/sharding_state.cpp b/src/mongo/db/s/sharding_state.cpp
index 90b1a83af2a..1013fddd32e 100644
--- a/src/mongo/db/s/sharding_state.cpp
+++ b/src/mongo/db/s/sharding_state.cpp
@@ -140,7 +140,7 @@ string ShardingState::getShardName() {
     return _shardName;
 }
 
-void ShardingState::initialize(const string& server) {
+void ShardingState::initialize(OperationContext* txn, const string& server) {
     uassert(18509,
             "Unable to obtain host name during sharding initialization.",
             !getHostName().empty());
@@ -156,7 +156,7 @@ void ShardingState::initialize(const string& server) {
     ShardedConnectionInfo::addHook();
 
     ConnectionString configServerCS = uassertStatusOK(ConnectionString::parse(server));
-    uassertStatusOK(initializeGlobalShardingState(configServerCS));
+    uassertStatusOK(initializeGlobalShardingState(txn, configServerCS, false));
 
     _enabled = true;
 }
diff --git a/src/mongo/db/s/sharding_state.h b/src/mongo/db/s/sharding_state.h
index 0f7168355d8..b794a31df5d 100644
--- a/src/mongo/db/s/sharding_state.h
+++ b/src/mongo/db/s/sharding_state.h
@@ -88,7 +88,7 @@ public:
     // Initialize sharding state and begin authenticating outgoing connections and handling
     // shard versions.  If this is not run before sharded operations occur auth will not work
     // and versions will not be tracked.
-    void initialize(const std::string& server);
+    void initialize(OperationContext* txn, const std::string& server);
 
     // TODO: The only reason we need this method and cannot merge it together with the initialize
     // call is the setShardVersion request being sent by the config coordinator to the config server
diff --git a/src/mongo/dbtests/config_server_fixture.cpp b/src/mongo/dbtests/config_server_fixture.cpp
index 374e3c7827b..edc3019904e 100644
--- a/src/mongo/dbtests/config_server_fixture.cpp
+++ b/src/mongo/dbtests/config_server_fixture.cpp
@@ -79,7 +79,7 @@ void ConfigServerFixture::setUp() {
 
     const ConnectionString connStr(uassertStatusOK(ConnectionString::parse("$dummy:10000")));
 
-    ShardingState::get(getGlobalServiceContext())->initialize(connStr.toString());
+    ShardingState::get(getGlobalServiceContext())->initialize(&_txn, connStr.toString());
     ShardingState::get(getGlobalServiceContext())->setShardName(shardName());
 }
 
diff --git a/src/mongo/dbtests/framework.cpp b/src/mongo/dbtests/framework.cpp
index 9f181f49ac9..26f52257c9d 100644
--- a/src/mongo/dbtests/framework.cpp
+++ b/src/mongo/dbtests/framework.cpp
@@ -66,8 +66,11 @@ int runDbTests(int argc, char** argv) {
 
     getGlobalServiceContext()->initializeGlobalStorageEngine();
 
-    // Initialize the sharding state so we can run starding tests in isolation
-    ShardingState::get(getGlobalServiceContext())->initialize("$dummy:10000");
+    {
+        auto txn = cc().makeOperationContext();
+        // Initialize the sharding state so we can run sharding tests in isolation
+        ShardingState::get(getGlobalServiceContext())->initialize(txn.get(), "$dummy:10000");
+    }
 
     // Note: ShardingState::initialize also initializes the distLockMgr.
     {
diff --git a/src/mongo/s/catalog/catalog_manager.h b/src/mongo/s/catalog/catalog_manager.h
index b2e1dbe237f..66025e65c5a 100644
--- a/src/mongo/s/catalog/catalog_manager.h
+++ b/src/mongo/s/catalog/catalog_manager.h
@@ -94,7 +94,8 @@ public:
 
     /**
      * Performs implementation-specific startup tasks. Must be run after the catalog manager
-     * has been installed into the global 'grid' object.
+     * has been installed into the global 'grid' object. Implementation do not need to guarantee
+     * thread safety so callers should employ proper synchronization when calling this method.
      */
     virtual Status startup(OperationContext* txn, bool allowNetworking) = 0;
 
diff --git a/src/mongo/s/catalog/dist_lock_manager.h b/src/mongo/s/catalog/dist_lock_manager.h
index 883feaef843..973c33aa782 100644
--- a/src/mongo/s/catalog/dist_lock_manager.h
+++ b/src/mongo/s/catalog/dist_lock_manager.h
@@ -88,7 +88,17 @@ public:
 
     virtual ~DistLockManager() = default;
 
+    /**
+     * Performs bootstrapping for the manager. Implementation do not need to guarantee
+     * thread safety so callers should employ proper synchronization when calling this method.
+     */
     virtual void startUp() = 0;
+
+    /**
+     * Cleanup the manager's resources. Pass false to allowNetworking in order to do work that
+     * involves sending network messages. Implementation do not need to guarantee thread safety
+     * so callers should employ proper synchronization when calling this method.
+     */
     virtual void shutDown(bool allowNetworking) = 0;
 
     /**
diff --git a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
index 53c96a30f18..c8f913d1746 100644
--- a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
+++ b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
@@ -55,6 +55,9 @@ public:
      */
     Status init(const ConnectionString& configCS);
 
+    /**
+     * Can terminate the server if called more than once.
+     */
     Status startup(OperationContext* txn, bool allowNetworking) override;
 
     void shutDown(OperationContext* txn, bool allowNetworking) override;
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
index 5963d992e8b..483ffbdebb2 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
@@ -114,6 +114,7 @@ CatalogManagerReplicaSet::CatalogManagerReplicaSet(std::unique_ptr<DistLockManag
 CatalogManagerReplicaSet::~CatalogManagerReplicaSet() = default;
 
 Status CatalogManagerReplicaSet::startup(OperationContext* txn, bool allowNetworking) {
+    _distLockManager->startUp();
     return Status::OK();
 }
 
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set.h b/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
index 27a21adc638..88d6ed4a3e7 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
@@ -52,6 +52,9 @@ public:
         return ConfigServerMode::CSRS;
     }
 
+    /**
+     * Safe to call multiple times as long as they
+     */
     Status startup(OperationContext* txn, bool allowNetworking) override;
 
     void shutDown(OperationContext* txn, bool allowNetworking) override;
diff --git a/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp b/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
index 785bccdadef..4094b47ae66 100644
--- a/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
+++ b/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
@@ -70,7 +70,9 @@ ReplSetDistLockManager::ReplSetDistLockManager(ServiceContext* globalContext,
 ReplSetDistLockManager::~ReplSetDistLockManager() = default;
 
 void ReplSetDistLockManager::startUp() {
-    _execThread = stdx::make_unique<stdx::thread>(&ReplSetDistLockManager::doTask, this);
+    if (!_execThread) {
+        _execThread = stdx::make_unique<stdx::thread>(&ReplSetDistLockManager::doTask, this);
+    }
 }
 
 void ReplSetDistLockManager::shutDown(bool allowNetworking) {
diff --git a/src/mongo/s/d_migrate.cpp b/src/mongo/s/d_migrate.cpp
index 101ae6d077b..29443804d19 100644
--- a/src/mongo/s/d_migrate.cpp
+++ b/src/mongo/s/d_migrate.cpp
@@ -408,7 +408,7 @@ public:
             }
 
             const string configdb = cmdObj["configdb"].String();
-            shardingState->initialize(configdb);
+            shardingState->initialize(txn, configdb);
         }
 
         // Initialize our current shard name in the shard state if needed
@@ -1144,7 +1144,7 @@ public:
         if (!shardingState->enabled()) {
             if (!cmdObj["configServer"].eoo()) {
                 dassert(cmdObj["configServer"].type() == String);
-                shardingState->initialize(cmdObj["configServer"].String());
+                shardingState->initialize(txn, cmdObj["configServer"].String());
             } else {
                 errmsg = str::stream()
                     << "cannot start recv'ing chunk, "
diff --git a/src/mongo/s/d_split.cpp b/src/mongo/s/d_split.cpp
index 57780f47683..e203b635728 100644
--- a/src/mongo/s/d_split.cpp
+++ b/src/mongo/s/d_split.cpp
@@ -614,7 +614,7 @@ public:
             }
 
             const string configdb = cmdObj["configdb"].String();
-            shardingState->initialize(configdb);
+            shardingState->initialize(txn, configdb);
         }
 
         // Initialize our current shard name in the shard state if needed
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index 2decbd68615..401672f1d89 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -199,7 +199,7 @@ DBClientBase* createDirectClient(OperationContext* txn) {
 using namespace mongo;
 
 static Status initializeSharding(OperationContext* txn) {
-    Status status = initializeGlobalShardingState(mongosGlobalParams.configdbs);
+    Status status = initializeGlobalShardingState(txn, mongosGlobalParams.configdbs, true);
     if (!status.isOK()) {
         return status;
     }
@@ -210,11 +210,6 @@ static Status initializeSharding(OperationContext* txn) {
         return status;
     }
 
-    status = catalogManager->startup(txn, true);
-    if (!status.isOK()) {
-        return status;
-    }
-
     return Status::OK();
 }
 
diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp
index f31c3f63e4d..30bfc988586 100644
--- a/src/mongo/s/sharding_initialization.cpp
+++ b/src/mongo/s/sharding_initialization.cpp
@@ -67,7 +67,9 @@ std::unique_ptr<ThreadPoolTaskExecutor> makeTaskExecutor(std::unique_ptr<Network
 
 }  // namespace
 
-Status initializeGlobalShardingState(const ConnectionString& configCS) {
+Status initializeGlobalShardingState(OperationContext* txn,
+                                     const ConnectionString& configCS,
+                                     bool allowNetworking) {
     SyncClusterConnection::setConnectionValidationHook(
         [](const HostAndPort& target, const executor::RemoteCommandResponse& isMasterReply) {
             return ShardingNetworkConnectionHook::validateHostImpl(target, isMasterReply);
@@ -97,6 +99,11 @@ Status initializeGlobalShardingState(const ConnectionString& configCS) {
               std::move(shardRegistry),
               stdx::make_unique<ClusterCursorManager>(getGlobalServiceContext()->getClockSource()));
 
+    auto status = grid.catalogManager()->startup(txn, allowNetworking);
+    if (!status.isOK()) {
+        return status;
+    }
+
     return Status::OK();
 }
 
diff --git a/src/mongo/s/sharding_initialization.h b/src/mongo/s/sharding_initialization.h
index f57270d4f8e..691c5ee63f9 100644
--- a/src/mongo/s/sharding_initialization.h
+++ b/src/mongo/s/sharding_initialization.h
@@ -31,12 +31,15 @@
 namespace mongo {
 
 class ConnectionString;
+class OperationContext;
 class Status;
 
 /**
  * Takes in the connection string for reaching the config servers and initializes the global
  * CatalogManager, ShardingRegistry, and grid objects.
  */
-Status initializeGlobalShardingState(const ConnectionString& configCS);
+Status initializeGlobalShardingState(OperationContext* txn,
+                                     const ConnectionString& configCS,
+                                     bool allowNetworking);
 
 }  // namespace mongo