SERVER-19543 Differentiate between shutting down catalog manager for replacement and for process shutdown.

20 files changed, 43 insertions, 32 deletions

diff --git a/src/mongo/s/catalog/catalog_manager.h b/src/mongo/s/catalog/catalog_manager.h
index bed79840a62..47bbb6bf8e6 100644
--- a/src/mongo/s/catalog/catalog_manager.h
+++ b/src/mongo/s/catalog/catalog_manager.h
@@ -101,7 +101,7 @@ public:
     /**
      * Performs necessary cleanup when shutting down cleanly.
      */
-    virtual void shutDown() = 0;
+    virtual void shutDown(bool allowNetworking = true) = 0;
 
     /**
      * Returns what type of catalog manager this is - CSRS for the CatalogManagerReplicaSet and
diff --git a/src/mongo/s/catalog/catalog_manager_mock.cpp b/src/mongo/s/catalog/catalog_manager_mock.cpp
index ba866ab088e..628274e9038 100644
--- a/src/mongo/s/catalog/catalog_manager_mock.cpp
+++ b/src/mongo/s/catalog/catalog_manager_mock.cpp
@@ -49,7 +49,7 @@ Status CatalogManagerMock::startup() {
     return Status::OK();
 }
 
-void CatalogManagerMock::shutDown() {}
+void CatalogManagerMock::shutDown(bool allowNetworking) {}
 
 Status CatalogManagerMock::shardCollection(OperationContext* txn,
                                            const string& ns,
diff --git a/src/mongo/s/catalog/catalog_manager_mock.h b/src/mongo/s/catalog/catalog_manager_mock.h
index 33322cb9ad4..56026c2ac0a 100644
--- a/src/mongo/s/catalog/catalog_manager_mock.h
+++ b/src/mongo/s/catalog/catalog_manager_mock.h
@@ -47,7 +47,7 @@ public:
 
     Status startup() override;
 
-    void shutDown() override;
+    void shutDown(bool allowNetworking) override;
 
     Status shardCollection(OperationContext* txn,
                            const std::string& ns,
diff --git a/src/mongo/s/catalog/dist_lock_manager.h b/src/mongo/s/catalog/dist_lock_manager.h
index 5fbaf3583a4..883feaef843 100644
--- a/src/mongo/s/catalog/dist_lock_manager.h
+++ b/src/mongo/s/catalog/dist_lock_manager.h
@@ -89,7 +89,7 @@ public:
     virtual ~DistLockManager() = default;
 
     virtual void startUp() = 0;
-    virtual void shutDown() = 0;
+    virtual void shutDown(bool allowNetworking) = 0;
 
     /**
      * Tries multiple times to lock, using the specified lock try interval, until
diff --git a/src/mongo/s/catalog/dist_lock_manager_mock.cpp b/src/mongo/s/catalog/dist_lock_manager_mock.cpp
index 2c28aea9a0e..28dcbd5fa67 100644
--- a/src/mongo/s/catalog/dist_lock_manager_mock.cpp
+++ b/src/mongo/s/catalog/dist_lock_manager_mock.cpp
@@ -59,7 +59,7 @@ DistLockManagerMock::DistLockManagerMock()
 
 void DistLockManagerMock::startUp() {}
 
-void DistLockManagerMock::shutDown() {
+void DistLockManagerMock::shutDown(bool allowNetworking) {
     uassert(28659, "DistLockManagerMock shut down with outstanding locks present", _locks.empty());
 }
 
diff --git a/src/mongo/s/catalog/dist_lock_manager_mock.h b/src/mongo/s/catalog/dist_lock_manager_mock.h
index 6b9f9e87801..fa3f05357c2 100644
--- a/src/mongo/s/catalog/dist_lock_manager_mock.h
+++ b/src/mongo/s/catalog/dist_lock_manager_mock.h
@@ -43,7 +43,7 @@ public:
     virtual ~DistLockManagerMock() = default;
 
     virtual void startUp() override;
-    virtual void shutDown() override;
+    virtual void shutDown(bool allowNetworking) override;
 
     virtual StatusWith<DistLockManager::ScopedDistLock> lock(
         StringData name,
diff --git a/src/mongo/s/catalog/forwarding_catalog_manager.cpp b/src/mongo/s/catalog/forwarding_catalog_manager.cpp
index 53b57a45012..fb7047a8b5d 100644
--- a/src/mongo/s/catalog/forwarding_catalog_manager.cpp
+++ b/src/mongo/s/catalog/forwarding_catalog_manager.cpp
@@ -97,8 +97,11 @@ Status ForwardingCatalogManager::startup() {
     return retry([this] { return _actual->startup(); });
 }
 
-void ForwardingCatalogManager::shutDown() {
-    _actual->shutDown();
+void ForwardingCatalogManager::shutDown(bool allowNetworking) {
+    retry([this, allowNetworking] {
+        _actual->shutDown(allowNetworking);
+        return 1;
+    });
 }
 
 Status ForwardingCatalogManager::enableSharding(const std::string& dbName) {
diff --git a/src/mongo/s/catalog/forwarding_catalog_manager.h b/src/mongo/s/catalog/forwarding_catalog_manager.h
index ae66faba376..b646d663ccb 100644
--- a/src/mongo/s/catalog/forwarding_catalog_manager.h
+++ b/src/mongo/s/catalog/forwarding_catalog_manager.h
@@ -61,7 +61,7 @@ public:
 
     Status startup() override;
 
-    void shutDown() override;
+    void shutDown(bool allowNetworking = true) override;
 
     Status enableSharding(const std::string& dbName) override;
 
diff --git a/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp b/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp
index b8dc5c05f08..6c2715b05b1 100644
--- a/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp
+++ b/src/mongo/s/catalog/legacy/catalog_manager_legacy.cpp
@@ -239,7 +239,7 @@ Status CatalogManagerLegacy::_startConfigServerChecker() {
     return Status::OK();
 }
 
-void CatalogManagerLegacy::shutDown() {
+void CatalogManagerLegacy::shutDown(bool allowNetworking) {
     LOG(1) << "CatalogManagerLegacy::shutDown() called.";
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
@@ -252,7 +252,7 @@ void CatalogManagerLegacy::shutDown() {
         _consistencyCheckerThread.join();
 
     invariant(_distLockManager);
-    _distLockManager->shutDown();
+    _distLockManager->shutDown(allowNetworking);
 }
 
 Status CatalogManagerLegacy::shardCollection(OperationContext* txn,
diff --git a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
index f6a6eb282d4..16c5ac5ad48 100644
--- a/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
+++ b/src/mongo/s/catalog/legacy/catalog_manager_legacy.h
@@ -57,7 +57,7 @@ public:
 
     Status startup() override;
 
-    void shutDown() override;
+    void shutDown(bool allowNetworking) override;
 
     Status shardCollection(OperationContext* txn,
                            const std::string& ns,
diff --git a/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.cpp b/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.cpp
index aec1f596311..b7824d66d47 100644
--- a/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.cpp
+++ b/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.cpp
@@ -58,7 +58,7 @@ void LegacyDistLockManager::startUp() {
     _pinger = stdx::make_unique<LegacyDistLockPinger>();
 }
 
-void LegacyDistLockManager::shutDown() {
+void LegacyDistLockManager::shutDown(bool allowNetworking) {
     stdx::unique_lock<stdx::mutex> sl(_mutex);
     _isStopped = true;
 
@@ -67,7 +67,7 @@ void LegacyDistLockManager::shutDown() {
     }
 
     if (_pinger) {
-        _pinger->shutdown();
+        _pinger->shutdown(allowNetworking);
     }
 }
 
diff --git a/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.h b/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.h
index 238989a3cdb..8267dd44602 100644
--- a/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.h
+++ b/src/mongo/s/catalog/legacy/legacy_dist_lock_manager.h
@@ -50,7 +50,7 @@ public:
     virtual ~LegacyDistLockManager() = default;
 
     virtual void startUp() override;
-    virtual void shutDown() override;
+    virtual void shutDown(bool allowNetworking) override;
 
     virtual StatusWith<DistLockManager::ScopedDistLock> lock(
         StringData name,
diff --git a/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.cpp b/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.cpp
index 04457499b61..a534ab24319 100644
--- a/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.cpp
+++ b/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.cpp
@@ -287,10 +287,11 @@ void LegacyDistLockPinger::stopPing(const ConnectionString& conn, const string&
     }
 }
 
-void LegacyDistLockPinger::shutdown() {
+void LegacyDistLockPinger::shutdown(bool allowNetworking) {
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
         _inShutdown = true;
+        _allowNetworkingInShutdown = allowNetworking;
         _pingStoppedCV.notify_all();
     }
 
@@ -327,6 +328,10 @@ void LegacyDistLockPinger::acknowledgeStopPing(const ConnectionString& addr,
 
         _kill.erase(pingId);
         _seen.erase(pingId);
+
+        if (!_allowNetworkingInShutdown) {
+            return;
+        }
     }
 
     try {
diff --git a/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.h b/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.h
index 25955c58bdd..008b6bb69be 100644
--- a/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.h
+++ b/src/mongo/s/catalog/legacy/legacy_dist_lock_pinger.h
@@ -76,7 +76,7 @@ public:
     /**
      * Kills all ping threads and wait for them to cleanup.
      */
-    void shutdown();
+    void shutdown(bool allowNetworking);
 
 private:
     /**
@@ -136,6 +136,7 @@ private:
     // Contains all lock ids to keeping on retrying to unlock until success.
     std::list<DistLockHandle> _unlockList;  // (M)
 
-    bool _inShutdown = false;  // (M)
+    bool _inShutdown = false;                // (M)
+    bool _allowNetworkingInShutdown = true;  // (M)
 };
 }
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp b/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
index 0eca0fbb9bc..b0009f2905d 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set.cpp
@@ -117,7 +117,8 @@ Status CatalogManagerReplicaSet::startup() {
     return Status::OK();
 }
 
-void CatalogManagerReplicaSet::shutDown() {
+void CatalogManagerReplicaSet::shutDown(bool allowNetworking) {
+    invariant(allowNetworking);
     LOG(1) << "CatalogManagerReplicaSet::shutDown() called.";
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
@@ -125,7 +126,7 @@ void CatalogManagerReplicaSet::shutDown() {
     }
 
     invariant(_distLockManager);
-    _distLockManager->shutDown();
+    _distLockManager->shutDown(allowNetworking);
 }
 
 Status CatalogManagerReplicaSet::shardCollection(OperationContext* txn,
diff --git a/src/mongo/s/catalog/replset/catalog_manager_replica_set.h b/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
index 2650a8d2056..ceccfbb2869 100644
--- a/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
+++ b/src/mongo/s/catalog/replset/catalog_manager_replica_set.h
@@ -54,7 +54,7 @@ public:
 
     Status startup() override;
 
-    void shutDown() override;
+    void shutDown(bool allowNetworking) override;
 
     Status shardCollection(OperationContext* txn,
                            const std::string& ns,
diff --git a/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp b/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
index 9453db131fd..785bccdadef 100644
--- a/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
+++ b/src/mongo/s/catalog/replset/replset_dist_lock_manager.cpp
@@ -73,7 +73,8 @@ void ReplSetDistLockManager::startUp() {
     _execThread = stdx::make_unique<stdx::thread>(&ReplSetDistLockManager::doTask, this);
 }
 
-void ReplSetDistLockManager::shutDown() {
+void ReplSetDistLockManager::shutDown(bool allowNetworking) {
+    invariant(allowNetworking);
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
         _isShutDown = true;
diff --git a/src/mongo/s/catalog/replset/replset_dist_lock_manager.h b/src/mongo/s/catalog/replset/replset_dist_lock_manager.h
index 81da75af01e..31787811051 100644
--- a/src/mongo/s/catalog/replset/replset_dist_lock_manager.h
+++ b/src/mongo/s/catalog/replset/replset_dist_lock_manager.h
@@ -61,7 +61,7 @@ public:
     virtual ~ReplSetDistLockManager();
 
     virtual void startUp() override;
-    virtual void shutDown() override;
+    virtual void shutDown(bool allowNetworking) override;
 
     virtual StatusWith<DistLockManager::ScopedDistLock> lock(
         StringData name,
diff --git a/src/mongo/s/catalog/replset/replset_dist_lock_manager_test.cpp b/src/mongo/s/catalog/replset/replset_dist_lock_manager_test.cpp
index b0e64bf77f5..a30fce7cbb1 100644
--- a/src/mongo/s/catalog/replset/replset_dist_lock_manager_test.cpp
+++ b/src/mongo/s/catalog/replset/replset_dist_lock_manager_test.cpp
@@ -112,7 +112,7 @@ protected:
     void tearDown() override {
         // Don't care about what shutDown passes to stopPing here.
         _mockCatalog->expectStopPing([](StringData) {}, Status::OK());
-        _mgr.shutDown();
+        _mgr.shutDown(true);
     }
 
     TickSourceMock _tickSource;
@@ -447,7 +447,7 @@ TEST_F(RSDistLockMgrWithMockTickSource, LockFailsAfterRetry) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
@@ -588,7 +588,7 @@ TEST_F(ReplSetDistLockManagerFixture, MustUnlockOnLockError) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
@@ -638,7 +638,7 @@ TEST_F(ReplSetDistLockManagerFixture, LockPinging) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
@@ -721,7 +721,7 @@ TEST_F(ReplSetDistLockManagerFixture, UnlockUntilNoError) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
@@ -817,7 +817,7 @@ TEST_F(ReplSetDistLockManagerFixture, MultipleQueuedUnlock) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
@@ -841,7 +841,7 @@ TEST_F(ReplSetDistLockManagerFixture, CleanupPingOnShutdown) {
         stopPingCalled = true;
     }, Status::OK());
 
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
     ASSERT_TRUE(stopPingCalled);
 }
 
@@ -1605,7 +1605,7 @@ TEST_F(ReplSetDistLockManagerFixture, LockOvertakingResultsInError) {
     // Join the background thread before trying to call asserts. Shutdown calls
     // stopPing and we don't care in this test.
     getMockCatalog()->expectStopPing([](StringData) {}, Status::OK());
-    getMgr()->shutDown();
+    getMgr()->shutDown(true);
 
     // No assert until shutDown has been called to make sure that the background thread
     // won't be trying to access the local variables that were captured by lamdas that
diff --git a/src/mongo/s/distlock_test.cpp b/src/mongo/s/distlock_test.cpp
index 495d804b940..1017f2cfa51 100644
--- a/src/mongo/s/distlock_test.cpp
+++ b/src/mongo/s/distlock_test.cpp
@@ -339,7 +339,7 @@ public:
         result.append("errors", errors);
         result.append("timeMS", t.millis());
 
-        pinger.shutdown();
+        pinger.shutdown(true);
 
         return !errors;
     }