SERVER-41160 Add shutdown method for CatalogCacheLoader

12 files changed, 82 insertions, 13 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index e787f780827..3ba6eb79169 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -978,6 +978,10 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
         validator->shutDown();
     }
 
+    if (ShardingState::get(serviceContext)->enabled()) {
+        CatalogCacheLoader::get(serviceContext).shutDown();
+    }
+
 #if __has_feature(address_sanitizer)
     // When running under address sanitizer, we get false positive leaks due to disorder around
     // the lifecycle of a connection and request. When we are running under ASAN, we try a lot
diff --git a/src/mongo/db/s/catalog_cache_loader_mock.cpp b/src/mongo/db/s/catalog_cache_loader_mock.cpp
index a0112029736..406a16c26f6 100644
--- a/src/mongo/db/s/catalog_cache_loader_mock.cpp
+++ b/src/mongo/db/s/catalog_cache_loader_mock.cpp
@@ -82,6 +82,8 @@ void CatalogCacheLoaderMock::onStepUp() {
     MONGO_UNREACHABLE;
 }
 
+void CatalogCacheLoaderMock::shutDown() {}
+
 void CatalogCacheLoaderMock::notifyOfCollectionVersionUpdate(const NamespaceString& nss) {
     MONGO_UNREACHABLE;
 }
diff --git a/src/mongo/db/s/catalog_cache_loader_mock.h b/src/mongo/db/s/catalog_cache_loader_mock.h
index 692df021d22..b5a2f64a6b7 100644
--- a/src/mongo/db/s/catalog_cache_loader_mock.h
+++ b/src/mongo/db/s/catalog_cache_loader_mock.h
@@ -52,6 +52,7 @@ public:
     void initializeReplicaSetRole(bool isPrimary) override;
     void onStepDown() override;
     void onStepUp() override;
+    void shutDown() override;
     void notifyOfCollectionVersionUpdate(const NamespaceString& nss) override;
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;
     void waitForDatabaseFlush(OperationContext* opCtx, StringData dbName) override;
diff --git a/src/mongo/db/s/read_only_catalog_cache_loader.cpp b/src/mongo/db/s/read_only_catalog_cache_loader.cpp
index 0f173047844..b381c7e596d 100644
--- a/src/mongo/db/s/read_only_catalog_cache_loader.cpp
+++ b/src/mongo/db/s/read_only_catalog_cache_loader.cpp
@@ -35,6 +35,10 @@ namespace mongo {
 
 using CollectionAndChangedChunks = CatalogCacheLoader::CollectionAndChangedChunks;
 
+ReadOnlyCatalogCacheLoader::~ReadOnlyCatalogCacheLoader() {
+    shutDown();
+}
+
 void ReadOnlyCatalogCacheLoader::waitForCollectionFlush(OperationContext* opCtx,
                                                         const NamespaceString& nss) {
     MONGO_UNREACHABLE;
@@ -44,6 +48,10 @@ void ReadOnlyCatalogCacheLoader::waitForDatabaseFlush(OperationContext* opCtx, S
     MONGO_UNREACHABLE;
 }
 
+void ReadOnlyCatalogCacheLoader::shutDown() {
+    _configServerLoader.shutDown();
+}
+
 std::shared_ptr<Notification<void>> ReadOnlyCatalogCacheLoader::getChunksSince(
     const NamespaceString& nss, ChunkVersion version, GetChunksSinceCallbackFn callbackFn) {
     return _configServerLoader.getChunksSince(nss, version, callbackFn);
diff --git a/src/mongo/db/s/read_only_catalog_cache_loader.h b/src/mongo/db/s/read_only_catalog_cache_loader.h
index 03f05d1d7a5..83df1fd5114 100644
--- a/src/mongo/db/s/read_only_catalog_cache_loader.h
+++ b/src/mongo/db/s/read_only_catalog_cache_loader.h
@@ -40,9 +40,12 @@ namespace mongo {
  */
 class ReadOnlyCatalogCacheLoader final : public CatalogCacheLoader {
 public:
+    ~ReadOnlyCatalogCacheLoader();
+
     void initializeReplicaSetRole(bool isPrimary) override {}
     void onStepDown() override {}
     void onStepUp() override {}
+    void shutDown() override;
     void notifyOfCollectionVersionUpdate(const NamespaceString& nss) override {}
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;
     void waitForDatabaseFlush(OperationContext* opCtx, StringData dbName) override;
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
index 7266adfc233..db1aeb90804 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader.cpp
@@ -342,16 +342,7 @@ ShardServerCatalogCacheLoader::ShardServerCatalogCacheLoader(
 }
 
 ShardServerCatalogCacheLoader::~ShardServerCatalogCacheLoader() {
-    // Prevent further scheduling, then interrupt ongoing tasks.
-    _threadPool.shutdown();
-    {
-        stdx::lock_guard<stdx::mutex> lock(_mutex);
-        _contexts.interrupt(ErrorCodes::InterruptedAtShutdown);
-        ++_term;
-    }
-
-    _threadPool.join();
-    invariant(_contexts.isEmpty());
+    shutDown();
 }
 
 void ShardServerCatalogCacheLoader::notifyOfCollectionVersionUpdate(const NamespaceString& nss) {
@@ -384,6 +375,30 @@ void ShardServerCatalogCacheLoader::onStepUp() {
     _role = ReplicaSetRole::Primary;
 }
 
+void ShardServerCatalogCacheLoader::shutDown() {
+    {
+        stdx::lock_guard<stdx::mutex> lg(_mutex);
+        if (_inShutdown) {
+            return;
+        }
+
+        _inShutdown = true;
+    }
+
+    // Prevent further scheduling, then interrupt ongoing tasks.
+    _threadPool.shutdown();
+    {
+        stdx::lock_guard<stdx::mutex> lock(_mutex);
+        _contexts.interrupt(ErrorCodes::InterruptedAtShutdown);
+        ++_term;
+    }
+
+    _threadPool.join();
+    invariant(_contexts.isEmpty());
+
+    _configServerLoader->shutDown();
+}
+
 std::shared_ptr<Notification<void>> ShardServerCatalogCacheLoader::getChunksSince(
     const NamespaceString& nss, ChunkVersion version, GetChunksSinceCallbackFn callbackFn) {
     auto notify = std::make_shared<Notification<void>>();
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader.h b/src/mongo/db/s/shard_server_catalog_cache_loader.h
index b6472941e99..2a456c5a9ef 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader.h
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader.h
@@ -69,6 +69,8 @@ public:
      */
     void onStepUp() override;
 
+    void shutDown() override;
+
     /**
      * Sets any notifications waiting for this version to arrive and invalidates the catalog cache's
      * chunk metadata for collection 'nss' so that the next caller provokes a refresh.
@@ -473,7 +475,7 @@ private:
 
     // Loader used by the shard primary to retrieve the authoritative routing metadata from the
     // config server
-    const std::unique_ptr<CatalogCacheLoader> _configServerLoader;
+    std::unique_ptr<CatalogCacheLoader> _configServerLoader;
 
     // Thread pool used to run blocking tasks which perform disk reads and writes
     ThreadPool _threadPool;
@@ -484,6 +486,9 @@ private:
     // Protects the class state below
     stdx::mutex _mutex;
 
+    // True if shutDown was called.
+    bool _inShutdown{false};
+
     // This value is bumped every time the set of currently scheduled tasks should no longer be
     // running. This includes, replica set state transitions and shutdown.
     long long _term{0};
diff --git a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp
index 368b4a52421..902b09c3a64 100644
--- a/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp
+++ b/src/mongo/db/s/shard_server_catalog_cache_loader_test.cpp
@@ -107,6 +107,7 @@ void ShardServerCatalogCacheLoaderTest::setUp() {
 }
 
 void ShardServerCatalogCacheLoaderTest::tearDown() {
+    _shardLoader->shutDown();
     _shardLoader.reset();
     ShardServerTestFixture::tearDown();
 }
diff --git a/src/mongo/s/catalog_cache_loader.h b/src/mongo/s/catalog_cache_loader.h
index 91a314bbabf..58cb081f2be 100644
--- a/src/mongo/s/catalog_cache_loader.h
+++ b/src/mongo/s/catalog_cache_loader.h
@@ -109,6 +109,12 @@ public:
     virtual void onStepUp() = 0;
 
     /**
+     * Transitions into shut down and cleans up state. Once this transitions to shut down, should
+     * not be able to transition back to normal. Should be safe to be called more than once.
+     */
+    virtual void shutDown() = 0;
+
+    /**
      * Notifies the loader that the persisted collection version for 'nss' has been updated.
      */
     virtual void notifyOfCollectionVersionUpdate(const NamespaceString& nss) = 0;
diff --git a/src/mongo/s/config_server_catalog_cache_loader.cpp b/src/mongo/s/config_server_catalog_cache_loader.cpp
index 202d8fdb852..de8fb1a493f 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.cpp
+++ b/src/mongo/s/config_server_catalog_cache_loader.cpp
@@ -143,8 +143,7 @@ ConfigServerCatalogCacheLoader::ConfigServerCatalogCacheLoader()
 }
 
 ConfigServerCatalogCacheLoader::~ConfigServerCatalogCacheLoader() {
-    _threadPool.shutdown();
-    _threadPool.join();
+    shutDown();
 }
 
 void ConfigServerCatalogCacheLoader::initializeReplicaSetRole(bool isPrimary) {
@@ -159,6 +158,20 @@ void ConfigServerCatalogCacheLoader::onStepUp() {
     MONGO_UNREACHABLE;
 }
 
+void ConfigServerCatalogCacheLoader::shutDown() {
+    {
+        stdx::lock_guard<stdx::mutex> lg(_mutex);
+        if (_inShutdown) {
+            return;
+        }
+
+        _inShutdown = true;
+    }
+
+    _threadPool.shutdown();
+    _threadPool.join();
+}
+
 void ConfigServerCatalogCacheLoader::notifyOfCollectionVersionUpdate(const NamespaceString& nss) {
     MONGO_UNREACHABLE;
 }
diff --git a/src/mongo/s/config_server_catalog_cache_loader.h b/src/mongo/s/config_server_catalog_cache_loader.h
index d70c86f0d88..81c81240100 100644
--- a/src/mongo/s/config_server_catalog_cache_loader.h
+++ b/src/mongo/s/config_server_catalog_cache_loader.h
@@ -45,6 +45,7 @@ public:
     void initializeReplicaSetRole(bool isPrimary) override;
     void onStepDown() override;
     void onStepUp() override;
+    void shutDown() override;
     void notifyOfCollectionVersionUpdate(const NamespaceString& nss) override;
     void waitForCollectionFlush(OperationContext* opCtx, const NamespaceString& nss) override;
     void waitForDatabaseFlush(OperationContext* opCtx, StringData dbName) override;
@@ -61,6 +62,12 @@ public:
 private:
     // Thread pool to be used to perform metadata load
     ThreadPool _threadPool;
+
+    // Protects the class state below
+    stdx::mutex _mutex;
+
+    // True if shutDown was called.
+    bool _inShutdown{false};
 };
 
 }  // namespace mongo
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index b31b90bde7d..71d54f49ccd 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -237,6 +237,10 @@ void cleanupTask(ServiceContext* serviceContext) {
             shardRegistry->shutdown();
         }
 
+        if (Grid::get(serviceContext)->isShardingInitialized()) {
+            CatalogCacheLoader::get(serviceContext).shutDown();
+        }
+
 #if __has_feature(address_sanitizer)
         // When running under address sanitizer, we get false positive leaks due to disorder around
         // the lifecycle of a connection and request. When we are running under ASAN, we try a lot