SERVER-48221 Shut down ftdc after storage engine

(cherry picked from commit 185000ad894d5cb95a3c946158712054db57cb7a)

7 files changed, 39 insertions, 12 deletions

diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 92280ce3d26..cbe4683784a 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -77,6 +77,7 @@
 #include "mongo/db/exec/working_set_common.h"
 #include "mongo/db/free_mon/free_mon_mongod.h"
 #include "mongo/db/ftdc/ftdc_mongod.h"
+#include "mongo/db/ftdc/util.h"
 #include "mongo/db/global_settings.h"
 #include "mongo/db/index/index_access_method_factory_impl.h"
 #include "mongo/db/index_builds_coordinator_mongod.h"
@@ -1156,11 +1157,13 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
         repl::ReplicationStateTransitionLockGuard rstl(
             opCtx, MODE_X, repl::ReplicationStateTransitionLockGuard::EnqueueOnly());
 
-        // Kill all operations. And, makes all newly created opCtx to be immediately interrupted.
-        // After this point, the opCtx will have been marked as killed and will not be usable other
-        // than to kill all transactions directly below.
+        // Kill all operations except FTDC to continue gathering metrics. This makes all newly
+        // created opCtx to be immediately interrupted. After this point, the opCtx will have been
+        // marked as killed and will not be usable other than to kill all transactions directly
+        // below.
         LOGV2_OPTIONS(4784912, {LogComponent::kDefault}, "Killing all operations for shutdown");
-        serviceContext->setKillAllOperations();
+        const std::set<std::string> excludedClients = {std::string(kFTDCThreadName)};
+        serviceContext->setKillAllOperations(excludedClients);
 
         // Destroy all stashed transaction resources, in order to release locks.
         LOGV2_OPTIONS(4784913, {LogComponent::kCommand}, "Shutting down all open transactions");
@@ -1257,10 +1260,6 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
     LOGV2_OPTIONS(4784925, {LogComponent::kControl}, "Shutting down free monitoring");
     stopFreeMonitoring();
 
-    // Shutdown Full-Time Data Capture
-    LOGV2_OPTIONS(4784926, {LogComponent::kFTDC}, "Shutting down full-time data capture");
-    stopMongoDFTDC();
-
     LOGV2(4784927, "Shutting down the HealthLog");
     HealthLog::get(serviceContext).shutdown();
 
@@ -1285,6 +1284,10 @@ void shutdownTask(const ShutdownTaskArgs& shutdownArgs) {
     LOGV2_OPTIONS(4784931, {LogComponent::kDefault}, "Dropping the scope cache for shutdown");
     ScriptEngine::dropScopeCache();
 
+    // Shutdown Full-Time Data Capture
+    LOGV2_OPTIONS(4784926, {LogComponent::kFTDC}, "Shutting down full-time data capture");
+    stopMongoDFTDC();
+
     LOGV2_OPTIONS(20565, {LogComponent::kControl}, "Now exiting");
 
     audit::logShutdown(client);
diff --git a/src/mongo/db/ftdc/controller.cpp b/src/mongo/db/ftdc/controller.cpp
index 4486ec7e345..907186b076b 100644
--- a/src/mongo/db/ftdc/controller.cpp
+++ b/src/mongo/db/ftdc/controller.cpp
@@ -191,7 +191,7 @@ void FTDCController::stop() {
 void FTDCController::doLoop() noexcept {
     // Note: All exceptions thrown in this loop are considered process fatal. The default terminate
     // is used to provide a good stack trace of the issue.
-    Client::initThread("ftdc");
+    Client::initThread(kFTDCThreadName);
     Client* client = &cc();
 
     // Update config
diff --git a/src/mongo/db/ftdc/util.h b/src/mongo/db/ftdc/util.h
index 4d47c610559..428a1c48a27 100644
--- a/src/mongo/db/ftdc/util.h
+++ b/src/mongo/db/ftdc/util.h
@@ -39,6 +39,8 @@
 
 namespace mongo {
 
+constexpr StringData kFTDCThreadName = "ftdc"_sd;
+
 /**
  * Utilities for inflating and deflating BSON documents and metric arrays
  */
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index c00e792c139..c3c0a029010 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -2931,6 +2931,10 @@ Status ReplicationCoordinatorImpl::processReplSetGetStatus(
         ReplicationMetrics::get(getServiceContext()).getElectionParticipantMetricsBSON();
 
     stdx::lock_guard<Latch> lk(_mutex);
+    if (_inShutdown) {
+        return Status(ErrorCodes::ShutdownInProgress, "shutdown in progress");
+    }
+
     Status result(ErrorCodes::InternalError, "didn't set status in prepareStatusResponse");
     _topCoord->prepareStatusResponse(
         TopologyCoordinator::ReplSetStatusArgs{
diff --git a/src/mongo/db/service_context.cpp b/src/mongo/db/service_context.cpp
index 9d9f907aa1b..45cacbb5786 100644
--- a/src/mongo/db/service_context.cpp
+++ b/src/mongo/db/service_context.cpp
@@ -319,7 +319,7 @@ Client* ServiceContext::LockedClientsCursor::next() {
     return result;
 }
 
-void ServiceContext::setKillAllOperations() {
+void ServiceContext::setKillAllOperations(const std::set<std::string>& excludedClients) {
     stdx::lock_guard<Latch> clientLock(_mutex);
 
     // Ensure that all newly created operation contexts will immediately be in the interrupted state
@@ -329,6 +329,12 @@ void ServiceContext::setKillAllOperations() {
     // Interrupt all active operations
     for (auto&& client : _clients) {
         stdx::lock_guard<Client> lk(*client);
+
+        // Do not kill operations from the excluded clients.
+        if (excludedClients.find(client->desc()) != excludedClients.end()) {
+            continue;
+        }
+
         auto opCtxToKill = client->getOperationContext();
         if (opCtxToKill) {
             killOperation(lk, opCtxToKill, ErrorCodes::InterruptedAtShutdown);
diff --git a/src/mongo/db/service_context.h b/src/mongo/db/service_context.h
index 49cb7a602ba..b93e97e4c2b 100644
--- a/src/mongo/db/service_context.h
+++ b/src/mongo/db/service_context.h
@@ -404,9 +404,10 @@ public:
     //
 
     /**
-     * Signal all OperationContext(s) that they have been killed.
+     * Signal all OperationContext(s) that they have been killed except the ones belonging to the
+     * excluded clients.
      */
-    void setKillAllOperations();
+    void setKillAllOperations(const std::set<std::string>& excludedClients = {});
 
     /**
      * Reset the operation kill state after a killAllOperations.
diff --git a/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp b/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
index 71d2421f5e5..195070ee745 100644
--- a/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
+++ b/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
@@ -27,12 +27,15 @@
  *    it in the license file.
  */
 
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kFTDC
+
 #include "mongo/platform/basic.h"
 
 #include "mongo/db/catalog/database_holder.h"
 #include "mongo/db/commands/server_status.h"
 #include "mongo/db/db_raii.h"
 #include "mongo/db/namespace_string.h"
+#include "mongo/logv2/log.h"
 
 namespace mongo {
 class OplogStonesServerStatusSection : public ServerStatusSection {
@@ -54,6 +57,14 @@ public:
         if (!opCtx->getServiceContext()->getStorageEngine()->supportsOplogStones()) {
             return builder.obj();
         }
+
+        Lock::GlobalLock lk(
+            opCtx, LockMode::MODE_IS, Date_t::now(), Lock::InterruptBehavior::kLeaveUnlocked);
+        if (!lk.isLocked()) {
+            LOGV2_DEBUG(4822100, 2, "Failed to retrieve oplogTruncation statistics");
+            return BSONObj();
+        }
+
         AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
         auto oplog = oplogRead.getCollection();
         if (oplog) {