SERVER-70971 Skip RSTL lock acquisition during FTDC data capture

11 files changed, 166 insertions, 27 deletions

diff --git a/jstests/noPassthrough/serverStatus_does_not_block_on_RSTL.js b/jstests/noPassthrough/serverStatus_does_not_block_on_RSTL.js
new file mode 100644
index 00000000000..40eb22187fb
--- /dev/null
+++ b/jstests/noPassthrough/serverStatus_does_not_block_on_RSTL.js
@@ -0,0 +1,72 @@
+/**
+ * Tests that serverStatus is not blocked by an exclusive RSTL lock. Only enforcing on WT.
+ *
+ * @tags: [
+ *   # Certain serverStatus sections might pivot to taking the RSTL lock if an action is unsupported
+ *   # by a non-WT storage engine.
+ *   requires_wiredtiger,
+ *   # Replication requires journaling support so this tag also implies exclusion from --nojournal
+ *   # test configurations.
+ *   requires_sharding,
+ *   requires_replication,
+ * ]
+ */
+(function() {
+"use strict";
+
+load("jstests/libs/parallel_shell_helpers.js");  // startParallelShell
+load("jstests/libs/wait_for_command.js");        // waitForCommand
+
+// Use a sharding environment in order to exercise the sharding specific serverStatus sections.
+const st = new ShardingTest(
+    {mongos: 1, config: 1, shards: 1, rs: {nodes: 1, setParameter: {watchdogPeriodSeconds: 60}}});
+const testDB = st.rs0.getPrimary().getDB("test");
+
+jsTestLog("Starting the sleep command in a parallel thread to take the RSTL MODE_X lock");
+let rstlXLockSleepJoin = startParallelShell(() => {
+    jsTestLog("Parallel Shell: about to start sleep command");
+    assert.commandFailedWithCode(db.adminCommand({
+        sleep: 1,
+        secs: 60 * 60,
+        // RSTL MODE_X lock.
+        lockTarget: "RSTL",
+        $comment: "RSTL lock sleep"
+    }),
+                                 ErrorCodes.Interrupted);
+}, testDB.getMongo().port);
+
+jsTestLog("Waiting for the sleep command to start and fetch the opID");
+const sleepCmdOpID =
+    waitForCommand("RSTL lock", op => (op["command"]["$comment"] == "RSTL lock sleep"), testDB);
+
+jsTestLog("Wait for the sleep command to log that the RSTL MODE_X lock was acquired");
+checkLog.containsJson(testDB, 6001600);
+
+try {
+    jsTestLog("Running serverStatus concurrently with the RSTL X lock held by the sleep cmd");
+    const serverStatusResult = assert.commandWorked(testDB.adminCommand({
+        serverStatus: 1,
+        repl: 1,
+        mirroredReads: 1,
+        advisoryHostFQDNs: 1,
+        defaultRWConcern: 1,
+        heapProfile: 1,
+        http_client: 1,
+        latchAnalysis: 1,
+        opWriteConcernCounters: 1,
+        oplog: 1,
+        resourceConsumption: 1,
+        sharding: 1,
+        tenantMigrationAccessBlocker: 1,
+        watchdog: 1,
+        maxTimeMS: 20 * 1000
+    }));
+    jsTestLog("ServerStatus results: " + tojson(serverStatusResult));
+} finally {
+    jsTestLog("Ensure the sleep cmd releases the lock so that the server can shutdown");
+    assert.commandWorked(testDB.killOp(sleepCmdOpID));  // kill the sleep cmd
+    rstlXLockSleepJoin();  // wait for the thread running the sleep cmd to finish
+}
+
+st.stop();
+})();
diff --git a/src/mongo/db/catalog_raii.cpp b/src/mongo/db/catalog_raii.cpp
index 481eae8d513..18dc4d1419a 100644
--- a/src/mongo/db/catalog_raii.cpp
+++ b/src/mongo/db/catalog_raii.cpp
@@ -191,14 +191,18 @@ ReadSourceScope::~ReadSourceScope() {
     }
 }
 
-AutoGetOplog::AutoGetOplog(OperationContext* opCtx, OplogAccessMode mode, Date_t deadline)
+AutoGetOplog::AutoGetOplog(OperationContext* opCtx,
+                           OplogAccessMode mode,
+                           Date_t deadline,
+                           bool skipRSTLLock)
     : _shouldNotConflictWithSecondaryBatchApplicationBlock(opCtx->lockState()) {
     auto lockMode = (mode == OplogAccessMode::kRead) ? MODE_IS : MODE_IX;
     if (mode == OplogAccessMode::kLogOp) {
         // Invariant that global lock is already held for kLogOp mode.
         invariant(opCtx->lockState()->isWriteLocked());
     } else {
-        _globalLock.emplace(opCtx, lockMode, deadline, Lock::InterruptBehavior::kThrow);
+        _globalLock.emplace(
+            opCtx, lockMode, deadline, Lock::InterruptBehavior::kThrow, skipRSTLLock);
     }
 
     // Obtain database and collection intent locks for non-document-locking storage engines.
diff --git a/src/mongo/db/catalog_raii.h b/src/mongo/db/catalog_raii.h
index ef1ea02d454..817f6ff4c7e 100644
--- a/src/mongo/db/catalog_raii.h
+++ b/src/mongo/db/catalog_raii.h
@@ -274,7 +274,10 @@ class AutoGetOplog {
     AutoGetOplog& operator=(const AutoGetOplog&) = delete;
 
 public:
-    AutoGetOplog(OperationContext* opCtx, OplogAccessMode mode, Date_t deadline = Date_t::max());
+    AutoGetOplog(OperationContext* opCtx,
+                 OplogAccessMode mode,
+                 Date_t deadline = Date_t::max(),
+                 bool skipRSTLLock = false);
 
     /**
      * Return a pointer to the per-service-context LocalOplogInfo.
diff --git a/src/mongo/db/commands/sleep_command.cpp b/src/mongo/db/commands/sleep_command.cpp
index 625dab44b37..8475eaa75d6 100644
--- a/src/mongo/db/commands/sleep_command.cpp
+++ b/src/mongo/db/commands/sleep_command.cpp
@@ -78,6 +78,9 @@ public:
                       const StringData& ns) {
         if (ns.empty()) {
             Lock::GlobalLock lk(opCtx, mode, Date_t::max(), Lock::InterruptBehavior::kThrow);
+            LOGV2(6001601,
+                  "Global lock acquired by sleep command.",
+                  "lockMode"_attr = modeName(mode));
             opCtx->sleepFor(Milliseconds(millis));
             return;
         }
@@ -94,6 +97,9 @@ public:
         Lock::DBLock dbLock(opCtx, nss.db(), dbMode, Date_t::max());
 
         if (nsIsDbOnly(ns)) {
+            LOGV2(6001602,
+                  "Database lock acquired by sleep command.",
+                  "lockMode"_attr = modeName(dbMode));
             opCtx->sleepFor(Milliseconds(millis));
             return;
         }
@@ -103,6 +109,9 @@ public:
                 "lockTarget is not a valid namespace",
                 NamespaceString::validCollectionComponent(ns));
         Lock::CollectionLock collLock(opCtx, nss, mode, Date_t::max());
+        LOGV2(6001603,
+              "Collection lock acquired by sleep command.",
+              "lockMode"_attr = modeName(mode));
         opCtx->sleepFor(Milliseconds(millis));
     }
 
@@ -113,6 +122,14 @@ public:
         pbwm.unlock();
     }
 
+    void _sleepInRSTL(mongo::OperationContext* opCtx, long long millis) {
+        Lock::ResourceLock rstl(opCtx->lockState(), resourceIdReplicationStateTransitionLock);
+        rstl.lock(nullptr, MODE_X);
+        LOGV2(6001600, "RSTL MODE_X lock acquired by sleep command.");
+        opCtx->sleepFor(Milliseconds(millis));
+        rstl.unlock();
+    }
+
     CmdSleep() : BasicCommand("sleep") {}
     bool run(OperationContext* opCtx,
              const std::string& ns,
@@ -171,6 +188,11 @@ public:
                 continue;
             }
 
+            if (lockTarget == "RSTL") {
+                _sleepInRSTL(opCtx, msRemaining.count());
+                continue;
+            }
+
             if (!cmdObj["lock"]) {
                 // The caller may specify either 'w' as true or false to take a global X lock or
                 // global S lock, respectively.
diff --git a/src/mongo/db/concurrency/d_concurrency.cpp b/src/mongo/db/concurrency/d_concurrency.cpp
index fd1f809cc94..ab0e4d3866c 100644
--- a/src/mongo/db/concurrency/d_concurrency.cpp
+++ b/src/mongo/db/concurrency/d_concurrency.cpp
@@ -139,12 +139,14 @@ bool Lock::ResourceMutex::isAtLeastReadLocked(Locker* locker) {
 Lock::GlobalLock::GlobalLock(OperationContext* opCtx,
                              LockMode lockMode,
                              Date_t deadline,
-                             InterruptBehavior behavior)
+                             InterruptBehavior behavior,
+                             bool skipRSTLLock)
     : _opCtx(opCtx),
       _result(LOCK_INVALID),
       _pbwm(opCtx->lockState(), resourceIdParallelBatchWriterMode),
       _fcvLock(opCtx->lockState(), resourceIdFeatureCompatibilityVersion),
       _interruptBehavior(behavior),
+      _skipRSTLLock(skipRSTLLock),
       _isOutermostLock(!opCtx->lockState()->isLocked()) {
     _opCtx->lockState()->getFlowControlTicket(_opCtx, lockMode);
 
@@ -167,17 +169,14 @@ Lock::GlobalLock::GlobalLock(OperationContext* opCtx,
             }
         });
 
-        _opCtx->lockState()->lock(
-            _opCtx, resourceIdReplicationStateTransitionLock, MODE_IX, deadline);
-
-        auto unlockRSTL = makeGuard(
-            [this] { _opCtx->lockState()->unlock(resourceIdReplicationStateTransitionLock); });
-
         _result = LOCK_INVALID;
-        _opCtx->lockState()->lockGlobal(_opCtx, lockMode, deadline);
+        if (skipRSTLLock) {
+            _takeGlobalLockOnly(lockMode, deadline);
+        } else {
+            _takeGlobalAndRSTLLocks(lockMode, deadline);
+        }
         _result = LOCK_OK;
 
-        unlockRSTL.dismiss();
         unlockFCVLock.dismiss();
         unlockPBWM.dismiss();
     } catch (const ExceptionForCat<ErrorCategory::Interruption>&) {
@@ -189,12 +188,27 @@ Lock::GlobalLock::GlobalLock(OperationContext* opCtx,
     _opCtx->lockState()->setGlobalLockTakenInMode(acquiredLockMode);
 }
 
+void Lock::GlobalLock::_takeGlobalLockOnly(LockMode lockMode, Date_t deadline) {
+    _opCtx->lockState()->lockGlobal(_opCtx, lockMode, deadline);
+}
+
+void Lock::GlobalLock::_takeGlobalAndRSTLLocks(LockMode lockMode, Date_t deadline) {
+    _opCtx->lockState()->lock(_opCtx, resourceIdReplicationStateTransitionLock, MODE_IX, deadline);
+    auto unlockRSTL = makeGuard(
+        [this] { _opCtx->lockState()->unlock(resourceIdReplicationStateTransitionLock); });
+
+    _opCtx->lockState()->lockGlobal(_opCtx, lockMode, deadline);
+
+    unlockRSTL.dismiss();
+}
+
 Lock::GlobalLock::GlobalLock(GlobalLock&& otherLock)
     : _opCtx(otherLock._opCtx),
       _result(otherLock._result),
       _pbwm(std::move(otherLock._pbwm)),
       _fcvLock(std::move(otherLock._fcvLock)),
       _interruptBehavior(otherLock._interruptBehavior),
+      _skipRSTLLock(otherLock._skipRSTLLock),
       _isOutermostLock(otherLock._isOutermostLock) {
     // Mark as moved so the destructor doesn't invalidate the newly-constructed lock.
     otherLock._result = LOCK_INVALID;
diff --git a/src/mongo/db/concurrency/d_concurrency.h b/src/mongo/db/concurrency/d_concurrency.h
index d5268777695..866e9c0ece4 100644
--- a/src/mongo/db/concurrency/d_concurrency.h
+++ b/src/mongo/db/concurrency/d_concurrency.h
@@ -224,7 +224,8 @@ public:
         GlobalLock(OperationContext* opCtx,
                    LockMode lockMode,
                    Date_t deadline,
-                   InterruptBehavior behavior);
+                   InterruptBehavior behavior,
+                   bool skipRSTLLock = false);
 
         GlobalLock(GlobalLock&&);
 
@@ -242,7 +243,7 @@ public:
                 }
                 _unlock();
             }
-            if (lockResult == LOCK_OK || lockResult == LOCK_WAITING) {
+            if (!_skipRSTLLock && (lockResult == LOCK_OK || lockResult == LOCK_WAITING)) {
                 _opCtx->lockState()->unlock(resourceIdReplicationStateTransitionLock);
             }
         }
@@ -252,6 +253,12 @@ public:
         }
 
     private:
+        /**
+         * Constructor helper functions, to handle skipping or taking the RSTL lock.
+         */
+        void _takeGlobalLockOnly(LockMode lockMode, Date_t deadline);
+        void _takeGlobalAndRSTLLocks(LockMode lockMode, Date_t deadline);
+
         void _unlock();
 
         OperationContext* const _opCtx;
@@ -259,6 +266,7 @@ public:
         ResourceLock _pbwm;
         ResourceLock _fcvLock;
         InterruptBehavior _interruptBehavior;
+        bool _skipRSTLLock;
         const bool _isOutermostLock;
     };
 
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript
index 70159d87de0..00a99e83b3a 100644
--- a/src/mongo/db/repl/SConscript
+++ b/src/mongo/db/repl/SConscript
@@ -1143,6 +1143,7 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/commands/server_status',
+        '$BUILD_DIR/mongo/db/repl/local_oplog_info',
         '$BUILD_DIR/mongo/db/repl/speculative_authenticate',
         '$BUILD_DIR/mongo/db/stats/counters',
         '$BUILD_DIR/mongo/transport/message_compressor',
diff --git a/src/mongo/db/repl/replication_info.cpp b/src/mongo/db/repl/replication_info.cpp
index 0e827661882..0e5a4b90a3c 100644
--- a/src/mongo/db/repl/replication_info.cpp
+++ b/src/mongo/db/repl/replication_info.cpp
@@ -276,23 +276,28 @@ public:
         result.append("latestOptime", replCoord->getMyLastAppliedOpTime().getTimestamp());
 
         auto earliestOplogTimestampFetch = [&]() -> Timestamp {
-            AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
-            if (!oplogRead.getCollection()) {
-                return Timestamp();
-            }
-
             // Try to get the lock. If it's already locked, immediately return null timestamp.
-            Lock::GlobalLock lk(
-                opCtx, MODE_IS, Date_t::now(), Lock::InterruptBehavior::kLeaveUnlocked);
+            Lock::GlobalLock lk(opCtx,
+                                MODE_IS,
+                                Date_t::now(),
+                                Lock::InterruptBehavior::kLeaveUnlocked,
+                                /* Not sensitive to replication so we can skip RSTL locking*/
+                                true);
             if (!lk.isLocked()) {
                 LOGV2_DEBUG(
                     6294100, 2, "Failed to get global lock for oplog server status section");
                 return Timestamp();
             }
 
+            AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead, Date_t::max(), true);
+            auto oplog = oplogRead.getCollection();
+            if (!oplog) {
+                return Timestamp();
+            }
+
             // Try getting earliest oplog timestamp using getEarliestOplogTimestamp
             auto swEarliestOplogTimestamp =
-                oplogRead.getCollection()->getRecordStore()->getEarliestOplogTimestamp(opCtx);
+                oplog->getRecordStore()->getEarliestOplogTimestamp(opCtx);
 
             if (swEarliestOplogTimestamp.getStatus() == ErrorCodes::OplogOperationUnsupported) {
                 // Falling back to use getSingleton if the storage engine does not support
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index 3c0a9a77466..2744b5e0cd0 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -88,6 +88,7 @@ if wiredtiger:
             '$BUILD_DIR/mongo/db/catalog/database_holder',
             '$BUILD_DIR/mongo/db/commands/server_status',
             '$BUILD_DIR/mongo/db/mongod_options',
+            '$BUILD_DIR/mongo/db/repl/local_oplog_info',
             '$BUILD_DIR/mongo/db/snapshot_window_options',
             '$BUILD_DIR/mongo/db/storage/downgraded_unique_indexes',
             '$BUILD_DIR/mongo/db/storage/storage_repair_observer',
diff --git a/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp b/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
index 195070ee745..6caa85a02c0 100644
--- a/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
+++ b/src/mongo/db/storage/wiredtiger/oplog_stones_server_status_section.cpp
@@ -58,14 +58,19 @@ public:
             return builder.obj();
         }
 
-        Lock::GlobalLock lk(
-            opCtx, LockMode::MODE_IS, Date_t::now(), Lock::InterruptBehavior::kLeaveUnlocked);
+        Lock::GlobalLock lk(opCtx,
+                            LockMode::MODE_IS,
+                            Date_t::now(),
+                            Lock::InterruptBehavior::kLeaveUnlocked,
+                            // Replication state change does not affect the following operation.
+                            true /* skipRSTLLock */);
+
         if (!lk.isLocked()) {
             LOGV2_DEBUG(4822100, 2, "Failed to retrieve oplogTruncation statistics");
             return BSONObj();
         }
 
-        AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
+        AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead, Date_t::max(), true);
         auto oplog = oplogRead.getCollection();
         if (oplog) {
             const auto localDb =
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
index b05d7d0d552..89ca98b23fd 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_server_status.cpp
@@ -57,8 +57,12 @@ bool WiredTigerServerStatusSection::includeByDefault() const {
 
 BSONObj WiredTigerServerStatusSection::generateSection(OperationContext* opCtx,
                                                        const BSONElement& configElement) const {
-    Lock::GlobalLock lk(
-        opCtx, LockMode::MODE_IS, Date_t::now(), Lock::InterruptBehavior::kLeaveUnlocked);
+    Lock::GlobalLock lk(opCtx,
+                        LockMode::MODE_IS,
+                        Date_t::now(),
+                        Lock::InterruptBehavior::kLeaveUnlocked,
+                        // Replication state change does not affect the following operation.
+                        true /* skipRSTLLock */);
     if (!lk.isLocked()) {
         LOGV2_DEBUG(3088800, 2, "Failed to retrieve wiredTiger statistics");
         return BSONObj();