SERVER-59074 Do not acquire storage tickets just to set/wait on oplog visibility

(cherry picked from commit 0dd96157fb1fffc8eefe6be4b26aff9805bfbfac)

4 files changed, 51 insertions, 8 deletions

diff --git a/src/mongo/db/concurrency/lock_state.h b/src/mongo/db/concurrency/lock_state.h
index 39298f3ac84..07c39b631fd 100644
--- a/src/mongo/db/concurrency/lock_state.h
+++ b/src/mongo/db/concurrency/lock_state.h
@@ -404,6 +404,35 @@ public:
 };
 
 /**
+ * RAII-style class to opt out of the ticket acquisition mechanism when acquiring a global lock.
+ *
+ * Operations that acquire the global lock but do not use any storage engine resources are eligible
+ * to skip ticket acquisition. Otherwise, a ticket acquisition is required to prevent throughput
+ * from suffering under high load.
+ */
+class SkipTicketAcquisitionForLock {
+public:
+    SkipTicketAcquisitionForLock(const SkipTicketAcquisitionForLock&) = delete;
+    SkipTicketAcquisitionForLock& operator=(const SkipTicketAcquisitionForLock&) = delete;
+    explicit SkipTicketAcquisitionForLock(OperationContext* opCtx)
+        : _opCtx(opCtx), _shouldAcquireTicket(_opCtx->lockState()->shouldAcquireTicket()) {
+        if (_shouldAcquireTicket) {
+            _opCtx->lockState()->skipAcquireTicket();
+        }
+    }
+
+    ~SkipTicketAcquisitionForLock() {
+        if (_shouldAcquireTicket) {
+            _opCtx->lockState()->setAcquireTicket();
+        }
+    }
+
+private:
+    OperationContext* _opCtx;
+    const bool _shouldAcquireTicket;
+};
+
+/**
  * Retrieves the global lock manager instance.
  */
 LockManager* getGlobalLockManager();
diff --git a/src/mongo/db/concurrency/locker.h b/src/mongo/db/concurrency/locker.h
index 48674103bd5..1ed35919092 100644
--- a/src/mongo/db/concurrency/locker.h
+++ b/src/mongo/db/concurrency/locker.h
@@ -479,14 +479,19 @@ public:
     }
 
     /**
-     * This will opt out of the ticket mechanism. This should be used sparingly for special purpose
-     * threads, such as FTDC and committing or aborting prepared transactions.
+     * This will opt in or out of the ticket mechanism. This should be used sparingly for special
+     * purpose threads, such as FTDC and committing or aborting prepared transactions.
      */
     void skipAcquireTicket() {
         // Should not hold or wait for the ticket.
         invariant(isNoop() || getClientState() == Locker::ClientState::kInactive);
         _shouldAcquireTicket = false;
     }
+    void setAcquireTicket() {
+        // Should hold or wait for the ticket.
+        invariant(isNoop() || getClientState() == Locker::ClientState::kInactive);
+        _shouldAcquireTicket = true;
+    }
 
     bool shouldAcquireTicket() const {
         return _shouldAcquireTicket;
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
index 4b3ffc8f101..834bd679109 100644
--- a/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_external_state_impl.cpp
@@ -452,14 +452,14 @@ Status ReplicationCoordinatorExternalStateImpl::initializeReplSetStorage(Operati
                                const auto msgObj = BSON("msg" << kInitiatingSetMsg);
                                _service->getOpObserver()->onOpMessage(opCtx, msgObj);
                                wuow.commit();
-                               // ReplSetTest assumes that immediately after the replSetInitiate
-                               // command returns, it can allow other nodes to initial sync with no
-                               // retries and they will succeed.  Unfortunately, initial sync will
-                               // fail if it finds its sync source has an empty oplog.  Thus, we
-                               // need to wait here until the seed document is visible in our oplog.
-                               _storageInterface->waitForAllEarlierOplogWritesToBeVisible(opCtx);
                            });
 
+        // ReplSetTest assumes that immediately after the replSetInitiate command returns, it can
+        // allow other nodes to initial sync with no retries and they will succeed.  Unfortunately,
+        // initial sync will fail if it finds its sync source has an empty oplog.  Thus, we need to
+        // wait here until the seed document is visible in our oplog.
+        _storageInterface->waitForAllEarlierOplogWritesToBeVisible(opCtx);
+
         FeatureCompatibilityVersion::setIfCleanStartup(opCtx, _storageInterface);
     } catch (const DBException& ex) {
         return ex.toStatus();
diff --git a/src/mongo/db/repl/storage_interface_impl.cpp b/src/mongo/db/repl/storage_interface_impl.cpp
index 329a0ee03a0..b11ab834531 100644
--- a/src/mongo/db/repl/storage_interface_impl.cpp
+++ b/src/mongo/db/repl/storage_interface_impl.cpp
@@ -52,6 +52,7 @@
 #include "mongo/db/catalog/index_catalog.h"
 #include "mongo/db/client.h"
 #include "mongo/db/concurrency/d_concurrency.h"
+#include "mongo/db/concurrency/lock_state.h"
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/curop.h"
 #include "mongo/db/db_raii.h"
@@ -1308,6 +1309,10 @@ Status StorageInterfaceImpl::isAdminDbValid(OperationContext* opCtx) {
 
 void StorageInterfaceImpl::waitForAllEarlierOplogWritesToBeVisible(OperationContext* opCtx,
                                                                    bool primaryOnly) {
+    // Waiting for oplog writes to be visible in the oplog does not use any storage engine resources
+    // and must skip ticket acquisition to avoid deadlocks with updating oplog visibility.
+    SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx);
+
     AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
     if (primaryOnly &&
         !repl::ReplicationCoordinator::get(opCtx)->canAcceptWritesForDatabase(opCtx, "admin"))
@@ -1320,6 +1325,10 @@ void StorageInterfaceImpl::waitForAllEarlierOplogWritesToBeVisible(OperationCont
 void StorageInterfaceImpl::oplogDiskLocRegister(OperationContext* opCtx,
                                                 const Timestamp& ts,
                                                 bool orderedCommit) {
+    // Setting the oplog visibility does not use any storage engine resources and must skip ticket
+    // acquisition to avoid deadlocks with updating oplog visibility.
+    SkipTicketAcquisitionForLock skipTicketAcquisition(opCtx);
+
     AutoGetOplog oplogRead(opCtx, OplogAccessMode::kRead);
     fassert(28557,
             oplogRead.getCollection()->getRecordStore()->oplogDiskLocRegister(