SERVER-64170 Move ticket holders from LockManager to TicketHolders

10 files changed, 111 insertions, 95 deletions

diff --git a/src/mongo/db/concurrency/SConscript b/src/mongo/db/concurrency/SConscript
index 97be34b25aa..848de10e74e 100644
--- a/src/mongo/db/concurrency/SConscript
+++ b/src/mongo/db/concurrency/SConscript
@@ -62,6 +62,7 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/service_context',
+        '$BUILD_DIR/mongo/db/storage/storage_engine_parameters',
         '$BUILD_DIR/mongo/util/background_job',
         '$BUILD_DIR/mongo/util/concurrency/spin_lock',
         '$BUILD_DIR/mongo/util/concurrency/ticketholder',
diff --git a/src/mongo/db/concurrency/d_concurrency_test.cpp b/src/mongo/db/concurrency/d_concurrency_test.cpp
index 9b359b9b6c4..905f3c9a366 100644
--- a/src/mongo/db/concurrency/d_concurrency_test.cpp
+++ b/src/mongo/db/concurrency/d_concurrency_test.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/concurrency/write_conflict_exception.h"
 #include "mongo/db/service_context_d_test_fixture.h"
 #include "mongo/db/storage/recovery_unit_noop.h"
+#include "mongo/db/storage/ticketholders.h"
 #include "mongo/logv2/log.h"
 #include "mongo/stdx/future.h"
 #include "mongo/stdx/thread.h"
@@ -69,19 +70,15 @@ const auto kMaxClockJitterMillis = Milliseconds(0);
  */
 class UseGlobalThrottling {
 public:
-    explicit UseGlobalThrottling(OperationContext* opCtx, int numTickets) : _opCtx(opCtx) {
-        auto lockManager = LockManager::get(_opCtx);
-        lockManager->setTicketHolders(std::make_unique<SemaphoreTicketHolder>(numTickets),
-                                      std::make_unique<SemaphoreTicketHolder>(numTickets));
+    explicit UseGlobalThrottling(OperationContext* opCtx, int numTickets) {
+        auto& ticketHolders = ticketHoldersDecoration(getGlobalServiceContext());
+        ticketHolders.setGlobalThrottling(std::make_unique<SemaphoreTicketHolder>(numTickets),
+                                          std::make_unique<SemaphoreTicketHolder>(numTickets));
     }
     ~UseGlobalThrottling() noexcept(false) {
-        // Reset the global setting as we're about to destroy the ticket holder.
-        auto lockManager = LockManager::get(_opCtx);
-        lockManager->setTicketHolders(nullptr, nullptr);
+        auto& ticketHolders = ticketHoldersDecoration(getGlobalServiceContext());
+        ticketHolders.setGlobalThrottling(nullptr, nullptr);
     }
-
-private:
-    OperationContext* _opCtx;
 };
 
 
diff --git a/src/mongo/db/concurrency/lock_manager.cpp b/src/mongo/db/concurrency/lock_manager.cpp
index 5650d282ee6..0737febeb25 100644
--- a/src/mongo/db/concurrency/lock_manager.cpp
+++ b/src/mongo/db/concurrency/lock_manager.cpp
@@ -47,7 +47,6 @@
 #include "mongo/db/service_context.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/assert_util.h"
-#include "mongo/util/concurrency/ticketholder.h"
 #include "mongo/util/decorable.h"
 #include "mongo/util/str.h"
 #include "mongo/util/timer.h"
@@ -879,24 +878,6 @@ void LockManager::getLockInfoBSON(const std::map<LockerId, BSONObj>& lockToClien
     _buildLocksArray(lockToClientMap, false, this, &lockInfoArr);
 }
 
-void LockManager::setTicketHolders(std::unique_ptr<TicketHolder> reading,
-                                   std::unique_ptr<TicketHolder> writing) {
-    _readingTicketholder = std::move(reading);
-    _writingTicketholder = std::move(writing);
-}
-
-TicketHolder* LockManager::getTicketHolder(LockMode mode) {
-    switch (mode) {
-        case MODE_IS:
-        case MODE_S:
-            return _readingTicketholder.get();
-        case MODE_IX:
-            return _writingTicketholder.get();
-        default:
-            return nullptr;
-    }
-}
-
 void LockManager::_buildLocksArray(const std::map<LockerId, BSONObj>& lockToClientMap,
                                    bool forLogging,
                                    LockManager* mutableThis,
diff --git a/src/mongo/db/concurrency/lock_manager.h b/src/mongo/db/concurrency/lock_manager.h
index 594e42057b8..e54524489c1 100644
--- a/src/mongo/db/concurrency/lock_manager.h
+++ b/src/mongo/db/concurrency/lock_manager.h
@@ -49,7 +49,6 @@ namespace mongo {
 
 class OperationContext;
 class ServiceContext;
-class TicketHolder;
 
 /**
  * Entry point for the lock manager scheduling functionality. Don't use it directly, but
@@ -168,18 +167,6 @@ public:
     void getLockInfoBSON(const std::map<LockerId, BSONObj>& lockToClientMap,
                          BSONObjBuilder* result);
 
-    /**
-     * Sets the TicketHolder implementation to use to obtain tickets from 'reading' (for MODE_S and
-     * MODE_IS), and from 'writing' (for MODE_IX) in order to throttle database access. There is no
-     * throttling for MODE_X, as there can only ever be a single locker using this mode. The
-     * throttling is intended to defend against large drops in throughput under high load due to too
-     * much concurrency.
-     */
-    void setTicketHolders(std::unique_ptr<TicketHolder> readTickets,
-                          std::unique_ptr<TicketHolder> writeTickets);
-
-    TicketHolder* getTicketHolder(LockMode mode);
-
 private:
     // The lockheads need access to the partitions
     friend struct LockHead;
@@ -250,8 +237,5 @@ private:
 
     static const unsigned _numPartitions;
     Partition* _partitions;
-
-    std::unique_ptr<TicketHolder> _readingTicketholder;
-    std::unique_ptr<TicketHolder> _writingTicketholder;
 };
 }  // namespace mongo
diff --git a/src/mongo/db/concurrency/lock_state.cpp b/src/mongo/db/concurrency/lock_state.cpp
index d1f6059ea6c..2675b35759e 100644
--- a/src/mongo/db/concurrency/lock_state.cpp
+++ b/src/mongo/db/concurrency/lock_state.cpp
@@ -41,6 +41,7 @@
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/storage/flow_control.h"
+#include "mongo/db/storage/ticketholders.h"
 #include "mongo/logv2/log.h"
 #include "mongo/platform/compiler.h"
 #include "mongo/stdx/new.h"
@@ -359,8 +360,8 @@ void LockerImpl::reacquireTicket(OperationContext* opCtx) {
 
 bool LockerImpl::_acquireTicket(OperationContext* opCtx, LockMode mode, Date_t deadline) {
     const bool reader = isSharedLockMode(mode);
-    auto lockManager = getGlobalLockManager();
-    auto holder = shouldAcquireTicket() ? lockManager->getTicketHolder(mode) : nullptr;
+    auto& ticketHolders = ticketHoldersDecoration(getGlobalServiceContext());
+    auto holder = shouldAcquireTicket() ? ticketHolders.getTicketHolder(mode) : nullptr;
     if (holder) {
         _clientState.store(reader ? kQueuedReader : kQueuedWriter);
 
@@ -1069,8 +1070,8 @@ void LockerImpl::releaseTicket() {
 }
 
 void LockerImpl::_releaseTicket() {
-    auto ticketManager = getGlobalLockManager();
-    auto holder = shouldAcquireTicket() ? ticketManager->getTicketHolder(_modeForTicket) : nullptr;
+    auto& ticketHolders = ticketHoldersDecoration(getGlobalServiceContext());
+    auto holder = shouldAcquireTicket() ? ticketHolders.getTicketHolder(_modeForTicket) : nullptr;
     if (holder) {
         holder->release();
     }
diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index a78827b2446..22c96f38cba 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -195,7 +195,6 @@ env.Library(
     ],
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
-        '$BUILD_DIR/mongo/db/concurrency/lock_manager',
         '$BUILD_DIR/mongo/db/service_context',
         '$BUILD_DIR/mongo/idl/server_parameter',
     ],
diff --git a/src/mongo/db/storage/storage_engine_init.cpp b/src/mongo/db/storage/storage_engine_init.cpp
index 1e93bcac3d4..f5749385ac2 100644
--- a/src/mongo/db/storage/storage_engine_init.cpp
+++ b/src/mongo/db/storage/storage_engine_init.cpp
@@ -45,11 +45,14 @@
 #include "mongo/db/storage/storage_engine_change_context.h"
 #include "mongo/db/storage/storage_engine_lock_file.h"
 #include "mongo/db/storage/storage_engine_metadata.h"
+#include "mongo/db/storage/storage_engine_parameters.h"
+#include "mongo/db/storage/storage_engine_parameters_gen.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/storage_repair_observer.h"
 #include "mongo/logv2/log.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/concurrency/ticketholder.h"
 #include "mongo/util/str.h"
 
 namespace mongo {
@@ -156,6 +159,32 @@ StorageEngine::LastShutdownState initializeStorageEngine(OperationContext* opCtx
         uassertStatusOK(factory->validateMetadata(*metadata, storageGlobalParams));
     }
 
+    if (storageGlobalParams.engine != "ephemeralForTest") {
+        auto readTransactions = gConcurrentReadTransactions.load();
+        static constexpr auto DEFAULT_TICKETS_VALUE = 128;
+        readTransactions = readTransactions == 0 ? DEFAULT_TICKETS_VALUE : readTransactions;
+        auto writeTransactions = gConcurrentWriteTransactions.load();
+        writeTransactions = writeTransactions == 0 ? DEFAULT_TICKETS_VALUE : writeTransactions;
+
+        // TODO SERVER-64467: Remove the globalServiceContext for TicketHolders
+        auto serviceContext = getGlobalServiceContext();
+        auto& ticketHolders = ticketHoldersDecoration(serviceContext);
+        switch (gTicketQueueingPolicy) {
+            case QueueingPolicyEnum::Semaphore:
+                LOGV2_DEBUG(6382201, 1, "Using Semaphore-based ticketing scheduler");
+                ticketHolders.setGlobalThrottling(
+                    std::make_unique<SemaphoreTicketHolder>(readTransactions),
+                    std::make_unique<SemaphoreTicketHolder>(writeTransactions));
+                break;
+            case QueueingPolicyEnum::FifoQueue:
+                LOGV2_DEBUG(6382200, 1, "Using FIFO queue-based ticketing scheduler");
+                ticketHolders.setGlobalThrottling(
+                    std::make_unique<FifoTicketHolder>(readTransactions),
+                    std::make_unique<FifoTicketHolder>(writeTransactions));
+                break;
+        }
+    }
+
     ScopeGuard guard([&] {
         auto& lockFile = StorageEngineLockFile::get(service);
         if (lockFile) {
@@ -214,6 +243,13 @@ void shutdownGlobalStorageEngineCleanly(ServiceContext* service, Status errorToR
         lockFile->clearPidAndUnlock();
         lockFile = boost::none;
     }
+    // TODO SERVER-64467: Remove the globalServiceContext for TicketHolders
+    // Cleanup the ticket holders.
+    if (hasGlobalServiceContext()) {
+        auto serviceContext = getGlobalServiceContext();
+        auto& ticketHolders = ticketHoldersDecoration(serviceContext);
+        ticketHolders.setGlobalThrottling(nullptr, nullptr);
+    }
 }
 } /* namespace */
 
diff --git a/src/mongo/db/storage/ticketholders.cpp b/src/mongo/db/storage/ticketholders.cpp
index 4702774a96e..1fcc8371946 100644
--- a/src/mongo/db/storage/ticketholders.cpp
+++ b/src/mongo/db/storage/ticketholders.cpp
@@ -28,7 +28,6 @@
  */
 
 #include "mongo/db/storage/ticketholders.h"
-#include "mongo/db/concurrency/lock_manager.h"
 #include "mongo/util/concurrency/ticketholder.h"
 
 namespace mongo {
@@ -36,10 +35,10 @@ namespace mongo {
 Status TicketHolders::updateConcurrentWriteTransactions(const int& newWriteTransactions) {
     if (hasGlobalServiceContext()) {
         auto serviceContext = getGlobalServiceContext();
-        auto lockManager = LockManager::get(serviceContext);
-        auto ticketHolder = lockManager->getTicketHolder(LockMode::MODE_IX);
-        if (ticketHolder) {
-            return ticketHolder->resize(newWriteTransactions);
+        auto& ticketHolders = ticketHoldersDecoration(serviceContext);
+        auto& writer = ticketHolders._openWriteTransaction;
+        if (writer) {
+            return writer->resize(newWriteTransactions);
         }
     }
     return Status::OK();
@@ -48,13 +47,35 @@ Status TicketHolders::updateConcurrentWriteTransactions(const int& newWriteTrans
 Status TicketHolders::updateConcurrentReadTransactions(const int& newReadTransactions) {
     if (hasGlobalServiceContext()) {
         auto serviceContext = getGlobalServiceContext();
-        auto lockManager = LockManager::get(serviceContext);
-        auto ticketHolder = lockManager->getTicketHolder(LockMode::MODE_IS);
-        if (ticketHolder) {
-            return ticketHolder->resize(newReadTransactions);
+        auto& ticketHolders = ticketHoldersDecoration(serviceContext);
+        auto& reader = ticketHolders._openReadTransaction;
+        if (reader) {
+            return reader->resize(newReadTransactions);
         }
     }
     return Status::OK();
-};
+}
+
+void TicketHolders::setGlobalThrottling(std::unique_ptr<TicketHolder> reading,
+                                        std::unique_ptr<TicketHolder> writing) {
+    _openReadTransaction = std::move(reading);
+    _openWriteTransaction = std::move(writing);
+}
+
+TicketHolder* TicketHolders::getTicketHolder(LockMode mode) {
+    switch (mode) {
+        case MODE_S:
+        case MODE_IS:
+            return _openReadTransaction.get();
+        case MODE_IX:
+            return _openWriteTransaction.get();
+        default:
+            return nullptr;
+    }
+}
+
+const Decorable<ServiceContext>::Decoration<TicketHolders> ticketHoldersDecoration =
+    ServiceContext::declareDecoration<TicketHolders>();
+
 
 }  // namespace mongo
diff --git a/src/mongo/db/storage/ticketholders.h b/src/mongo/db/storage/ticketholders.h
index 4fd4d877936..31fdfb8db32 100644
--- a/src/mongo/db/storage/ticketholders.h
+++ b/src/mongo/db/storage/ticketholders.h
@@ -30,15 +30,41 @@
 #pragma once
 
 #include "mongo/base/status.h"
+#include "mongo/db/concurrency/lock_manager_defs.h"
+#include "mongo/db/service_context.h"
 
 #include <memory>
 
 namespace mongo {
 
-struct TicketHolders {
+class TicketHolder;
+
+class TicketHolders {
+public:
     static Status updateConcurrentWriteTransactions(const int& newWriteTransactions);
 
     static Status updateConcurrentReadTransactions(const int& newReadTransactions);
+
+    /**
+     * Sets the TicketHolder implementation to use to obtain tickets from 'reading' (for MODE_S and
+     * MODE_IS), and from 'writing' (for MODE_IX) in order to throttle database access. There is no
+     * throttling for MODE_X, as there can only ever be a single locker using this mode. The
+     * throttling is intended to defend against large drops in throughput under high load due to too
+     * much concurrency.
+     */
+    void setGlobalThrottling(std::unique_ptr<TicketHolder> reading,
+                             std::unique_ptr<TicketHolder> writing);
+
+    TicketHolder* getTicketHolder(LockMode mode);
+
+private:
+    std::unique_ptr<TicketHolder> _openWriteTransaction;
+    std::unique_ptr<TicketHolder> _openReadTransaction;
 };
 
+/**
+ * Decorated accessor to the 'TicketHolders' stored in 'ServiceContext'.
+ */
+extern const Decorable<ServiceContext>::Decoration<TicketHolders> ticketHoldersDecoration;
+
 }  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index 9f6970013c2..68fd69eb132 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -74,12 +74,11 @@
 #include "mongo/db/snapshot_window_options_gen.h"
 #include "mongo/db/storage/journal_listener.h"
 #include "mongo/db/storage/key_format.h"
-#include "mongo/db/storage/storage_engine_parameters.h"
-#include "mongo/db/storage/storage_engine_parameters_gen.h"
 #include "mongo/db/storage/storage_file_util.h"
 #include "mongo/db/storage/storage_options.h"
 #include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/storage_repair_observer.h"
+#include "mongo/db/storage/ticketholders.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_cursor.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_customization_hooks.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_extensions.h"
@@ -572,28 +571,6 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
 
     _sizeStorer = std::make_unique<WiredTigerSizeStorer>(_conn, _sizeStorerUri, _readOnly);
 
-    auto readTransactions = gConcurrentReadTransactions.load();
-    static constexpr auto DEFAULT_TICKETS_VALUE = 128;
-    readTransactions = readTransactions == 0 ? DEFAULT_TICKETS_VALUE : readTransactions;
-    auto writeTransactions = gConcurrentWriteTransactions.load();
-    writeTransactions = writeTransactions == 0 ? DEFAULT_TICKETS_VALUE : writeTransactions;
-
-    auto serviceContext = getGlobalServiceContext();
-    auto lockManager = LockManager::get(serviceContext);
-    switch (gTicketQueueingPolicy) {
-        case QueueingPolicyEnum::Semaphore:
-            LOGV2_DEBUG(6382201, 1, "Using Semaphore-based ticketing scheduler");
-            lockManager->setTicketHolders(
-                std::make_unique<SemaphoreTicketHolder>(readTransactions),
-                std::make_unique<SemaphoreTicketHolder>(writeTransactions));
-            break;
-        case QueueingPolicyEnum::FifoQueue:
-            LOGV2_DEBUG(6382200, 1, "Using FIFO queue-based ticketing scheduler");
-            lockManager->setTicketHolders(std::make_unique<FifoTicketHolder>(readTransactions),
-                                          std::make_unique<FifoTicketHolder>(writeTransactions));
-            break;
-    }
-
     _runTimeConfigParam.reset(makeServerParameter<WiredTigerEngineRuntimeConfigParameter>(
         "wiredTigerEngineRuntimeConfig", ServerParameterType::kRuntimeOnly));
     _runTimeConfigParam->_data.second = this;
@@ -606,13 +583,6 @@ WiredTigerKVEngine::~WiredTigerKVEngine() {
 
     cleanShutdown();
 
-    // Cleanup the ticket holders.
-    if (hasGlobalServiceContext()) {
-        auto serviceContext = getGlobalServiceContext();
-        auto lockManager = LockManager::get(serviceContext);
-        lockManager->setTicketHolders(nullptr, nullptr);
-    }
-
     _sessionCache.reset(nullptr);
 }
 
@@ -624,10 +594,9 @@ void WiredTigerKVEngine::notifyStartupComplete() {
 void WiredTigerKVEngine::appendGlobalStats(BSONObjBuilder& b) {
     BSONObjBuilder bb(b.subobjStart("concurrentTransactions"));
     auto serviceContext = getGlobalServiceContext();
-    auto lockManager = LockManager::get(serviceContext);
-    auto writer = lockManager->getTicketHolder(MODE_IX);
-    auto reader = lockManager->getTicketHolder(MODE_IS);
+    auto& ticketHolders = ticketHoldersDecoration(serviceContext);
     {
+        auto writer = ticketHolders.getTicketHolder(MODE_IX);
         BSONObjBuilder bbb(bb.subobjStart("write"));
         bbb.append("out", writer->used());
         bbb.append("available", writer->available());
@@ -635,6 +604,7 @@ void WiredTigerKVEngine::appendGlobalStats(BSONObjBuilder& b) {
         bbb.done();
     }
     {
+        auto reader = ticketHolders.getTicketHolder(MODE_IS);
         BSONObjBuilder bbb(bb.subobjStart("read"));
         bbb.append("out", reader->used());
         bbb.append("available", reader->available());