SERVER-63822 Change TicketHolder implementation to use a FIFO queueing system

10 files changed, 284 insertions, 68 deletions

diff --git a/src/mongo/db/storage/SConscript b/src/mongo/db/storage/SConscript
index ead54123d8f..ebfa82e9951 100644
--- a/src/mongo/db/storage/SConscript
+++ b/src/mongo/db/storage/SConscript
@@ -175,6 +175,7 @@ env.Library(
     target='storage_options',
     source=[
         'storage_options.cpp',
+        'storage_parameters.cpp',
         'storage_parameters.idl',
     ],
     LIBDEPS=[
@@ -183,6 +184,17 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/idl/server_parameter',
+        'ticketholders'
+    ],
+)
+
+env.Library(
+    target='ticketholders',
+    source=[
+        'ticketholders.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
     ],
 )
 
diff --git a/src/mongo/db/storage/storage_parameters.cpp b/src/mongo/db/storage/storage_parameters.cpp
new file mode 100644
index 00000000000..da14416936d
--- /dev/null
+++ b/src/mongo/db/storage/storage_parameters.cpp
@@ -0,0 +1,56 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/storage/storage_parameters.h"
+#include "mongo/db/storage/ticketholders.h"
+#include "mongo/util/concurrency/ticketholder.h"
+
+namespace mongo {
+
+QueueingPolicyEnum gTicketQueueingPolicy{QueueingPolicyEnum::Semaphore};
+
+void TicketQueueingPolicy::append(OperationContext*,
+                                  BSONObjBuilder& builder,
+                                  const std::string& name) {
+    builder.append(name, QueueingPolicy_serializer(gTicketQueueingPolicy));
+}
+
+Status TicketQueueingPolicy::setFromString(const std::string& protocolStr) {
+    if (protocolStr == QueueingPolicy_serializer(QueueingPolicyEnum::Semaphore)) {
+        gTicketQueueingPolicy = QueueingPolicyEnum::Semaphore;
+    } else if (protocolStr == QueueingPolicy_serializer(QueueingPolicyEnum::FifoQueue)) {
+        gTicketQueueingPolicy = QueueingPolicyEnum::FifoQueue;
+    } else {
+        return Status{ErrorCodes::BadValue,
+                      str::stream() << "Unrecognized ticketQueueingPolicy '" << protocolStr << "'"};
+    }
+    return Status::OK();
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/storage_parameters.h b/src/mongo/db/storage/storage_parameters.h
new file mode 100644
index 00000000000..d7395e51d2d
--- /dev/null
+++ b/src/mongo/db/storage/storage_parameters.h
@@ -0,0 +1,38 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/storage/storage_parameters_gen.h"
+
+namespace mongo {
+
+extern QueueingPolicyEnum gTicketQueueingPolicy;
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/storage_parameters.idl b/src/mongo/db/storage/storage_parameters.idl
index b6a4f3a4ab4..d46525f113e 100644
--- a/src/mongo/db/storage/storage_parameters.idl
+++ b/src/mongo/db/storage/storage_parameters.idl
@@ -30,6 +30,7 @@ global:
     cpp_namespace: "mongo"
     cpp_includes:
         - "mongo/bson/bson_depth.h"
+        - "mongo/db/storage/ticketholders.h"
 
 server_parameters:
     notablescan:
@@ -114,6 +115,39 @@ server_parameters:
         cpp_vartype: bool
         default: false
 
+    # The wiredTigerQueueingPolicy specifies which policy to use and is essentially
+    # the same as the values of the enum QueueingPolicy
+    storageEngineQueueingPolicy:
+        description: "Storage Engine queueing policy to use"
+        set_at: [ startup ]
+        cpp_class:
+            name: TicketQueueingPolicy
+
+    storageEngineConcurrentWriteTransactions:
+        description: "Storage Engine Concurrent Write Transactions"
+        set_at: [ startup, runtime ]
+        cpp_vartype: AtomicWord<int>
+        cpp_varname: gConcurrentWriteTransactions
+        on_update: "TicketHolders::updateConcurrentWriteTransactions"
+        deprecated_name:
+            - wiredTigerConcurrentWriteTransactions
+        # Default value being 0 means we're allowing the underlying storage engines to use their default values.
+        default: 0
+        validator:
+            gt: 0
+
+    storageEngineConcurrentReadTransactions:
+        description: "Storage Engine Concurrent Read Transactions"
+        set_at: [ startup, runtime ]
+        cpp_vartype: AtomicWord<int>
+        cpp_varname: gConcurrentReadTransactions
+        on_update: "TicketHolders::updateConcurrentReadTransactions"
+        deprecated_name:
+            - wiredTigerConcurrentReadTransactions
+        # Default value being 0 means we're allowing the underlying storage engines to use their default values.
+        default: 0
+        validator:
+            gt: 0
 
 feature_flags:
     featureFlagClusteredIndexes:
@@ -157,3 +191,11 @@ feature_flags:
         description: "When enabled, allow the use of the $listCatalog aggregation stage"
         cpp_varname: feature_flags::gDocumentSourceListCatalog
         default: false
+
+enums:
+    QueueingPolicy:
+        description: Queueing policy to use for obtaining tickets
+        type: string
+        values:
+            Semaphore: semaphore
+            FifoQueue: fifoQueue
diff --git a/src/mongo/db/storage/ticketholders.cpp b/src/mongo/db/storage/ticketholders.cpp
new file mode 100644
index 00000000000..834453e1a34
--- /dev/null
+++ b/src/mongo/db/storage/ticketholders.cpp
@@ -0,0 +1,54 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/storage/ticketholders.h"
+#include "mongo/util/concurrency/ticketholder.h"
+
+namespace mongo {
+
+std::unique_ptr<TicketHolder> TicketHolders::openReadTransaction =
+    std::unique_ptr<TicketHolder>(nullptr);
+std::unique_ptr<TicketHolder> TicketHolders::openWriteTransaction =
+    std::unique_ptr<TicketHolder>(nullptr);
+
+Status TicketHolders::updateConcurrentWriteTransactions(const int& newWriteTransactions) {
+    if (openWriteTransaction) {
+        return openWriteTransaction->resize(newWriteTransactions);
+    }
+    return Status::OK();
+};
+
+Status TicketHolders::updateConcurrentReadTransactions(const int& newReadTransactions) {
+    if (openReadTransaction) {
+        return openReadTransaction->resize(newReadTransactions);
+    }
+    return Status::OK();
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/ticketholders.h b/src/mongo/db/storage/ticketholders.h
new file mode 100644
index 00000000000..241e31d0157
--- /dev/null
+++ b/src/mongo/db/storage/ticketholders.h
@@ -0,0 +1,48 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/base/status.h"
+#include <memory>
+
+namespace mongo {
+
+class TicketHolder;
+
+struct TicketHolders {
+    static std::unique_ptr<TicketHolder> openWriteTransaction;
+    static std::unique_ptr<TicketHolder> openReadTransaction;
+
+    static Status updateConcurrentWriteTransactions(const int& newWriteTransactions);
+
+    static Status updateConcurrentReadTransactions(const int& newReadTransactions);
+};
+
+}  // namespace mongo
diff --git a/src/mongo/db/storage/wiredtiger/SConscript b/src/mongo/db/storage/wiredtiger/SConscript
index bbda707a308..74cbdc1a06f 100644
--- a/src/mongo/db/storage/wiredtiger/SConscript
+++ b/src/mongo/db/storage/wiredtiger/SConscript
@@ -87,6 +87,7 @@ wtEnv.Library(
         '$BUILD_DIR/mongo/db/snapshot_window_options',
         '$BUILD_DIR/mongo/db/storage/backup_block',
         '$BUILD_DIR/mongo/db/storage/storage_repair_observer',
+        '$BUILD_DIR/mongo/db/storage/ticketholders',
         '$BUILD_DIR/mongo/util/log_and_backoff',
         '$BUILD_DIR/mongo/util/options_parser/options_parser',
         'oplog_stone_parameters',
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
index e169682e6d4..335b12200be 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_kv_engine.cpp
@@ -76,6 +76,7 @@
 #include "mongo/db/storage/key_format.h"
 #include "mongo/db/storage/storage_file_util.h"
 #include "mongo/db/storage/storage_options.h"
+#include "mongo/db/storage/storage_parameters.h"
 #include "mongo/db/storage/storage_parameters_gen.h"
 #include "mongo/db/storage/storage_repair_observer.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_cursor.h"
@@ -286,53 +287,6 @@ std::string toString(const StorageEngine::OldestActiveTransactionTimestampResult
     }
 }
 
-namespace {
-SemaphoreTicketHolder openWriteTransaction(128);
-SemaphoreTicketHolder openReadTransaction(128);
-}  // namespace
-
-OpenWriteTransactionParam::OpenWriteTransactionParam(StringData name, ServerParameterType spt)
-    : ServerParameter(name, spt), _data(&openWriteTransaction) {}
-
-void OpenWriteTransactionParam::append(OperationContext* opCtx,
-                                       BSONObjBuilder& b,
-                                       const std::string& name) {
-    b.append(name, _data->outof());
-}
-
-Status OpenWriteTransactionParam::setFromString(const std::string& str) {
-    int num = 0;
-    Status status = NumberParser{}(str, &num);
-    if (!status.isOK()) {
-        return status;
-    }
-    if (num <= 0) {
-        return {ErrorCodes::BadValue, str::stream() << name() << " has to be > 0"};
-    }
-    return _data->resize(num);
-}
-
-OpenReadTransactionParam::OpenReadTransactionParam(StringData name, ServerParameterType spt)
-    : ServerParameter(name, spt), _data(&openReadTransaction) {}
-
-void OpenReadTransactionParam::append(OperationContext* opCtx,
-                                      BSONObjBuilder& b,
-                                      const std::string& name) {
-    b.append(name, _data->outof());
-}
-
-Status OpenReadTransactionParam::setFromString(const std::string& str) {
-    int num = 0;
-    Status status = NumberParser{}(str, &num);
-    if (!status.isOK()) {
-        return status;
-    }
-    if (num <= 0) {
-        return {ErrorCodes::BadValue, str::stream() << name() << " has to be > 0"};
-    }
-    return _data->resize(num);
-}
-
 StringData WiredTigerKVEngine::kTableUriPrefix = "table:"_sd;
 
 WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
@@ -616,7 +570,31 @@ WiredTigerKVEngine::WiredTigerKVEngine(const std::string& canonicalName,
 
     _sizeStorer = std::make_unique<WiredTigerSizeStorer>(_conn, _sizeStorerUri, _readOnly);
 
-    Locker::setGlobalThrottling(&openReadTransaction, &openWriteTransaction);
+    auto readTransactions = gConcurrentReadTransactions.load();
+    static constexpr auto DEFAULT_TICKETS_VALUE = 128;
+    readTransactions = readTransactions == 0 ? DEFAULT_TICKETS_VALUE : readTransactions;
+    auto writeTransactions = gConcurrentWriteTransactions.load();
+    writeTransactions = writeTransactions == 0 ? DEFAULT_TICKETS_VALUE : writeTransactions;
+
+    switch (gTicketQueueingPolicy) {
+        case QueueingPolicyEnum::Semaphore:
+            LOGV2_DEBUG(6382201, 1, "Using Semaphore-based ticketing scheduler");
+            TicketHolders::openReadTransaction =
+                std::make_unique<SemaphoreTicketHolder>(readTransactions);
+            TicketHolders::openWriteTransaction =
+                std::make_unique<SemaphoreTicketHolder>(writeTransactions);
+            break;
+        case QueueingPolicyEnum::FifoQueue:
+            LOGV2_DEBUG(6382200, 1, "Using FIFO queue-based ticketing scheduler");
+            TicketHolders::openReadTransaction =
+                std::make_unique<FifoTicketHolder>(readTransactions);
+            TicketHolders::openWriteTransaction =
+                std::make_unique<FifoTicketHolder>(writeTransactions);
+            break;
+    }
+
+    Locker::setGlobalThrottling(TicketHolders::openReadTransaction.get(),
+                                TicketHolders::openWriteTransaction.get());
 
     _runTimeConfigParam.reset(new WiredTigerEngineRuntimeConfigParameter(
         "wiredTigerEngineRuntimeConfig", ServerParameterType::kRuntimeOnly));
@@ -642,16 +620,16 @@ void WiredTigerKVEngine::appendGlobalStats(BSONObjBuilder& b) {
     BSONObjBuilder bb(b.subobjStart("concurrentTransactions"));
     {
         BSONObjBuilder bbb(bb.subobjStart("write"));
-        bbb.append("out", openWriteTransaction.used());
-        bbb.append("available", openWriteTransaction.available());
-        bbb.append("totalTickets", openWriteTransaction.outof());
+        bbb.append("out", TicketHolders::openWriteTransaction->used());
+        bbb.append("available", TicketHolders::openWriteTransaction->available());
+        bbb.append("totalTickets", TicketHolders::openWriteTransaction->outof());
         bbb.done();
     }
     {
         BSONObjBuilder bbb(bb.subobjStart("read"));
-        bbb.append("out", openReadTransaction.used());
-        bbb.append("available", openReadTransaction.available());
-        bbb.append("totalTickets", openReadTransaction.outof());
+        bbb.append("out", TicketHolders::openReadTransaction->used());
+        bbb.append("available", TicketHolders::openReadTransaction->available());
+        bbb.append("totalTickets", TicketHolders::openReadTransaction->outof());
         bbb.done();
     }
     bb.done();
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
index 1e6f7816f65..60aea459dc7 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.cpp
@@ -30,6 +30,7 @@
 
 #include "mongo/platform/basic.h"
 
+#include "mongo/db/storage/storage_parameters.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_global_options.h"
 #include "mongo/db/storage/wiredtiger/wiredtiger_parameters_gen.h"
 #include "mongo/logv2/log.h"
diff --git a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl
index 0c74f2ec1a5..7a26b0190e5 100644
--- a/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl
+++ b/src/mongo/db/storage/wiredtiger/wiredtiger_parameters.idl
@@ -34,20 +34,6 @@ global:
         - "mongo/util/debug_util.h"
 
 server_parameters:
-    wiredTigerConcurrentWriteTransactions:
-        description: "WiredTiger Concurrent Write Transactions"
-        set_at: [ startup, runtime ]
-        cpp_class:
-            name: OpenWriteTransactionParam
-            data: 'TicketHolder*'
-            override_ctor: true
-    wiredTigerConcurrentReadTransactions:
-        description: "WiredTiger Concurrent Read Transactions"
-        set_at: [ startup, runtime ]
-        cpp_class:
-            name: OpenReadTransactionParam
-            data: 'TicketHolder*'
-            override_ctor: true
     wiredTigerEngineRuntimeConfig:
         description: 'WiredTiger Configuration'
         set_at: runtime