SERVER-25027 Configurable connpool in mongos

Export server parameters for sharding connection pool

(cherry picked from commit b6c29702d8dcadb6c1ee90a876fac4117e0ca062)

5 files changed, 58 insertions, 15 deletions

diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index 7b0e206469f..5f1b194dded 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -166,9 +166,11 @@ private:
     State _state;
 };
 
-Milliseconds const ConnectionPool::kDefaultRefreshTimeout = Seconds(20);
-Milliseconds const ConnectionPool::kDefaultRefreshRequirement = Seconds(60);
-Milliseconds const ConnectionPool::kDefaultHostTimeout = Minutes(5);
+constexpr Milliseconds ConnectionPool::kDefaultHostTimeout;
+size_t const ConnectionPool::kDefaultMaxConns = std::numeric_limits<size_t>::max();
+size_t const ConnectionPool::kDefaultMinConns = 1;
+constexpr Milliseconds ConnectionPool::kDefaultRefreshRequirement;
+constexpr Milliseconds ConnectionPool::kDefaultRefreshTimeout;
 
 const Status ConnectionPool::kConnectionStateUnknown =
     Status(ErrorCodes::InternalError, "Connection is in an unknown state");
diff --git a/src/mongo/executor/connection_pool.h b/src/mongo/executor/connection_pool.h
index a5bcc1c3925..ea4d9234cbf 100644
--- a/src/mongo/executor/connection_pool.h
+++ b/src/mongo/executor/connection_pool.h
@@ -67,9 +67,11 @@ public:
 
     using GetConnectionCallback = stdx::function<void(StatusWith<ConnectionHandle>)>;
 
-    static const Milliseconds kDefaultRefreshTimeout;
-    static const Milliseconds kDefaultRefreshRequirement;
-    static const Milliseconds kDefaultHostTimeout;
+    static constexpr Milliseconds kDefaultHostTimeout = Milliseconds(300000);  // 5mins
+    static const size_t kDefaultMaxConns;
+    static const size_t kDefaultMinConns;
+    static constexpr Milliseconds kDefaultRefreshRequirement = Milliseconds(60000);  // 1min
+    static constexpr Milliseconds kDefaultRefreshTimeout = Milliseconds(20000);      // 20secs
 
     static const Status kConnectionStateUnknown;
 
@@ -80,14 +82,14 @@ public:
          * The minimum number of connections to keep alive while the pool is in
          * operation
          */
-        size_t minConnections = 1;
+        size_t minConnections = kDefaultMinConns;
 
         /**
          * The maximum number of connections to spawn for a host. This includes
          * pending connections in setup and connections checked out of the pool
          * as well as the obvious live connections in the pool.
          */
-        size_t maxConnections = std::numeric_limits<size_t>::max();
+        size_t maxConnections = kDefaultMaxConns;
 
         /**
          * Amount of time to wait before timing out a refresh attempt
diff --git a/src/mongo/executor/network_interface_factory.cpp b/src/mongo/executor/network_interface_factory.cpp
index 8a3f0e8f7cc..2dc7d3f5845 100644
--- a/src/mongo/executor/network_interface_factory.cpp
+++ b/src/mongo/executor/network_interface_factory.cpp
@@ -36,6 +36,7 @@
 #include "mongo/executor/async_stream_factory.h"
 #include "mongo/executor/async_stream_interface.h"
 #include "mongo/executor/async_timer_asio.h"
+#include "mongo/executor/connection_pool.h"
 #include "mongo/executor/network_connection_hook.h"
 #include "mongo/executor/network_interface_asio.h"
 #include "mongo/rpc/metadata/metadata_hook.h"
@@ -52,12 +53,14 @@ std::unique_ptr<NetworkInterface> makeNetworkInterface(std::string instanceName)
 std::unique_ptr<NetworkInterface> makeNetworkInterface(
     std::string instanceName,
     std::unique_ptr<NetworkConnectionHook> hook,
-    std::unique_ptr<rpc::EgressMetadataHook> metadataHook) {
+    std::unique_ptr<rpc::EgressMetadataHook> metadataHook,
+    ConnectionPool::Options connPoolOptions) {
     NetworkInterfaceASIO::Options options{};
     options.instanceName = std::move(instanceName);
     options.networkConnectionHook = std::move(hook);
     options.metadataHook = std::move(metadataHook);
     options.timerFactory = stdx::make_unique<AsyncTimerFactoryASIO>();
+    options.connectionPoolOptions = connPoolOptions;
 
 #ifdef MONGO_CONFIG_SSL
     if (SSLManagerInterface* manager = getSSLManager()) {
diff --git a/src/mongo/executor/network_interface_factory.h b/src/mongo/executor/network_interface_factory.h
index 20fe061081a..7aa8a8cdce5 100644
--- a/src/mongo/executor/network_interface_factory.h
+++ b/src/mongo/executor/network_interface_factory.h
@@ -31,6 +31,7 @@
 #include <memory>
 #include <string>
 
+#include "mongo/executor/connection_pool.h"
 #include "mongo/executor/network_interface.h"
 
 namespace mongo {
@@ -54,7 +55,8 @@ std::unique_ptr<NetworkInterface> makeNetworkInterface(std::string instanceName)
 std::unique_ptr<NetworkInterface> makeNetworkInterface(
     std::string instanceName,
     std::unique_ptr<NetworkConnectionHook> hook,
-    std::unique_ptr<rpc::EgressMetadataHook> metadataHook);
+    std::unique_ptr<rpc::EgressMetadataHook> metadataHook,
+    ConnectionPool::Options options = ConnectionPool::Options());
 
 }  // namespace executor
 }  // namespace mongo
diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp
index 99079c97c8c..12261687d32 100644
--- a/src/mongo/s/sharding_initialization.cpp
+++ b/src/mongo/s/sharding_initialization.cpp
@@ -39,7 +39,9 @@
 #include "mongo/client/syncclusterconnection.h"
 #include "mongo/db/audit.h"
 #include "mongo/db/server_options.h"
+#include "mongo/db/server_parameters.h"
 #include "mongo/db/service_context.h"
+#include "mongo/executor/connection_pool.h"
 #include "mongo/executor/network_interface_factory.h"
 #include "mongo/executor/network_interface_thread_pool.h"
 #include "mongo/executor/task_executor.h"
@@ -47,7 +49,6 @@
 #include "mongo/executor/thread_pool_task_executor.h"
 #include "mongo/rpc/metadata/config_server_metadata.h"
 #include "mongo/rpc/metadata/metadata_hook.h"
-#include "mongo/rpc/metadata/config_server_metadata.h"
 #include "mongo/s/catalog/forwarding_catalog_manager.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/client/sharding_network_connection_hook.h"
@@ -61,6 +62,22 @@
 
 namespace mongo {
 
+using executor::ConnectionPool;
+
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolHostTimeout,
+                                      int,
+                                      ConnectionPool::kDefaultHostTimeout.count());
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMaxSize, int, -1);
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolMinSize,
+                                      int,
+                                      static_cast<int>(ConnectionPool::kDefaultMinConns));
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolRefreshRequirement,
+                                      int,
+                                      ConnectionPool::kDefaultRefreshRequirement.count());
+MONGO_EXPORT_STARTUP_SERVER_PARAMETER(ShardingTaskExecutorPoolRefreshTimeout,
+                                      int,
+                                      ConnectionPool::kDefaultRefreshTimeout.count());
+
 namespace {
 
 using executor::NetworkInterface;
@@ -124,13 +141,16 @@ std::unique_ptr<ThreadPoolTaskExecutor> makeTaskExecutor(std::unique_ptr<Network
         stdx::make_unique<NetworkInterfaceThreadPool>(netPtr), std::move(net));
 }
 
-std::unique_ptr<TaskExecutorPool> makeTaskExecutorPool(std::unique_ptr<NetworkInterface> fixedNet) {
+std::unique_ptr<TaskExecutorPool> makeTaskExecutorPool(std::unique_ptr<NetworkInterface> fixedNet,
+                                                       ConnectionPool::Options connPoolOptions) {
     std::vector<std::unique_ptr<executor::TaskExecutor>> executors;
+
     for (size_t i = 0; i < TaskExecutorPool::getSuggestedPoolSize(); ++i) {
         auto net = executor::makeNetworkInterface(
             "NetworkInterfaceASIO-TaskExecutorPool-" + std::to_string(i),
             stdx::make_unique<ShardingNetworkConnectionHook>(),
-            stdx::make_unique<ShardingEgressMetadataHook>());
+            stdx::make_unique<ShardingEgressMetadataHook>(),
+            connPoolOptions);
         auto netPtr = net.get();
         auto exec = stdx::make_unique<ThreadPoolTaskExecutor>(
             stdx::make_unique<NetworkInterfaceThreadPool>(netPtr), std::move(net));
@@ -157,14 +177,28 @@ Status initializeGlobalShardingState(OperationContext* txn,
         [](const HostAndPort& target, const executor::RemoteCommandResponse& isMasterReply) {
             return ShardingNetworkConnectionHook::validateHostImpl(target, isMasterReply, true);
         });
+
+    // We don't set the ConnectionPool's static const variables to be the default value in
+    // MONGO_EXPORT_STARTUP_SERVER_PARAMETER because it's not guaranteed to be initialized.
+    // The following code is a workaround.
+    ConnectionPool::Options connPoolOptions;
+    connPoolOptions.hostTimeout = Milliseconds(ShardingTaskExecutorPoolHostTimeout);
+    connPoolOptions.maxConnections = (ShardingTaskExecutorPoolMaxSize != -1)
+        ? ShardingTaskExecutorPoolMaxSize
+        : ConnectionPool::kDefaultMaxConns;
+    connPoolOptions.minConnections = ShardingTaskExecutorPoolMinSize;
+    connPoolOptions.refreshRequirement = Milliseconds(ShardingTaskExecutorPoolRefreshRequirement);
+    connPoolOptions.refreshTimeout = Milliseconds(ShardingTaskExecutorPoolRefreshTimeout);
+
     auto network =
         executor::makeNetworkInterface("NetworkInterfaceASIO-ShardRegistry",
                                        stdx::make_unique<ShardingNetworkConnectionHook>(),
-                                       stdx::make_unique<ShardingEgressMetadataHook>());
+                                       stdx::make_unique<ShardingEgressMetadataHook>(),
+                                       connPoolOptions);
     auto networkPtr = network.get();
     auto shardRegistry(
         stdx::make_unique<ShardRegistry>(stdx::make_unique<RemoteCommandTargeterFactoryImpl>(),
-                                         makeTaskExecutorPool(std::move(network)),
+                                         makeTaskExecutorPool(std::move(network), connPoolOptions),
                                          networkPtr,
                                          makeTaskExecutor(executor::makeNetworkInterface(
                                              "NetworkInterfaceASIO-ShardRegistry-TaskExecutor")),