SERVER-46121 Make separate Controllers with taskExecutorPoolSize > 1

This commit also introduces a new variant to verify that
taskExecutorPoolSize=4 works correctly.

(cherry picked from commit ac7c4b8079b8927bab3fbabf179a5be125c9c4bf)

4 files changed, 43 insertions, 6 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 62bfc2315bd..b2259440240 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -10668,6 +10668,32 @@ buildvariants:
     distros:
     - ubuntu1604-build
 
+- name: enterprise-ubuntu-task-executor-pool-size-1604-64-bit
+  display_name: "~ Enterprise Ubuntu 16.04 (with {taskExecutorPoolSize: 4})"
+  batchtime: 1440 # 1 day
+  run_on:
+    - ubuntu1604-test
+  modules:
+  - enterprise
+  expansions:
+    scons_cache_scope: shared
+    compile_flags: MONGO_DISTMOD=ubuntu1604 -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_v3_gcc.vars
+    test_flags: |- # Set the taskExecutorPoolSize for all tests
+        --mongosSetParameters="taskExecutorPoolSize: 4"
+  tasks:
+  - name: compile_all_run_unittests_TG
+    distros:
+    - ubuntu1604-build
+  - name: .aggregation !.no_async
+  - name: .sharding .auth
+  - name: .sharding .causally_consistent !.wo_snapshot
+  - name: .concurrency .common !.kill_terminate
+  - name: .integration !.audit
+  - name: .jscore .common
+  - name: .logical_session_cache .one_sec
+  - name: .sharding .jscore !.wo_snapshot !.multi_stmt
+  - name: .sharding .common !.csrs
+
 - name: shared-scons-cache-pruning
   display_name: "Shared SCons Cache Pruning"
   run_on:
diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index f3f2a61bbb8..3812a0a508e 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -195,6 +195,11 @@ protected:
     stdx::unordered_map<PoolId, PoolData> _poolData;
 };
 
+
+auto ConnectionPool::makeLimitController() noexcept -> std::shared_ptr<ControllerInterface> {
+    return std::make_shared<LimitController>();
+}
+
 /**
  * A pool for a specific HostAndPort
  *
@@ -439,15 +444,13 @@ ConnectionPool::ConnectionPool(std::shared_ptr<DependentTypeFactoryInterface> im
     : _name(std::move(name)),
       _factory(std::move(impl)),
       _options(std::move(options)),
-      _controller(std::move(_options.controller)),
+      _controller(_options.controllerFactory()),
       _manager(options.egressTagCloserManager) {
     if (_manager) {
         _manager->add(this);
     }
 
-    if (!_controller) {
-        _controller = std::make_shared<LimitController>();
-    }
+    invariant(_controller);
     _controller->init(this);
 }
 
diff --git a/src/mongo/executor/connection_pool.h b/src/mongo/executor/connection_pool.h
index d94196e8e06..1329e6aa4b7 100644
--- a/src/mongo/executor/connection_pool.h
+++ b/src/mongo/executor/connection_pool.h
@@ -90,6 +90,11 @@ public:
 
     static const Status kConnectionStateUnknown;
 
+    /**
+     * Make a vanilla LimitController as a decent default option
+     */
+    static std::shared_ptr<ControllerInterface> makeLimitController() noexcept;
+
     struct Options {
         Options() {}
 
@@ -144,7 +149,8 @@ public:
          */
         bool skipAuthentication = false;
 
-        std::shared_ptr<ControllerInterface> controller;
+        std::function<std::shared_ptr<ControllerInterface>(void)> controllerFactory =
+            &ConnectionPool::makeLimitController;
     };
 
     /**
diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp
index b241c9e7377..9b8860419aa 100644
--- a/src/mongo/s/sharding_initialization.cpp
+++ b/src/mongo/s/sharding_initialization.cpp
@@ -174,7 +174,9 @@ Status initializeGlobalShardingState(OperationContext* opCtx,
     }
 
     ConnectionPool::Options connPoolOptions;
-    connPoolOptions.controller = std::make_shared<ShardingTaskExecutorPoolController>();
+    connPoolOptions.controllerFactory = []() noexcept {
+        return std::make_shared<ShardingTaskExecutorPoolController>();
+    };
 
     auto network = executor::makeNetworkInterface(
         "ShardRegistry", std::make_unique<ShardingNetworkConnectionHook>(), hookBuilder());