SERVER-21597 Fix connPoolStats to work with many NetworkInterfaces

(cherry picked from commit 7c67e25f37853c60c106d2cf08eca1b81c4133ae)

33 files changed, 308 insertions, 125 deletions

diff --git a/jstests/sharding/conn_pool_stats.js b/jstests/sharding/conn_pool_stats.js
index ef15f73bdab..8cbf8788ae7 100644
--- a/jstests/sharding/conn_pool_stats.js
+++ b/jstests/sharding/conn_pool_stats.js
@@ -10,26 +10,10 @@ stats = cluster.s.getDB("admin").runCommand({connPoolStats : 1});
 printjson(stats);
 assert.commandWorked(stats);
 assert("replicaSets" in stats);
-
-assert("pools" in stats);
-var pools = stats["pools"];
-
-// Stats from dbclient pool
-assert("DBClient (Global)" in pools);
-var dbclient = pools["DBClient (Global)"];
-assert("hosts" in dbclient);
-assert("numClientConnection" in dbclient);
-assert("numAScopedConnection" in dbclient);
-assert("totalInUse" in dbclient);
-assert("totalAvailable" in dbclient);
-assert("totalCreated" in dbclient);
-assert.lte(dbclient["totalInUse"] + dbclient["totalAvailable"], dbclient["totalCreated"], tojson(dbclient));
-
-// Stats from ASIO pool
-assert("NetworkInterfaceASIO (Sharding)" in pools);
-var asio = pools["NetworkInterfaceASIO (Sharding)"];
-assert("hosts" in asio);
-assert("totalInUse" in asio);
-assert("totalAvailable" in asio);
-assert("totalCreated" in asio);
-assert.lte(asio["totalInUse"] + asio["totalAvailable"], asio["totalCreated"], tojson(asio));
+assert("hosts" in stats);
+assert("numClientConnections" in stats);
+assert("numAScopedConnections" in stats);
+assert("totalInUse" in stats);
+assert("totalAvailable" in stats);
+assert("totalCreated" in stats);
+assert.lte(stats["totalInUse"] + stats["totalAvailable"], stats["totalCreated"], tojson(stats));
diff --git a/src/mongo/client/SConscript b/src/mongo/client/SConscript
index e0a9f7fedc0..fabc96d0712 100644
--- a/src/mongo/client/SConscript
+++ b/src/mongo/client/SConscript
@@ -137,6 +137,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/auth/authcommon',
         '$BUILD_DIR/mongo/db/commands/test_commands_enabled',
         '$BUILD_DIR/mongo/db/dbmessage',
+        '$BUILD_DIR/mongo/executor/connection_pool_stats',
         '$BUILD_DIR/mongo/rpc/command_status',
         '$BUILD_DIR/mongo/rpc/rpc',
         '$BUILD_DIR/mongo/util/net/network',
diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp
index ef070e456d6..2a019db9787 100644
--- a/src/mongo/client/connpool.cpp
+++ b/src/mongo/client/connpool.cpp
@@ -35,9 +35,14 @@
 #include "mongo/platform/basic.h"
 
 #include "mongo/client/connpool.h"
+
+#include <string>
+
+#include "mongo/client/connection_string.h"
 #include "mongo/client/global_conn_pool.h"
 #include "mongo/client/replica_set_monitor.h"
 #include "mongo/client/syncclusterconnection.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/util/exit.h"
 #include "mongo/util/log.h"
 
@@ -341,41 +346,28 @@ void DBConnectionPool::onDestroy(DBClientBase* conn) {
     }
 }
 
-void DBConnectionPool::appendInfo(BSONObjBuilder& b) {
-    int totalInUse = 0;
-    int totalAvailable = 0;
-    long long totalCreated = 0;
-
-    BSONObjBuilder bb(b.subobjStart("hosts"));
+void DBConnectionPool::appendConnectionStats(executor::ConnectionPoolStats* stats) const {
     {
         stdx::lock_guard<stdx::mutex> lk(_mutex);
-        for (PoolMap::iterator i = _pools.begin(); i != _pools.end(); ++i) {
+        for (PoolMap::const_iterator i = _pools.begin(); i != _pools.end(); ++i) {
             if (i->second.numCreated() == 0)
                 continue;
 
-            auto inUse = i->second.numInUse();
-            auto available = i->second.numAvailable();
-            auto created = i->second.numCreated();
-
-            string s = str::stream() << i->first.ident << "::" << i->first.timeout;
-            BSONObjBuilder temp(bb.subobjStart(s));
-
-            temp.append("inUse", inUse);
-            temp.append("available", available);
-            temp.appendNumber("created", created);
-
-            temp.done();
-
-            totalInUse += inUse;
-            totalAvailable += available;
-            totalCreated += created;
+            // Mongos may use either a replica set uri or a list of addresses as
+            // the identifier here, so we always take the first server parsed out
+            // as our label for connPoolStats. Note that these stats will collide
+            // with any existing stats for the chosen host.
+            auto uri = ConnectionString::parse(i->first.ident);
+            invariant(uri.isOK());
+            HostAndPort host = uri.getValue().getServers().front();
+
+            executor::ConnectionStatsPerHost hostStats{
+                static_cast<size_t>(i->second.numInUse()),
+                static_cast<size_t>(i->second.numAvailable()),
+                static_cast<size_t>(i->second.numCreated())};
+            stats->updateStatsForHost(host, hostStats);
         }
     }
-    bb.done();
-
-    b.append("totalInUse", totalInUse);
-    b.append("totalAvailable", totalAvailable);
-    b.appendNumber("totalCreated", totalCreated);
 }
 
 bool DBConnectionPool::serverNameCompare::operator()(const string& a, const string& b) const {
diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h
index 570510ee2b3..586f1958319 100644
--- a/src/mongo/client/connpool.h
+++ b/src/mongo/client/connpool.h
@@ -42,6 +42,10 @@ namespace mongo {
 class BSONObjBuilder;
 class DBConnectionPool;
 
+namespace executor {
+struct ConnectionPoolStats;
+}  // namespace executor
+
 /**
  * not thread safe
  * thread safety is handled by DBConnectionPool
@@ -224,7 +228,7 @@ public:
     void release(const std::string& host, DBClientBase* c);
 
     void addHook(DBConnectionHook* hook);  // we take ownership
-    void appendInfo(BSONObjBuilder& b);
+    void appendConnectionStats(executor::ConnectionPoolStats* stats) const;
 
     /**
      * Clears all connections for all host.
@@ -274,7 +278,7 @@ private:
 
     typedef std::map<PoolKey, PoolForHost, poolKeyCompare> PoolMap;  // servername -> pool
 
-    stdx::mutex _mutex;
+    mutable stdx::mutex _mutex;
     std::string _name;
 
     // The maximum number of connections we'll save in the pool per-host
diff --git a/src/mongo/db/commands/conn_pool_stats.cpp b/src/mongo/db/commands/conn_pool_stats.cpp
index a8e5af9bd64..39d27c1200e 100644
--- a/src/mongo/db/commands/conn_pool_stats.cpp
+++ b/src/mongo/db/commands/conn_pool_stats.cpp
@@ -37,6 +37,7 @@
 #include "mongo/db/commands.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/repl/replication_coordinator_global.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/executor/network_interface_factory.h"
 #include "mongo/s/client/shard_registry.h"
 #include "mongo/s/grid.h"
@@ -69,32 +70,29 @@ public:
              int,
              std::string&,
              mongo::BSONObjBuilder& result) override {
-        BSONObjBuilder poolStats(result.subobjStart("pools"));
+        executor::ConnectionPoolStats stats{};
 
-        // Global connection pool
-        BSONObjBuilder globalStats(poolStats.subobjStart("DBClient (Global)"));
-        globalConnPool.appendInfo(globalStats);
-        globalStats.append("numClientConnection", DBClientConnection::getNumConnections());
-        globalStats.append("numAScopedConnection", AScopedConnection::getNumConnections());
-        globalStats.doneFast();
+        // Global connection pool connections.
+        globalConnPool.appendConnectionStats(&stats);
+        result.appendNumber("numClientConnections", DBClientConnection::getNumConnections());
+        result.appendNumber("numAScopedConnections", AScopedConnection::getNumConnections());
 
-        // Replication ASIO, if we have one
+        // Replication connections, if we have them.
         auto replCoord = repl::ReplicationCoordinator::get(txn);
         if (replCoord && replCoord->isReplEnabled()) {
-            BSONObjBuilder replStats(poolStats.subobjStart("NetworkInterfaceASIO (Replication)"));
-            replCoord->appendConnectionStats(&replStats);
+            replCoord->appendConnectionStats(&stats);
         }
 
-        // Sharding ASIO, if we have one
+        // Sharding connections, if we have any.
         auto registry = grid.shardRegistry();
         if (registry) {
-            BSONObjBuilder shardStats(poolStats.subobjStart("NetworkInterfaceASIO (Sharding)"));
-            registry->getExecutor()->appendConnectionStats(&shardStats);
+            registry->appendConnectionStats(&stats);
         }
 
-        poolStats.doneFast();
+        // Output to a BSON object.
+        stats.appendToBSON(result);
 
-        // Always report all replica sets being tracked
+        // Always report all replica sets being tracked.
         BSONObjBuilder setStats(result.subobjStart("replicaSets"));
         globalRSMonitorManager.report(&setStats);
         setStats.doneFast();
diff --git a/src/mongo/db/repl/replication_coordinator.h b/src/mongo/db/repl/replication_coordinator.h
index 05c5d3c212e..1d31b059bb7 100644
--- a/src/mongo/db/repl/replication_coordinator.h
+++ b/src/mongo/db/repl/replication_coordinator.h
@@ -50,6 +50,10 @@ class SnapshotName;
 class Timestamp;
 struct WriteConcernOptions;
 
+namespace executor {
+struct ConnectionPoolStats;
+}  // namespace executor
+
 namespace rpc {
 
 class ReplSetMetadata;
@@ -726,7 +730,7 @@ public:
     /**
      * Appends connection information to the provided BSONObjBuilder.
      */
-    virtual void appendConnectionStats(BSONObjBuilder* b) = 0;
+    virtual void appendConnectionStats(executor::ConnectionPoolStats* stats) const = 0;
 
     /**
      * Gets the number of uncommitted snapshots currently held.
diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp
index 1840b7aa67a..95b674e5e4c 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.cpp
+++ b/src/mongo/db/repl/replication_coordinator_impl.cpp
@@ -65,6 +65,7 @@
 #include "mongo/db/repl/vote_requester.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/write_concern_options.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/rpc/metadata/repl_set_metadata.h"
 #include "mongo/rpc/request_interface.h"
 #include "mongo/stdx/functional.h"
@@ -283,8 +284,8 @@ OpTime ReplicationCoordinatorImpl::getCurrentCommittedSnapshotOpTime() {
     return OpTime();
 }
 
-void ReplicationCoordinatorImpl::appendConnectionStats(BSONObjBuilder* b) {
-    _replExecutor.appendConnectionStats(b);
+void ReplicationCoordinatorImpl::appendConnectionStats(executor::ConnectionPoolStats* stats) const {
+    _replExecutor.appendConnectionStats(stats);
 }
 
 void ReplicationCoordinatorImpl::_updateLastVote(const LastVote& lastVote) {
diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h
index 6d2c110241d..8252ac53412 100644
--- a/src/mongo/db/repl/replication_coordinator_impl.h
+++ b/src/mongo/db/repl/replication_coordinator_impl.h
@@ -60,6 +60,10 @@ class Timer;
 template <typename T>
 class StatusWith;
 
+namespace executor {
+struct ConnectionPoolStats;
+}  // namespace executor
+
 namespace rpc {
 class ReplSetMetadata;
 }  // namespace rpc
@@ -307,7 +311,7 @@ public:
     virtual void waitUntilSnapshotCommitted(OperationContext* txn,
                                             const SnapshotName& untilSnapshot) override;
 
-    virtual void appendConnectionStats(BSONObjBuilder* b) override;
+    virtual void appendConnectionStats(executor::ConnectionPoolStats* stats) const override;
 
     virtual size_t getNumUncommittedSnapshots() override;
 
diff --git a/src/mongo/db/repl/replication_coordinator_mock.cpp b/src/mongo/db/repl/replication_coordinator_mock.cpp
index bf203b720d7..70457f0548c 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.cpp
+++ b/src/mongo/db/repl/replication_coordinator_mock.cpp
@@ -228,7 +228,8 @@ void ReplicationCoordinatorMock::fillIsMasterForReplSet(IsMasterResponse* result
 
 void ReplicationCoordinatorMock::appendSlaveInfoData(BSONObjBuilder* result) {}
 
-void ReplicationCoordinatorMock::appendConnectionStats(BSONObjBuilder* b) {}
+void ReplicationCoordinatorMock::appendConnectionStats(executor::ConnectionPoolStats* stats) const {
+}
 
 Status ReplicationCoordinatorMock::setMaintenanceMode(bool activate) {
     return Status::OK();
diff --git a/src/mongo/db/repl/replication_coordinator_mock.h b/src/mongo/db/repl/replication_coordinator_mock.h
index 6fb18b19007..60f1b1b23c4 100644
--- a/src/mongo/db/repl/replication_coordinator_mock.h
+++ b/src/mongo/db/repl/replication_coordinator_mock.h
@@ -34,6 +34,11 @@
 #include "mongo/platform/atomic_word.h"
 
 namespace mongo {
+
+namespace executor {
+struct ConnectionPoolStats;
+}  // namespace executor
+
 namespace repl {
 
 /**
@@ -135,7 +140,7 @@ public:
 
     virtual void appendSlaveInfoData(BSONObjBuilder* result);
 
-    void appendConnectionStats(BSONObjBuilder* b) override;
+    void appendConnectionStats(executor::ConnectionPoolStats* stats) const override;
 
     virtual ReplicaSetConfig getConfig() const;
 
diff --git a/src/mongo/db/repl/replication_executor.cpp b/src/mongo/db/repl/replication_executor.cpp
index d905f3b1fdd..4081e3fb876 100644
--- a/src/mongo/db/repl/replication_executor.cpp
+++ b/src/mongo/db/repl/replication_executor.cpp
@@ -468,8 +468,8 @@ ReplicationExecutor::scheduleWorkWithGlobalExclusiveLock(const CallbackFn& work)
     return handle;
 }
 
-void ReplicationExecutor::appendConnectionStats(BSONObjBuilder* b) {
-    _networkInterface->appendConnectionStats(b);
+void ReplicationExecutor::appendConnectionStats(executor::ConnectionPoolStats* stats) const {
+    _networkInterface->appendConnectionStats(stats);
 }
 
 std::pair<ReplicationExecutor::WorkItem, ReplicationExecutor::CallbackHandle>
diff --git a/src/mongo/db/repl/replication_executor.h b/src/mongo/db/repl/replication_executor.h
index 0684ba63071..c46ef4a659a 100644
--- a/src/mongo/db/repl/replication_executor.h
+++ b/src/mongo/db/repl/replication_executor.h
@@ -55,6 +55,7 @@ class NamespaceString;
 class OperationContext;
 
 namespace executor {
+struct ConnectionPoolStats;
 class NetworkInterface;
 }  // namespace executor
 
@@ -125,7 +126,7 @@ public:
     void cancel(const CallbackHandle& cbHandle) override;
     void wait(const CallbackHandle& cbHandle) override;
 
-    void appendConnectionStats(BSONObjBuilder* b) override;
+    void appendConnectionStats(executor::ConnectionPoolStats* stats) const override;
 
     /**
      * Executes the run loop. May be called up to one time.
diff --git a/src/mongo/executor/SConscript b/src/mongo/executor/SConscript
index 1ec87aa4fd5..c9786c22558 100644
--- a/src/mongo/executor/SConscript
+++ b/src/mongo/executor/SConscript
@@ -4,6 +4,16 @@ Import("env")
 
 env.InjectThirdPartyIncludePaths('asio')
 
+env.Library(
+    target='connection_pool_stats',
+    source=[
+        'connection_pool_stats.cpp',
+    ],
+    LIBDEPS=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/util/net/hostandport',
+    ])
+
 env.Library(target='remote_command',
             source=[
                 'remote_command_request.cpp',
@@ -79,6 +89,7 @@ env.Library(
     LIBDEPS=[
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/util/net/hostandport',
+        'connection_pool_stats',
         'remote_command',
     ],
 )
diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index df6791a9590..1d41cded111 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -31,6 +31,7 @@
 
 #include "mongo/executor/connection_pool.h"
 
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/bson/bsonobjbuilder.h"
 #include "mongo/executor/remote_command_request.h"
 #include "mongo/stdx/memory.h"
@@ -211,42 +212,18 @@ void ConnectionPool::get(const HostAndPort& hostAndPort,
     pool->getConnection(hostAndPort, timeout, std::move(lk), std::move(cb));
 }
 
-void ConnectionPool::appendConnectionStats(BSONObjBuilder* b) {
-    size_t inUse = 0u;
-    size_t available = 0u;
-    size_t created = 0u;
-
-    BSONObjBuilder hostBuilder(b->subobjStart("hosts"));
-
+void ConnectionPool::appendConnectionStats(ConnectionPoolStats* stats) const {
     stdx::unique_lock<stdx::mutex> lk(_mutex);
 
     for (const auto& kv : _pools) {
-        std::string label = kv.first.toString();
-        BSONObjBuilder hostInfo(hostBuilder.subobjStart(label));
+        HostAndPort host = kv.first;
 
         auto& pool = kv.second;
-        auto inUseConnections = pool->inUseConnections(lk);
-        auto availableConnections = pool->availableConnections(lk);
-        auto createdConnections = pool->createdConnections(lk);
-        hostInfo.appendNumber("inUse", inUseConnections);
-        hostInfo.appendNumber("available", availableConnections);
-        hostInfo.appendNumber("created", createdConnections);
-
-        hostInfo.done();
-
-        // update available and created
-        inUse += inUseConnections;
-        available += availableConnections;
-        created += createdConnections;
+        ConnectionStatsPerHost hostStats{pool->inUseConnections(lk),
+                                         pool->availableConnections(lk),
+                                         pool->createdConnections(lk)};
+        stats->updateStatsForHost(host, hostStats);
     }
-
-    hostBuilder.done();
-
-    b->appendNumber("totalInUse", inUse);
-    b->appendNumber("totalAvailable", available);
-    b->appendNumber("totalCreated", created);
-
-    return;
 }
 
 void ConnectionPool::returnConnection(ConnectionInterface* conn) {
diff --git a/src/mongo/executor/connection_pool.h b/src/mongo/executor/connection_pool.h
index b1b0549365c..a5bcc1c3925 100644
--- a/src/mongo/executor/connection_pool.h
+++ b/src/mongo/executor/connection_pool.h
@@ -43,6 +43,8 @@ class BSONObjBuilder;
 
 namespace executor {
 
+struct ConnectionPoolStats;
+
 /**
  * The actual user visible connection pool.
  *
@@ -115,7 +117,7 @@ public:
 
     void get(const HostAndPort& hostAndPort, Milliseconds timeout, GetConnectionCallback cb);
 
-    void appendConnectionStats(BSONObjBuilder* b);
+    void appendConnectionStats(ConnectionPoolStats* stats) const;
 
 private:
     void returnConnection(ConnectionInterface* connection);
@@ -127,7 +129,7 @@ private:
     const std::unique_ptr<DependentTypeFactoryInterface> _factory;
 
     // The global mutex for specific pool access and the generation counter
-    stdx::mutex _mutex;
+    mutable stdx::mutex _mutex;
     std::unordered_map<HostAndPort, std::unique_ptr<SpecificPool>> _pools;
 };
 
diff --git a/src/mongo/executor/connection_pool_stats.cpp b/src/mongo/executor/connection_pool_stats.cpp
new file mode 100644
index 00000000000..02ff34da6c1
--- /dev/null
+++ b/src/mongo/executor/connection_pool_stats.cpp
@@ -0,0 +1,81 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/executor/connection_pool_stats.h"
+
+#include "mongo/bson/bsonobjbuilder.h"
+#include "mongo/util/map_util.h"
+
+namespace mongo {
+namespace executor {
+
+ConnectionStatsPerHost::ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated)
+    : inUse(nInUse), available(nAvailable), created(nCreated) {}
+
+ConnectionStatsPerHost::ConnectionStatsPerHost() = default;
+
+ConnectionStatsPerHost& ConnectionStatsPerHost::operator+=(const ConnectionStatsPerHost& other) {
+    inUse += other.inUse;
+    available += other.available;
+    created += other.created;
+
+    return *this;
+}
+
+void ConnectionPoolStats::updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats) {
+    // Update stats for this host.
+    auto hostStats = mapFindWithDefault(statsByHost, host);
+    hostStats += newStats;
+    statsByHost.insert(std::make_pair(host, hostStats));
+
+    // Update total connection stats.
+    totalInUse += newStats.inUse;
+    totalAvailable += newStats.available;
+    totalCreated += newStats.created;
+}
+
+void ConnectionPoolStats::appendToBSON(mongo::BSONObjBuilder& result) {
+    result.appendNumber("totalInUse", totalInUse);
+    result.appendNumber("totalAvailable", totalAvailable);
+    result.appendNumber("totalCreated", totalCreated);
+
+    BSONObjBuilder hostBuilder(result.subobjStart("hosts"));
+    for (auto&& host : statsByHost) {
+        BSONObjBuilder hostInfo(hostBuilder.subobjStart(host.first.toString()));
+
+        auto hostStats = host.second;
+        hostInfo.appendNumber("inUse", hostStats.inUse);
+        hostInfo.appendNumber("available", hostStats.available);
+        hostInfo.appendNumber("created", hostStats.created);
+    }
+}
+
+}  // namespace executor
+}  // namespace mongo
diff --git a/src/mongo/executor/connection_pool_stats.h b/src/mongo/executor/connection_pool_stats.h
new file mode 100644
index 00000000000..5e9bd0c7633
--- /dev/null
+++ b/src/mongo/executor/connection_pool_stats.h
@@ -0,0 +1,72 @@
+/**
+ *    Copyright (C) 2015 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <unordered_map>
+
+#include "mongo/util/net/hostandport.h"
+
+namespace mongo {
+namespace executor {
+
+/**
+ * Holds connection information for a specific remote host. These objects are maintained by
+ * a parent ConnectionPoolStats object and should not need to be created directly.
+ */
+struct ConnectionStatsPerHost {
+    ConnectionStatsPerHost(size_t nInUse, size_t nAvailable, size_t nCreated);
+
+    ConnectionStatsPerHost();
+
+    ConnectionStatsPerHost& operator+=(const ConnectionStatsPerHost& other);
+
+    size_t inUse = 0u;
+    size_t available = 0u;
+    size_t created = 0u;
+};
+
+/**
+ * Aggregates connection information for the connPoolStats command. Connection pools should
+ * use the updateStatsForHost() method to append their host-specific information to this object.
+ * Total connection counts will then be updated accordingly.
+ */
+struct ConnectionPoolStats {
+    void updateStatsForHost(HostAndPort host, ConnectionStatsPerHost newStats);
+
+    void appendToBSON(mongo::BSONObjBuilder& result);
+
+    size_t totalInUse = 0u;
+    size_t totalAvailable = 0u;
+    size_t totalCreated = 0u;
+
+    std::unordered_map<HostAndPort, ConnectionStatsPerHost> statsByHost;
+};
+
+}  // namespace executor
+}  // namespace mongo
diff --git a/src/mongo/executor/network_interface.h b/src/mongo/executor/network_interface.h
index 40bb72ffbd9..e35f8ed9c52 100644
--- a/src/mongo/executor/network_interface.h
+++ b/src/mongo/executor/network_interface.h
@@ -62,9 +62,9 @@ public:
     virtual std::string getDiagnosticString() = 0;
 
     /**
-     * Appends connection information to the provided BSONObjBuilder.
+     * Appends information about the connections on this NetworkInterface.
      */
-    virtual void appendConnectionStats(BSONObjBuilder* b) = 0;
+    virtual void appendConnectionStats(ConnectionPoolStats* stats) const = 0;
 
     /**
      * Starts up the network interface.
diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp
index cf2f7f36d94..84028a95194 100644
--- a/src/mongo/executor/network_interface_asio.cpp
+++ b/src/mongo/executor/network_interface_asio.cpp
@@ -40,6 +40,7 @@
 #include "mongo/executor/async_timer_interface.h"
 #include "mongo/executor/async_timer_mock.h"
 #include "mongo/executor/connection_pool_asio.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/rpc/metadata/metadata_hook.h"
 #include "mongo/stdx/chrono.h"
 #include "mongo/stdx/memory.h"
@@ -96,8 +97,8 @@ std::string NetworkInterfaceASIO::getDiagnosticString() {
     return output;
 }
 
-void NetworkInterfaceASIO::appendConnectionStats(BSONObjBuilder* b) {
-    _connectionPool.appendConnectionStats(b);
+void NetworkInterfaceASIO::appendConnectionStats(ConnectionPoolStats* stats) const {
+    _connectionPool.appendConnectionStats(stats);
 }
 
 std::string NetworkInterfaceASIO::getHostName() {
diff --git a/src/mongo/executor/network_interface_asio.h b/src/mongo/executor/network_interface_asio.h
index 7c01ad281e3..f4eb6eac110 100644
--- a/src/mongo/executor/network_interface_asio.h
+++ b/src/mongo/executor/network_interface_asio.h
@@ -102,7 +102,7 @@ public:
     NetworkInterfaceASIO(Options = Options());
 
     std::string getDiagnosticString() override;
-    void appendConnectionStats(BSONObjBuilder* b) override;
+    void appendConnectionStats(ConnectionPoolStats* stats) const override;
     std::string getHostName() override;
     void startup() override;
     void shutdown() override;
diff --git a/src/mongo/executor/network_interface_impl.cpp b/src/mongo/executor/network_interface_impl.cpp
index b80b51d63e2..b089c3dd33e 100644
--- a/src/mongo/executor/network_interface_impl.cpp
+++ b/src/mongo/executor/network_interface_impl.cpp
@@ -87,7 +87,7 @@ std::string NetworkInterfaceImpl::getDiagnosticString() {
     return output;
 }
 
-void NetworkInterfaceImpl::appendConnectionStats(BSONObjBuilder* b) {}
+void NetworkInterfaceImpl::appendConnectionStats(ConnectionPoolStats* stats) const {}
 
 void NetworkInterfaceImpl::startup() {
     stdx::unique_lock<stdx::mutex> lk(_mutex);
diff --git a/src/mongo/executor/network_interface_impl.h b/src/mongo/executor/network_interface_impl.h
index 7657417fef3..5915d2d99d6 100644
--- a/src/mongo/executor/network_interface_impl.h
+++ b/src/mongo/executor/network_interface_impl.h
@@ -43,6 +43,7 @@
 namespace mongo {
 namespace executor {
 
+struct ConnectionPoolStats;
 class NetworkConnectionHook;
 
 /**
@@ -77,7 +78,7 @@ public:
     NetworkInterfaceImpl(std::unique_ptr<NetworkConnectionHook> hook);
     ~NetworkInterfaceImpl();
     std::string getDiagnosticString() override;
-    void appendConnectionStats(BSONObjBuilder* b) override;
+    void appendConnectionStats(ConnectionPoolStats* stats) const override;
     void startup() override;
     void shutdown() override;
     void waitForWork() override;
diff --git a/src/mongo/executor/network_interface_mock.cpp b/src/mongo/executor/network_interface_mock.cpp
index 85e281db47e..34b6a2d4fc6 100644
--- a/src/mongo/executor/network_interface_mock.cpp
+++ b/src/mongo/executor/network_interface_mock.cpp
@@ -36,6 +36,7 @@
 #include <algorithm>
 #include <iterator>
 
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/stdx/functional.h"
 #include "mongo/util/log.h"
 #include "mongo/util/time_support.h"
@@ -63,7 +64,7 @@ std::string NetworkInterfaceMock::getDiagnosticString() {
     return "NetworkInterfaceMock diagnostics here";
 }
 
-void NetworkInterfaceMock::appendConnectionStats(BSONObjBuilder* b) {}
+void NetworkInterfaceMock::appendConnectionStats(ConnectionPoolStats* stats) const {}
 
 Date_t NetworkInterfaceMock::now() {
     stdx::lock_guard<stdx::mutex> lk(_mutex);
diff --git a/src/mongo/executor/network_interface_mock.h b/src/mongo/executor/network_interface_mock.h
index afd769f61b5..439b33e5caf 100644
--- a/src/mongo/executor/network_interface_mock.h
+++ b/src/mongo/executor/network_interface_mock.h
@@ -75,7 +75,7 @@ public:
 
     NetworkInterfaceMock();
     virtual ~NetworkInterfaceMock();
-    virtual void appendConnectionStats(BSONObjBuilder* b);
+    virtual void appendConnectionStats(ConnectionPoolStats* stats) const;
     virtual std::string getDiagnosticString();
 
     ////////////////////////////////////////////////////////////////////////////////
diff --git a/src/mongo/executor/task_executor.h b/src/mongo/executor/task_executor.h
index a52f5003a0b..ec68a837c8e 100644
--- a/src/mongo/executor/task_executor.h
+++ b/src/mongo/executor/task_executor.h
@@ -48,6 +48,8 @@ class OperationContext;
 
 namespace executor {
 
+struct ConnectionPoolStats;
+
 /**
  * Generic event loop with notions of events and callbacks.
  *
@@ -235,7 +237,7 @@ public:
      * Appends information about the underlying network interface's connections to the given
      * builder.
      */
-    virtual void appendConnectionStats(BSONObjBuilder* b) = 0;
+    virtual void appendConnectionStats(ConnectionPoolStats* stats) const = 0;
 
 protected:
     // Retrieves the Callback from a given CallbackHandle
diff --git a/src/mongo/executor/task_executor_pool.cpp b/src/mongo/executor/task_executor_pool.cpp
index 2af37277a7c..89f0da22ed6 100644
--- a/src/mongo/executor/task_executor_pool.cpp
+++ b/src/mongo/executor/task_executor_pool.cpp
@@ -33,6 +33,7 @@
 #include <algorithm>
 
 #include "mongo/db/server_parameters.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/executor/task_executor.h"
 #include "mongo/util/processinfo.h"
 
@@ -95,5 +96,14 @@ TaskExecutor* TaskExecutorPool::getFixedExecutor() {
     return _fixedExecutor.get();
 }
 
+void TaskExecutorPool::appendConnectionStats(ConnectionPoolStats* stats) const {
+    // Get stats from our fixed executor.
+    _fixedExecutor->appendConnectionStats(stats);
+    // Get stats from our pooled executors.
+    for (auto&& executor : _executors) {
+        executor->appendConnectionStats(stats);
+    }
+}
+
 }  // namespace executor
 }  // namespace mongo
diff --git a/src/mongo/executor/task_executor_pool.h b/src/mongo/executor/task_executor_pool.h
index 00a828b3c27..15319589c4e 100644
--- a/src/mongo/executor/task_executor_pool.h
+++ b/src/mongo/executor/task_executor_pool.h
@@ -36,6 +36,7 @@
 namespace mongo {
 namespace executor {
 
+struct ConnectionPoolStats;
 class TaskExecutor;
 
 /**
@@ -101,6 +102,15 @@ public:
      */
     TaskExecutor* getFixedExecutor();
 
+    /**
+     * Appends connection information from all of the executors in the pool.
+     *
+     * NOTE: this method returns approximate stats. To avoid blocking operations on the
+     * pool, we don't lock for appendConnectionStats, so data gathered across connection pools
+     * will be from slightly different points in time.
+     */
+    void appendConnectionStats(ConnectionPoolStats* stats) const;
+
 private:
     AtomicUInt32 _counter;
 
diff --git a/src/mongo/executor/thread_pool_task_executor.cpp b/src/mongo/executor/thread_pool_task_executor.cpp
index c45bce684cf..ea43b5e09eb 100644
--- a/src/mongo/executor/thread_pool_task_executor.cpp
+++ b/src/mongo/executor/thread_pool_task_executor.cpp
@@ -38,6 +38,7 @@
 #include "mongo/base/checked_cast.h"
 #include "mongo/base/disallow_copying.h"
 #include "mongo/base/status_with.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/executor/network_interface.h"
 #include "mongo/platform/atomic_word.h"
 #include "mongo/util/concurrency/thread_pool_interface.h"
@@ -369,8 +370,8 @@ void ThreadPoolTaskExecutor::wait(const CallbackHandle& cbHandle) {
     }
 }
 
-void ThreadPoolTaskExecutor::appendConnectionStats(BSONObjBuilder* b) {
-    _net->appendConnectionStats(b);
+void ThreadPoolTaskExecutor::appendConnectionStats(ConnectionPoolStats* stats) const {
+    _net->appendConnectionStats(stats);
 }
 
 void ThreadPoolTaskExecutor::cancelAllCommands() {
diff --git a/src/mongo/executor/thread_pool_task_executor.h b/src/mongo/executor/thread_pool_task_executor.h
index aea52771f9b..f4afb7b58c9 100644
--- a/src/mongo/executor/thread_pool_task_executor.h
+++ b/src/mongo/executor/thread_pool_task_executor.h
@@ -43,6 +43,7 @@ class ThreadPoolInterface;
 
 namespace executor {
 
+struct ConnectionPoolStats;
 class NetworkInterface;
 
 /**
@@ -80,7 +81,7 @@ public:
     void cancel(const CallbackHandle& cbHandle) override;
     void wait(const CallbackHandle& cbHandle) override;
 
-    void appendConnectionStats(BSONObjBuilder* b) override;
+    void appendConnectionStats(ConnectionPoolStats* stats) const override;
 
     /**
      * Cancels all commands on the network interface.
diff --git a/src/mongo/s/client/SConscript b/src/mongo/s/client/SConscript
index 42e8a03d3ef..0cb92700213 100644
--- a/src/mongo/s/client/SConscript
+++ b/src/mongo/s/client/SConscript
@@ -15,6 +15,7 @@ env.Library(
         '$BUILD_DIR/mongo/client/fetcher',
         '$BUILD_DIR/mongo/client/remote_command_runner_impl',
         '$BUILD_DIR/mongo/client/remote_command_targeter',
+        '$BUILD_DIR/mongo/executor/connection_pool_stats',
         '$BUILD_DIR/mongo/rpc/metadata',
     ],
     LIBDEPS_TAGS=[
diff --git a/src/mongo/s/client/shard_connection.cpp b/src/mongo/s/client/shard_connection.cpp
index 9be2f45a028..4d810238744 100644
--- a/src/mongo/s/client/shard_connection.cpp
+++ b/src/mongo/s/client/shard_connection.cpp
@@ -36,6 +36,7 @@
 
 #include "mongo/db/commands.h"
 #include "mongo/db/lasterror.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/s/chunk_manager.h"
 #include "mongo/s/client/shard.h"
 #include "mongo/s/client/shard_registry.h"
@@ -115,10 +116,12 @@ public:
                      int options,
                      std::string& errmsg,
                      mongo::BSONObjBuilder& result) {
-        // Base pool info
-        shardConnectionPool.appendInfo(result);
+        // Connection information
+        executor::ConnectionPoolStats stats{};
+        shardConnectionPool.appendConnectionStats(&stats);
+        stats.appendToBSON(result);
 
-        // Thread connection info
+        // Thread connection information
         activeClientConnections.appendInfo(result);
 
         return true;
diff --git a/src/mongo/s/client/shard_registry.cpp b/src/mongo/s/client/shard_registry.cpp
index 9b6814901b1..556f4e8de46 100644
--- a/src/mongo/s/client/shard_registry.cpp
+++ b/src/mongo/s/client/shard_registry.cpp
@@ -42,6 +42,7 @@
 #include "mongo/client/replica_set_monitor.h"
 #include "mongo/db/client.h"
 #include "mongo/db/query/lite_parsed_query.h"
+#include "mongo/executor/connection_pool_stats.h"
 #include "mongo/executor/task_executor.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/rpc/metadata/config_server_metadata.h"
@@ -322,6 +323,13 @@ void ShardRegistry::toBSON(BSONObjBuilder* result) {
     }
 }
 
+void ShardRegistry::appendConnectionStats(executor::ConnectionPoolStats* stats) const {
+    // Get stats from the pool of task executors, including fixed executor within.
+    _executorPool->appendConnectionStats(stats);
+    // Get stats from the separate executor for addShard.
+    _executorForAddShard->appendConnectionStats(stats);
+}
+
 void ShardRegistry::_addConfigShard_inlock() {
     ShardType configServerShard;
     configServerShard.setName("config");
diff --git a/src/mongo/s/client/shard_registry.h b/src/mongo/s/client/shard_registry.h
index 897b05809ec..dec0f770803 100644
--- a/src/mongo/s/client/shard_registry.h
+++ b/src/mongo/s/client/shard_registry.h
@@ -60,6 +60,7 @@ class StatusWith;
 
 namespace executor {
 
+struct ConnectionPoolStats;
 class NetworkInterface;
 class TaskExecutor;
 
@@ -208,6 +209,11 @@ public:
     void toBSON(BSONObjBuilder* result);
 
     /**
+     * Append information about the sharding subsystem's connection pools.
+     */
+    void appendConnectionStats(executor::ConnectionPoolStats* stats) const;
+
+    /**
      * If the newly specified optime is newer than the one the ShardRegistry already knows, the
      * one in the registry will be advanced. Otherwise, it remains the same.
      */