SERVER-16436 cleanup unused connections in the NetworkInterface's ConenctionPool

1 files changed, 40 insertions, 1 deletions

diff --git a/src/mongo/db/repl/network_interface_impl.cpp b/src/mongo/db/repl/network_interface_impl.cpp
index 9f4b64d01fd..80b46cd22c5 100644
--- a/src/mongo/db/repl/network_interface_impl.cpp
+++ b/src/mongo/db/repl/network_interface_impl.cpp
@@ -58,6 +58,9 @@ namespace {
 
     const size_t kMinThreads = 1;
     const size_t kMaxThreads = 51;  // Set to 1 + max repl set size, for heartbeat + wiggle room.
+    const unsigned long long kNeverTooStale = std::numeric_limits<unsigned long long>::max();
+    // 5 Minutes (Note: Must be larger than kMaxConnectionAge below)
+    const long long kCleanUpInterval = 5 * 60 * 1000;
     const Seconds kMaxIdleThreadAge(30);
     const Seconds kMaxConnectionAge(30);
 
@@ -75,6 +78,7 @@ namespace {
 
         typedef stdx::list<ConnectionInfo> ConnectionList;
         typedef unordered_map<HostAndPort, ConnectionList> HostConnectionMap;
+        typedef std::map<HostAndPort, Date_t> HostLastUsedMap;
 
         /**
          * RAII class for connections from the pool.  To use the connection pool, instantiate one of
@@ -159,6 +163,11 @@ namespace {
         bool shouldKeepConnection(Date_t now, const ConnectionInfo& connInfo) const;
 
         /**
+         * Apply cleanup policy to any host(s) not active in the last kCleanupInterval milliseconds.
+         */
+        void cleanUpStaleHosts_inlock(Date_t now);
+
+        /**
          * Implementation of cleanUpOlderThan which assumes that _mutex is already held.
          */
         void cleanUpOlderThan_inlock(Date_t now);
@@ -183,6 +192,12 @@ namespace {
 
         // List of non-idle connections.
         ConnectionList _inUseConnections;
+
+        // Map of HostAndPorts to when they were last used.
+        HostLastUsedMap _lastUsedHosts;
+        
+        // Time representing when the connections were last cleaned.
+        Date_t _lastCleanUpTime;
     };
 
     /**
@@ -209,7 +224,7 @@ namespace {
         return _connInfo->conn;
     }
 
-    NetworkInterfaceImpl::ConnectionPool::ConnectionPool() {}
+    NetworkInterfaceImpl::ConnectionPool::ConnectionPool() : _lastCleanUpTime(0ULL) {}
 
     NetworkInterfaceImpl::ConnectionPool::~ConnectionPool() {
         cleanUpOlderThan(Date_t(~0ULL));
@@ -271,17 +286,40 @@ namespace {
         }
     }
 
+    void NetworkInterfaceImpl::ConnectionPool::cleanUpStaleHosts_inlock(Date_t now) {
+        if (now > _lastCleanUpTime + kCleanUpInterval) {
+            for (HostLastUsedMap::iterator itr = _lastUsedHosts.begin();
+                    itr != _lastUsedHosts.end();
+                    itr++) {
+                if (itr->second <= _lastCleanUpTime) {
+                    ConnectionList connList = _connections.find(itr->first)->second;
+                    cleanUpOlderThan_inlock(now, &connList);
+                    invariant(connList.empty());
+                    itr->second = Date_t(kNeverTooStale);
+                }
+            }
+            _lastCleanUpTime = now;
+        }
+    }
+
     NetworkInterfaceImpl::ConnectionPool::ConnectionList::iterator
     NetworkInterfaceImpl::ConnectionPool::acquireConnection(
             const HostAndPort& target,
             Date_t now,
             Milliseconds timeout) {
         boost::unique_lock<boost::mutex> lk(_mutex);
+
+        // Clean up connections on stale/unused hosts
+        cleanUpStaleHosts_inlock(now);
+
         for (HostConnectionMap::iterator hostConns;
              ((hostConns = _connections.find(target)) != _connections.end());) {
 
+            // Clean up the requested host to remove stale/unused connections
             cleanUpOlderThan_inlock(now, &hostConns->second);
             if (hostConns->second.empty()) {
+                // prevent host from causing unnecessary cleanups
+                _lastUsedHosts[hostConns->first] = Date_t(kNeverTooStale);
                 break;
             }
             _inUseConnections.splice(_inUseConnections.begin(),
@@ -341,6 +379,7 @@ namespace {
         ConnectionList& hostConns = _connections[iter->conn->getServerHostAndPort()];
         cleanUpOlderThan_inlock(now, &hostConns);
         hostConns.splice(hostConns.begin(), _inUseConnections, iter);
+        _lastUsedHosts[iter->conn->getServerHostAndPort()] = now;
     }
 
     void NetworkInterfaceImpl::ConnectionPool::destroyConnection(ConnectionList::iterator iter) {