SERVER-13374 change connection pool defaults

(cherry picked from commit f71610672b0506b233a449bfafb303497c97ae50)

9 files changed, 211 insertions, 30 deletions

diff --git a/src/mongo/SConscript b/src/mongo/SConscript
index e98c35925a4..daaa322c05d 100644
--- a/src/mongo/SConscript
+++ b/src/mongo/SConscript
@@ -484,6 +484,7 @@ coreServerFiles = [ "db/client_basic.cpp",
                     "db/stats/counters.cpp",
                     "db/stats/service_stats.cpp",
                     "db/log_process_details.cpp",
+                    "db/conn_pool_options.cpp"
                     ]
 
 env.Library('ntservice', ['util/ntservice.cpp'],
diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp
index 91ae84c21f9..6df97ea2d31 100644
--- a/src/mongo/client/connpool.cpp
+++ b/src/mongo/client/connpool.cpp
@@ -41,18 +41,23 @@ namespace mongo {
         }
     }
 
-    void PoolForHost::done( DBConnectionPool * pool, DBClientBase * c ) {
-        if (c->isFailed()) {
-            reportBadConnectionAt(c->getSockCreationMicroSec());
-            pool->onDestroy(c);
-            delete c;
-        }
-        else if (_pool.size() >= _maxPerHost ||
-                c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) {
+    void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) {
+
+        bool isFailed = c->isFailed();
+
+        // Remember that this host had a broken connection for later
+        if (isFailed) reportBadConnectionAt(c->getSockCreationMicroSec());
+
+        if (isFailed ||
+            // Another (later) connection was reported as broken to this host
+            (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) ||
+            // We have a pool size that we need to enforce
+            (_maxPoolSize >= 0 && static_cast<int>(_pool.size()) >= _maxPoolSize)) {
             pool->onDestroy(c);
             delete c;
         }
         else {
+            // The connection is probably fine, save for later
             _pool.push(c);
         }
     }
@@ -166,16 +171,17 @@ namespace mongo {
         }
     }
 
-    unsigned PoolForHost::_maxPerHost = 50;
-
     // ------ DBConnectionPool ------
 
     DBConnectionPool pool;
 
-    DBConnectionPool::DBConnectionPool() 
+    const int PoolForHost::kPoolSizeUnlimited(-1);
+
+    DBConnectionPool::DBConnectionPool()
         : _mutex("DBConnectionPool") , 
           _name( "dbconnectionpool" ) , 
-          _hooks( new list<DBConnectionHook*>() ) { 
+          _maxPoolSize(PoolForHost::kPoolSizeUnlimited) ,
+          _hooks( new list<DBConnectionHook*>() ) {
     }
 
     DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) {
@@ -183,6 +189,7 @@ namespace mongo {
                 !inShutdown());
         scoped_lock L(_mutex);
         PoolForHost& p = _pools[PoolKey(ident,socketTimeout)];
+        p.setMaxPoolSize(_maxPoolSize);
         p.initializeHostName(ident);
         return p.get( this , socketTimeout );
     }
@@ -191,6 +198,7 @@ namespace mongo {
         {
             scoped_lock L(_mutex);
             PoolForHost& p = _pools[PoolKey(host,socketTimeout)];
+            p.setMaxPoolSize(_maxPoolSize);
             p.initializeHostName(host);
             p.createdOne( conn );
         }
diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h
index 694ce53d6dc..9bf4583065f 100644
--- a/src/mongo/client/connpool.h
+++ b/src/mongo/client/connpool.h
@@ -35,18 +35,38 @@ namespace mongo {
      */
     class MONGO_CLIENT_API PoolForHost {
     public:
-        PoolForHost()
-            : _created(0), _minValidCreationTimeMicroSec(0) {}
 
-        PoolForHost( const PoolForHost& other ) {
+        // Sentinel value indicating pool has no cleanup limit
+        static const int kPoolSizeUnlimited;
+
+        PoolForHost() :
+            _created(0),
+            _minValidCreationTimeMicroSec(0),
+            _type(ConnectionString::INVALID),
+            _maxPoolSize(kPoolSizeUnlimited) {
+        }
+
+        PoolForHost(const PoolForHost& other) :
+            _created(other._created),
+            _minValidCreationTimeMicroSec(other._minValidCreationTimeMicroSec),
+            _type(other._type),
+            _maxPoolSize(other._maxPoolSize) {
+            verify(_created == 0);
             verify(other._pool.size() == 0);
-            _created = other._created;
-            _minValidCreationTimeMicroSec = other._minValidCreationTimeMicroSec;
-            verify( _created == 0 );
         }
 
         ~PoolForHost();
 
+        /**
+         * Returns the maximum number of connections stored in the pool
+         */
+        int getMaxPoolSize() { return _maxPoolSize; }
+
+        /**
+         * Sets the maximum number of connections stored in the pool
+         */
+        void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; }
+
         int numAvailable() const { return (int)_pool.size(); }
 
         void createdOne( DBClientBase * base );
@@ -85,8 +105,6 @@ namespace mongo {
          */
         void initializeHostName(const std::string& hostName);
 
-        static void setMaxPerHost( unsigned max ) { _maxPerHost = max; }
-        static unsigned getMaxPerHost() { return _maxPerHost; }
     private:
 
         struct StoredConnection {
@@ -105,7 +123,8 @@ namespace mongo {
         uint64_t _minValidCreationTimeMicroSec;
         ConnectionString::ConnectionType _type;
 
-        static unsigned _maxPerHost;
+        // The maximum number of connections we'll save in the pool
+        int _maxPoolSize;
     };
 
     class DBConnectionHook {
@@ -141,6 +160,22 @@ namespace mongo {
         /** right now just controls some asserts.  defaults to "dbconnectionpool" */
         void setName( const string& name ) { _name = name; }
 
+        /**
+         * Returns the maximum number of connections pooled per-host
+         *
+         * This setting only applies to new host connection pools, previously-pooled host pools are
+         * unaffected.
+         */
+        int getMaxPoolSize() { return _maxPoolSize; }
+
+        /**
+         * Sets the maximum number of connections pooled per-host.
+         *
+         * This setting only applies to new host connection pools, previously-pooled host pools are
+         * unaffected.
+         */
+        void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; }
+
         void onCreate( DBClientBase * conn );
         void onHandedOut( DBClientBase * conn );
         void onDestroy( DBClientBase * conn );
@@ -204,6 +239,11 @@ namespace mongo {
         mongo::mutex _mutex;
         string _name;
 
+        // The maximum number of connections we'll save in the pool per-host
+        // PoolForHost::kPoolSizeUnlimited is a sentinel value meaning "no limit"
+        // 0 effectively disables the pool
+        int _maxPoolSize;
+
         PoolMap _pools;
 
         // pointers owned by me, right now they leak on shutdown
diff --git a/src/mongo/client/init.cpp b/src/mongo/client/init.cpp
index 39e3e9b251c..ffdd3d5797c 100644
--- a/src/mongo/client/init.cpp
+++ b/src/mongo/client/init.cpp
@@ -20,6 +20,7 @@
 #include <cstdlib>
 
 #include "mongo/base/initializer.h"
+#include "mongo/client/connpool.h"
 #include "mongo/client/replica_set_monitor.h"
 #include "mongo/util/background.h"
 
@@ -47,6 +48,10 @@ namespace client {
         if (!result.isOK())
             return result;
 
+        // Setup default pool parameters
+        mongo::pool.setName("connection pool");
+        mongo::pool.setMaxPoolSize(50);
+
         PeriodicTask::startRunningPeriodicTasks();
 
         return Status::OK();
diff --git a/src/mongo/client/scoped_db_conn_test.cpp b/src/mongo/client/scoped_db_conn_test.cpp
index 9c651bf374c..09109fbaa09 100644
--- a/src/mongo/client/scoped_db_conn_test.cpp
+++ b/src/mongo/client/scoped_db_conn_test.cpp
@@ -193,7 +193,7 @@ namespace mongo_test {
     class DummyServerFixture: public mongo::unittest::Test {
     public:
         void setUp() {
-            _maxPoolSizePerHost = mongo::PoolForHost::getMaxPerHost();
+            _maxPoolSizePerHost = mongo::pool.getMaxPoolSize();
             _dummyServer = new DummyServer(TARGET_PORT);
 
             _dummyServer->run(&dummyHandler);
@@ -217,7 +217,7 @@ namespace mongo_test {
             ScopedDbConnection::clearPool();
             delete _dummyServer;
 
-            mongo::PoolForHost::setMaxPerHost(_maxPoolSizePerHost);
+            mongo::pool.setMaxPoolSize(_maxPoolSizePerHost);
         }
 
     protected:
@@ -349,7 +349,7 @@ namespace mongo_test {
     }
 
     TEST_F(DummyServerFixture, InvalidateBadConnEvenWhenPoolIsFull) {
-        mongo::PoolForHost::setMaxPerHost(2);
+        mongo::pool.setMaxPoolSize(2);
 
         ScopedDbConnection conn1(TARGET_HOST);
         ScopedDbConnection conn2(TARGET_HOST);
diff --git a/src/mongo/db/conn_pool_options.cpp b/src/mongo/db/conn_pool_options.cpp
new file mode 100644
index 00000000000..0ac2c985920
--- /dev/null
+++ b/src/mongo/db/conn_pool_options.cpp
@@ -0,0 +1,75 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/db/conn_pool_options.h"
+
+#include "mongo/base/init.h"
+#include "mongo/db/server_parameters.h"
+#include "mongo/client/connpool.h"
+#include "mongo/s/shard.h"
+
+namespace mongo {
+
+    int ConnPoolOptions::maxConnsPerHost(200);
+    int ConnPoolOptions::maxShardedConnsPerHost(200);
+
+    namespace {
+
+        ExportedServerParameter<int> //
+        maxConnsPerHostParameter(ServerParameterSet::getGlobal(),
+                                 "connPoolMaxConnsPerHost",
+                                 &ConnPoolOptions::maxConnsPerHost,
+                                 true,
+                                 false /* can't change at runtime */);
+
+        ExportedServerParameter<int> //
+        maxShardedConnsPerHostParameter(ServerParameterSet::getGlobal(),
+                                        "connPoolMaxShardedConnsPerHost",
+                                        &ConnPoolOptions::maxShardedConnsPerHost,
+                                        true,
+                                        false /* can't change at runtime */);
+
+        MONGO_INITIALIZER(InitializeConnectionPools)(InitializerContext* context) {
+
+            // Initialize the sharded and unsharded outgoing connection pools
+            // NOTES:
+            // - All mongods and mongoses have both pools
+            // - The connection hooks for sharding are added on startup (mongos) or on first sharded
+            //   operation (mongod)
+
+            pool.setName("connection pool");
+            pool.setMaxPoolSize(ConnPoolOptions::maxConnsPerHost);
+
+            shardConnectionPool.setName("sharded connection pool");
+            shardConnectionPool.setMaxPoolSize(ConnPoolOptions::maxShardedConnsPerHost);
+
+            return Status::OK();
+        }
+    }
+
+}
diff --git a/src/mongo/db/conn_pool_options.h b/src/mongo/db/conn_pool_options.h
new file mode 100644
index 00000000000..a5402c6a367
--- /dev/null
+++ b/src/mongo/db/conn_pool_options.h
@@ -0,0 +1,54 @@
+/**
+ *    Copyright (C) 2014 MongoDB Inc.
+ *
+ *    This program is free software: you can redistribute it and/or  modify
+ *    it under the terms of the GNU Affero General Public License, version 3,
+ *    as published by the Free Software Foundation.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    GNU Affero General Public License for more details.
+ *
+ *    You should have received a copy of the GNU Affero General Public License
+ *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the GNU Affero General Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+namespace mongo {
+
+    // NOTE:
+    // The connection pools themselves are placed in different files and are currently hard to move
+    // due to spaghetti dependencies.
+    // TODO: Extract conn pools from driver files and shardconnection.cpp
+
+    /**
+     * Struct namespace for connection pool options on mongos and mongod
+     */
+    struct ConnPoolOptions {
+
+        /**
+         * Maximum connections per host the connection pool should use
+         */
+        static int maxConnsPerHost;
+
+        /**
+         * Maximum connections per host the sharded conn pool should use
+         */
+        static int maxShardedConnsPerHost;
+    };
+
+}
diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp
index 4083b00ab95..3a3bb821ad3 100644
--- a/src/mongo/s/server.cpp
+++ b/src/mongo/s/server.cpp
@@ -324,11 +324,9 @@ static bool runMongosServer( bool doUpgrade ) {
 
     // set some global state
 
+    // Add sharding hooks to both connection pools - ShardingConnectionHook includes auth hooks
     pool.addHook( new ShardingConnectionHook( false ) );
-    pool.setName( "mongos connectionpool" );
-
     shardConnectionPool.addHook( new ShardingConnectionHook( true ) );
-    shardConnectionPool.setName( "mongos shardconnection connectionpool" );
 
     // Mongos shouldn't lazily kill cursors, otherwise we can end up with extras from migration
     DBClientConnection::setLazyKillCursor( false );
diff --git a/src/mongo/s/shard_conn_test.cpp b/src/mongo/s/shard_conn_test.cpp
index f80be5236ec..844ca701b24 100644
--- a/src/mongo/s/shard_conn_test.cpp
+++ b/src/mongo/s/shard_conn_test.cpp
@@ -51,7 +51,7 @@ namespace mongo_test {
     class ShardConnFixture: public mongo::unittest::Test {
     public:
         void setUp() {
-            _maxPoolSizePerHost = mongo::PoolForHost::getMaxPerHost();
+            _maxPoolSizePerHost = mongo::shardConnectionPool.getMaxPoolSize();
 
             mongo::ConnectionString::setConnectionHook(
                     mongo::MockConnRegistry::get()->getConnStrHook());
@@ -68,7 +68,7 @@ namespace mongo_test {
             mongo::MockConnRegistry::get()->removeServer(_dummyServer->getServerAddress());
             delete _dummyServer;
 
-            mongo::PoolForHost::setMaxPerHost(_maxPoolSizePerHost);
+            mongo::shardConnectionPool.setMaxPoolSize(_maxPoolSizePerHost);
 
             mongo::clearGlobalAuthorizationManager();
         }
@@ -251,7 +251,7 @@ namespace mongo_test {
     }
 
     TEST_F(ShardConnFixture, InvalidateBadConnEvenWhenPoolIsFull) {
-        mongo::PoolForHost::setMaxPerHost(2);
+        mongo::shardConnectionPool.setMaxPoolSize(2);
 
         ShardConnection conn1(TARGET_HOST, "test.user");
         ShardConnection conn2(TARGET_HOST, "test.user");