From a12d09c9757301c3872cf4b45027d287e3dcc366 Mon Sep 17 00:00:00 2001 From: Greg Studer Date: Thu, 27 Mar 2014 12:31:12 -0400 Subject: SERVER-13374 change connection pool defaults (cherry picked from commit f71610672b0506b233a449bfafb303497c97ae50) --- src/mongo/SConscript | 1 + src/mongo/client/connpool.cpp | 32 +++++++++----- src/mongo/client/connpool.h | 58 ++++++++++++++++++++---- src/mongo/client/init.cpp | 5 +++ src/mongo/client/scoped_db_conn_test.cpp | 6 +-- src/mongo/db/conn_pool_options.cpp | 75 ++++++++++++++++++++++++++++++++ src/mongo/db/conn_pool_options.h | 54 +++++++++++++++++++++++ src/mongo/s/server.cpp | 4 +- src/mongo/s/shard_conn_test.cpp | 6 +-- 9 files changed, 211 insertions(+), 30 deletions(-) create mode 100644 src/mongo/db/conn_pool_options.cpp create mode 100644 src/mongo/db/conn_pool_options.h diff --git a/src/mongo/SConscript b/src/mongo/SConscript index e98c35925a4..daaa322c05d 100644 --- a/src/mongo/SConscript +++ b/src/mongo/SConscript @@ -484,6 +484,7 @@ coreServerFiles = [ "db/client_basic.cpp", "db/stats/counters.cpp", "db/stats/service_stats.cpp", "db/log_process_details.cpp", + "db/conn_pool_options.cpp" ] env.Library('ntservice', ['util/ntservice.cpp'], diff --git a/src/mongo/client/connpool.cpp b/src/mongo/client/connpool.cpp index 91ae84c21f9..6df97ea2d31 100644 --- a/src/mongo/client/connpool.cpp +++ b/src/mongo/client/connpool.cpp @@ -41,18 +41,23 @@ namespace mongo { } } - void PoolForHost::done( DBConnectionPool * pool, DBClientBase * c ) { - if (c->isFailed()) { - reportBadConnectionAt(c->getSockCreationMicroSec()); - pool->onDestroy(c); - delete c; - } - else if (_pool.size() >= _maxPerHost || - c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) { + void PoolForHost::done(DBConnectionPool* pool, DBClientBase* c) { + + bool isFailed = c->isFailed(); + + // Remember that this host had a broken connection for later + if (isFailed) reportBadConnectionAt(c->getSockCreationMicroSec()); + + if (isFailed || + // Another (later) connection was reported as broken to this host + (c->getSockCreationMicroSec() < _minValidCreationTimeMicroSec) || + // We have a pool size that we need to enforce + (_maxPoolSize >= 0 && static_cast(_pool.size()) >= _maxPoolSize)) { pool->onDestroy(c); delete c; } else { + // The connection is probably fine, save for later _pool.push(c); } } @@ -166,16 +171,17 @@ namespace mongo { } } - unsigned PoolForHost::_maxPerHost = 50; - // ------ DBConnectionPool ------ DBConnectionPool pool; - DBConnectionPool::DBConnectionPool() + const int PoolForHost::kPoolSizeUnlimited(-1); + + DBConnectionPool::DBConnectionPool() : _mutex("DBConnectionPool") , _name( "dbconnectionpool" ) , - _hooks( new list() ) { + _maxPoolSize(PoolForHost::kPoolSizeUnlimited) , + _hooks( new list() ) { } DBClientBase* DBConnectionPool::_get(const string& ident , double socketTimeout ) { @@ -183,6 +189,7 @@ namespace mongo { !inShutdown()); scoped_lock L(_mutex); PoolForHost& p = _pools[PoolKey(ident,socketTimeout)]; + p.setMaxPoolSize(_maxPoolSize); p.initializeHostName(ident); return p.get( this , socketTimeout ); } @@ -191,6 +198,7 @@ namespace mongo { { scoped_lock L(_mutex); PoolForHost& p = _pools[PoolKey(host,socketTimeout)]; + p.setMaxPoolSize(_maxPoolSize); p.initializeHostName(host); p.createdOne( conn ); } diff --git a/src/mongo/client/connpool.h b/src/mongo/client/connpool.h index 694ce53d6dc..9bf4583065f 100644 --- a/src/mongo/client/connpool.h +++ b/src/mongo/client/connpool.h @@ -35,18 +35,38 @@ namespace mongo { */ class MONGO_CLIENT_API PoolForHost { public: - PoolForHost() - : _created(0), _minValidCreationTimeMicroSec(0) {} - PoolForHost( const PoolForHost& other ) { + // Sentinel value indicating pool has no cleanup limit + static const int kPoolSizeUnlimited; + + PoolForHost() : + _created(0), + _minValidCreationTimeMicroSec(0), + _type(ConnectionString::INVALID), + _maxPoolSize(kPoolSizeUnlimited) { + } + + PoolForHost(const PoolForHost& other) : + _created(other._created), + _minValidCreationTimeMicroSec(other._minValidCreationTimeMicroSec), + _type(other._type), + _maxPoolSize(other._maxPoolSize) { + verify(_created == 0); verify(other._pool.size() == 0); - _created = other._created; - _minValidCreationTimeMicroSec = other._minValidCreationTimeMicroSec; - verify( _created == 0 ); } ~PoolForHost(); + /** + * Returns the maximum number of connections stored in the pool + */ + int getMaxPoolSize() { return _maxPoolSize; } + + /** + * Sets the maximum number of connections stored in the pool + */ + void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; } + int numAvailable() const { return (int)_pool.size(); } void createdOne( DBClientBase * base ); @@ -85,8 +105,6 @@ namespace mongo { */ void initializeHostName(const std::string& hostName); - static void setMaxPerHost( unsigned max ) { _maxPerHost = max; } - static unsigned getMaxPerHost() { return _maxPerHost; } private: struct StoredConnection { @@ -105,7 +123,8 @@ namespace mongo { uint64_t _minValidCreationTimeMicroSec; ConnectionString::ConnectionType _type; - static unsigned _maxPerHost; + // The maximum number of connections we'll save in the pool + int _maxPoolSize; }; class DBConnectionHook { @@ -141,6 +160,22 @@ namespace mongo { /** right now just controls some asserts. defaults to "dbconnectionpool" */ void setName( const string& name ) { _name = name; } + /** + * Returns the maximum number of connections pooled per-host + * + * This setting only applies to new host connection pools, previously-pooled host pools are + * unaffected. + */ + int getMaxPoolSize() { return _maxPoolSize; } + + /** + * Sets the maximum number of connections pooled per-host. + * + * This setting only applies to new host connection pools, previously-pooled host pools are + * unaffected. + */ + void setMaxPoolSize( int maxPoolSize ) { _maxPoolSize = maxPoolSize; } + void onCreate( DBClientBase * conn ); void onHandedOut( DBClientBase * conn ); void onDestroy( DBClientBase * conn ); @@ -204,6 +239,11 @@ namespace mongo { mongo::mutex _mutex; string _name; + // The maximum number of connections we'll save in the pool per-host + // PoolForHost::kPoolSizeUnlimited is a sentinel value meaning "no limit" + // 0 effectively disables the pool + int _maxPoolSize; + PoolMap _pools; // pointers owned by me, right now they leak on shutdown diff --git a/src/mongo/client/init.cpp b/src/mongo/client/init.cpp index 39e3e9b251c..ffdd3d5797c 100644 --- a/src/mongo/client/init.cpp +++ b/src/mongo/client/init.cpp @@ -20,6 +20,7 @@ #include #include "mongo/base/initializer.h" +#include "mongo/client/connpool.h" #include "mongo/client/replica_set_monitor.h" #include "mongo/util/background.h" @@ -47,6 +48,10 @@ namespace client { if (!result.isOK()) return result; + // Setup default pool parameters + mongo::pool.setName("connection pool"); + mongo::pool.setMaxPoolSize(50); + PeriodicTask::startRunningPeriodicTasks(); return Status::OK(); diff --git a/src/mongo/client/scoped_db_conn_test.cpp b/src/mongo/client/scoped_db_conn_test.cpp index 9c651bf374c..09109fbaa09 100644 --- a/src/mongo/client/scoped_db_conn_test.cpp +++ b/src/mongo/client/scoped_db_conn_test.cpp @@ -193,7 +193,7 @@ namespace mongo_test { class DummyServerFixture: public mongo::unittest::Test { public: void setUp() { - _maxPoolSizePerHost = mongo::PoolForHost::getMaxPerHost(); + _maxPoolSizePerHost = mongo::pool.getMaxPoolSize(); _dummyServer = new DummyServer(TARGET_PORT); _dummyServer->run(&dummyHandler); @@ -217,7 +217,7 @@ namespace mongo_test { ScopedDbConnection::clearPool(); delete _dummyServer; - mongo::PoolForHost::setMaxPerHost(_maxPoolSizePerHost); + mongo::pool.setMaxPoolSize(_maxPoolSizePerHost); } protected: @@ -349,7 +349,7 @@ namespace mongo_test { } TEST_F(DummyServerFixture, InvalidateBadConnEvenWhenPoolIsFull) { - mongo::PoolForHost::setMaxPerHost(2); + mongo::pool.setMaxPoolSize(2); ScopedDbConnection conn1(TARGET_HOST); ScopedDbConnection conn2(TARGET_HOST); diff --git a/src/mongo/db/conn_pool_options.cpp b/src/mongo/db/conn_pool_options.cpp new file mode 100644 index 00000000000..0ac2c985920 --- /dev/null +++ b/src/mongo/db/conn_pool_options.cpp @@ -0,0 +1,75 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/db/conn_pool_options.h" + +#include "mongo/base/init.h" +#include "mongo/db/server_parameters.h" +#include "mongo/client/connpool.h" +#include "mongo/s/shard.h" + +namespace mongo { + + int ConnPoolOptions::maxConnsPerHost(200); + int ConnPoolOptions::maxShardedConnsPerHost(200); + + namespace { + + ExportedServerParameter // + maxConnsPerHostParameter(ServerParameterSet::getGlobal(), + "connPoolMaxConnsPerHost", + &ConnPoolOptions::maxConnsPerHost, + true, + false /* can't change at runtime */); + + ExportedServerParameter // + maxShardedConnsPerHostParameter(ServerParameterSet::getGlobal(), + "connPoolMaxShardedConnsPerHost", + &ConnPoolOptions::maxShardedConnsPerHost, + true, + false /* can't change at runtime */); + + MONGO_INITIALIZER(InitializeConnectionPools)(InitializerContext* context) { + + // Initialize the sharded and unsharded outgoing connection pools + // NOTES: + // - All mongods and mongoses have both pools + // - The connection hooks for sharding are added on startup (mongos) or on first sharded + // operation (mongod) + + pool.setName("connection pool"); + pool.setMaxPoolSize(ConnPoolOptions::maxConnsPerHost); + + shardConnectionPool.setName("sharded connection pool"); + shardConnectionPool.setMaxPoolSize(ConnPoolOptions::maxShardedConnsPerHost); + + return Status::OK(); + } + } + +} diff --git a/src/mongo/db/conn_pool_options.h b/src/mongo/db/conn_pool_options.h new file mode 100644 index 00000000000..a5402c6a367 --- /dev/null +++ b/src/mongo/db/conn_pool_options.h @@ -0,0 +1,54 @@ +/** + * Copyright (C) 2014 MongoDB Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the GNU Affero General Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { + + // NOTE: + // The connection pools themselves are placed in different files and are currently hard to move + // due to spaghetti dependencies. + // TODO: Extract conn pools from driver files and shardconnection.cpp + + /** + * Struct namespace for connection pool options on mongos and mongod + */ + struct ConnPoolOptions { + + /** + * Maximum connections per host the connection pool should use + */ + static int maxConnsPerHost; + + /** + * Maximum connections per host the sharded conn pool should use + */ + static int maxShardedConnsPerHost; + }; + +} diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 4083b00ab95..3a3bb821ad3 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -324,11 +324,9 @@ static bool runMongosServer( bool doUpgrade ) { // set some global state + // Add sharding hooks to both connection pools - ShardingConnectionHook includes auth hooks pool.addHook( new ShardingConnectionHook( false ) ); - pool.setName( "mongos connectionpool" ); - shardConnectionPool.addHook( new ShardingConnectionHook( true ) ); - shardConnectionPool.setName( "mongos shardconnection connectionpool" ); // Mongos shouldn't lazily kill cursors, otherwise we can end up with extras from migration DBClientConnection::setLazyKillCursor( false ); diff --git a/src/mongo/s/shard_conn_test.cpp b/src/mongo/s/shard_conn_test.cpp index f80be5236ec..844ca701b24 100644 --- a/src/mongo/s/shard_conn_test.cpp +++ b/src/mongo/s/shard_conn_test.cpp @@ -51,7 +51,7 @@ namespace mongo_test { class ShardConnFixture: public mongo::unittest::Test { public: void setUp() { - _maxPoolSizePerHost = mongo::PoolForHost::getMaxPerHost(); + _maxPoolSizePerHost = mongo::shardConnectionPool.getMaxPoolSize(); mongo::ConnectionString::setConnectionHook( mongo::MockConnRegistry::get()->getConnStrHook()); @@ -68,7 +68,7 @@ namespace mongo_test { mongo::MockConnRegistry::get()->removeServer(_dummyServer->getServerAddress()); delete _dummyServer; - mongo::PoolForHost::setMaxPerHost(_maxPoolSizePerHost); + mongo::shardConnectionPool.setMaxPoolSize(_maxPoolSizePerHost); mongo::clearGlobalAuthorizationManager(); } @@ -251,7 +251,7 @@ namespace mongo_test { } TEST_F(ShardConnFixture, InvalidateBadConnEvenWhenPoolIsFull) { - mongo::PoolForHost::setMaxPerHost(2); + mongo::shardConnectionPool.setMaxPoolSize(2); ShardConnection conn1(TARGET_HOST, "test.user"); ShardConnection conn2(TARGET_HOST, "test.user"); -- cgit v1.2.1