From b375698b7fe1f4e69761559f1cad50c5e1f18014 Mon Sep 17 00:00:00 2001 From: Anton Oyung Date: Wed, 13 Nov 2019 19:16:41 +0000 Subject: SERVER-44152: Pre-warm connection pools in mongos --- src/mongo/s/SConscript | 2 + src/mongo/s/mongos_options.idl | 16 ++++++ src/mongo/s/pre_warm_connection_pool_impl.cpp | 38 ++++++++++++++ src/mongo/s/pre_warm_connection_pool_impl.h | 41 ++++++++++++++++ src/mongo/s/server.cpp | 5 ++ src/mongo/s/sharding_initialization.cpp | 71 +++++++++++++++++++++++++++ src/mongo/s/sharding_initialization.h | 6 +++ 7 files changed, 179 insertions(+) create mode 100644 src/mongo/s/pre_warm_connection_pool_impl.cpp create mode 100644 src/mongo/s/pre_warm_connection_pool_impl.h (limited to 'src/mongo') diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript index 921c2a52b7a..1d4cc94c795 100644 --- a/src/mongo/s/SConscript +++ b/src/mongo/s/SConscript @@ -105,11 +105,13 @@ env.Library( source=[ 'sharding_initialization.cpp', 'sharding_task_executor_pool_controller.cpp', + 'pre_warm_connection_pool_impl.cpp', env.Idlc('sharding_task_executor_pool.idl')[0], 'client/sharding_connection_hook.cpp', 'client/sharding_network_connection_hook.cpp', ], LIBDEPS=[ + '$BUILD_DIR/mongo/executor/async_multicaster', '$BUILD_DIR/mongo/s/catalog/dist_lock_catalog_impl', '$BUILD_DIR/mongo/s/catalog/replset_dist_lock_manager', '$BUILD_DIR/mongo/s/catalog/sharding_catalog_client_impl', diff --git a/src/mongo/s/mongos_options.idl b/src/mongo/s/mongos_options.idl index f92d2474fad..5c8033a24b0 100644 --- a/src/mongo/s/mongos_options.idl +++ b/src/mongo/s/mongos_options.idl @@ -31,10 +31,26 @@ global: cpp_namespace: "mongo" cpp_includes: - "mongo/s/mongos_options.h" + - "mongo/s/pre_warm_connection_pool_impl.h" configs: section: "Sharding options" source: [ yaml, cli, ini ] +server_parameters: + warmMinConnectionsInShardingTaskExecutorPoolOnStartup: + description: <- + Enables prewarming of the connection pool. + set_at: [ startup ] + cpp_varname: "gWarmMinConnectionsInShardingTaskExecutorPoolOnStartup" + default: true + + warmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS: + description: <- + How long to wait for all hosts to have at least one connection. + set_at: [ startup ] + cpp_varname: "gWarmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS" + default: 2000 # 2secs + configs: "sharding.configDB": description: >- diff --git a/src/mongo/s/pre_warm_connection_pool_impl.cpp b/src/mongo/s/pre_warm_connection_pool_impl.cpp new file mode 100644 index 00000000000..cb082983ce1 --- /dev/null +++ b/src/mongo/s/pre_warm_connection_pool_impl.cpp @@ -0,0 +1,38 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#include "mongo/s/pre_warm_connection_pool_impl.h" + + +namespace mongo { + +bool gWarmMinConnectionsInShardingTaskExecutorPoolOnStartup = true; +int gWarmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS = 2000; + +} // namespace mongo \ No newline at end of file diff --git a/src/mongo/s/pre_warm_connection_pool_impl.h b/src/mongo/s/pre_warm_connection_pool_impl.h new file mode 100644 index 00000000000..f5b4a5dfbe0 --- /dev/null +++ b/src/mongo/s/pre_warm_connection_pool_impl.h @@ -0,0 +1,41 @@ +/** + * Copyright (C) 2019-present MongoDB, Inc. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + * + * As a special exception, the copyright holders give permission to link the + * code of portions of this program with the OpenSSL library under certain + * conditions as described in each individual source file and distribute + * linked combinations including the program with the OpenSSL library. You + * must comply with the Server Side Public License in all respects for + * all of the code used other than as permitted herein. If you modify file(s) + * with this exception, you may extend this exception to your version of the + * file(s), but you are not obligated to do so. If you do not wish to do so, + * delete this exception statement from your version. If you delete this + * exception statement from all source files in the program, then also delete + * it in the license file. + */ + +#pragma once + +namespace mongo { +/** + * Set parameters used to control whether or not the mongos attempts to warm up the connection + * pool on start up and for how long it should try. + */ + +extern bool gWarmMinConnectionsInShardingTaskExecutorPoolOnStartup; +extern int gWarmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS; + +} // namespace mongo diff --git a/src/mongo/s/server.cpp b/src/mongo/s/server.cpp index 18297cdd02e..ddc851749fe 100644 --- a/src/mongo/s/server.cpp +++ b/src/mongo/s/server.cpp @@ -426,6 +426,11 @@ Status initializeSharding(OperationContext* opCtx) { return status; } + status = preWarmConnectionPool(opCtx); + if (!status.isOK()) { + return status; + } + Grid::get(opCtx)->setShardingInitialized(); return Status::OK(); diff --git a/src/mongo/s/sharding_initialization.cpp b/src/mongo/s/sharding_initialization.cpp index db076c843ff..f3598d797f3 100644 --- a/src/mongo/s/sharding_initialization.cpp +++ b/src/mongo/s/sharding_initialization.cpp @@ -45,9 +45,12 @@ #include "mongo/db/server_options.h" #include "mongo/db/service_context.h" #include "mongo/db/time_proof_service.h" +#include "mongo/executor/async_multicaster.h" #include "mongo/executor/connection_pool.h" +#include "mongo/executor/connection_pool_stats.h" #include "mongo/executor/network_interface_factory.h" #include "mongo/executor/network_interface_thread_pool.h" +#include "mongo/executor/scoped_task_executor.h" #include "mongo/executor/task_executor.h" #include "mongo/executor/task_executor_pool.h" #include "mongo/executor/thread_pool_task_executor.h" @@ -57,11 +60,13 @@ #include "mongo/s/catalog/dist_lock_catalog_impl.h" #include "mongo/s/catalog/replset_dist_lock_manager.h" #include "mongo/s/catalog/sharding_catalog_client_impl.h" +#include "mongo/s/catalog/type_shard.h" #include "mongo/s/catalog_cache.h" #include "mongo/s/client/shard_factory.h" #include "mongo/s/client/sharding_network_connection_hook.h" #include "mongo/s/cluster_identity_loader.h" #include "mongo/s/grid.h" +#include "mongo/s/pre_warm_connection_pool_impl.h" #include "mongo/s/query/cluster_cursor_manager.h" #include "mongo/s/sharding_task_executor.h" #include "mongo/s/sharding_task_executor_pool_controller.h" @@ -140,6 +145,26 @@ std::unique_ptr makeShardingTaskExecutorPool( return executorPool; } +/** + * Uses an AsyncMulticaster to ping all of the hosts in order to establish + * ShardingTaskExecutorPoolMinSize connections. This does not wait + * for the connections to be established nor does it check how many were established. + */ +void preWarmConnections(OperationContext* opCtx, std::vector allHosts) { + auto const grid = Grid::get(opCtx); + auto arbi = grid->getExecutorPool()->getArbitraryExecutor(); + auto executor = executor::ScopedTaskExecutor(arbi); + executor::AsyncMulticaster::Options options; + + auto results = + executor::AsyncMulticaster(*executor, options) + .multicast(allHosts, + "admin", + BSON("ping" << 1), + opCtx, + Milliseconds(gWarmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS)); +} + } // namespace std::unique_ptr makeShardingTaskExecutor( @@ -264,4 +289,50 @@ Status preCacheMongosRoutingInfo(OperationContext* opCtx) { return Status::OK(); } +Status preWarmConnectionPool(OperationContext* opCtx) { + if (!gWarmMinConnectionsInShardingTaskExecutorPoolOnStartup) { + return Status::OK(); + } + + if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) { + return Status::OK(); + } + + // Should not be called by mongod + invariant(serverGlobalParams.clusterRole != ClusterRole::ShardServer); + + std::vector allHosts; + auto const grid = Grid::get(opCtx); + auto allShardsStatus = + grid->catalogClient()->getAllShards(opCtx, repl::ReadConcernLevel::kMajorityReadConcern); + if (!allShardsStatus.isOK()) { + return allShardsStatus.getStatus(); + } + auto allShards = allShardsStatus.getValue().value; + + for (auto& shard : allShards) { + auto connStrStatus = ConnectionString::parse(shard.getHost()); + if (!connStrStatus.isOK()) { + return connStrStatus.getStatus(); + } + auto connStr = connStrStatus.getValue(); + for (const auto& hostEntry : connStr.getServers()) { + allHosts.push_back(hostEntry); + } + } + try { + opCtx->runWithDeadline( + opCtx->getServiceContext()->getPreciseClockSource()->now() + + Milliseconds(gWarmMinConnectionsInShardingTaskExecutorPoolOnStartupWaitMS), + ErrorCodes::ExceededTimeLimit, + [&] { preWarmConnections(opCtx, allHosts); }); + } catch (const ExceptionFor&) { + // if we've timed out, eat the exception and continue + } catch (const DBException& ex) { + return ex.toStatus(); + } + + return Status::OK(); +} + } // namespace mongo diff --git a/src/mongo/s/sharding_initialization.h b/src/mongo/s/sharding_initialization.h index 4908ad48258..33ceee5e0d5 100644 --- a/src/mongo/s/sharding_initialization.h +++ b/src/mongo/s/sharding_initialization.h @@ -95,4 +95,10 @@ Status waitForShardRegistryReload(OperationContext* opCtx); Status preCacheMongosRoutingInfo(OperationContext* opCtx); +/** + * Warms up connections to shards with best effort strategy. + */ + +Status preWarmConnectionPool(OperationContext* opCtx); + } // namespace mongo -- cgit v1.2.1