From 5f3625880188f9ca76bacb02c541abc2bd2d0f96 Mon Sep 17 00:00:00 2001 From: Eric Milkie Date: Wed, 12 Feb 2020 12:54:17 -0500 Subject: SERVER-44984 reduce index build thread pool default and index build external sort memory limit default This reverts commit b48cb5d8e2a8ccab0f4be401cc2533e5c2b650ae. --- etc/evergreen.yml | 14 ++------------ src/mongo/db/catalog/multi_index_block.idl | 6 +++--- src/mongo/db/index_builds_coordinator_mongod.cpp | 6 +++++- src/mongo/db/index_builds_coordinator_mongod.h | 1 + src/mongo/db/index_builds_coordinator_mongod_test.cpp | 10 +++++++++- 5 files changed, 20 insertions(+), 17 deletions(-) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 6c80f3a961e..a00fc3cdbeb 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -5074,12 +5074,10 @@ tasks: num_tasks: 10 jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_continuous_stepdown - # TODO(SERVER-44896): Remove enableTwoPhaseIndexBuild from server parameters when two phase - # index builds work with this flow control setting. resmoke_args: >- --flowControlTicketOverride=3 --storageEngine=wiredTiger - --mongodSetParameters="{logComponentVerbosity: {command: 2}, enableTwoPhaseIndexBuild: false}" + --mongodSetParameters="{logComponentVerbosity: {command: 2}}" name: jstestfuzz_sharded_continuous_stepdown_flow_control ## jstestfuzz concurrent sharded cluster continuous stepdown with flow control engaged ## @@ -5094,13 +5092,10 @@ tasks: num_tasks: 2 jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_sharded_continuous_stepdown - # TODO(SERVER-44896): Remove enableTwoPhaseIndexBuild from server parameters when two phase - # index builds work with this flow control setting. resmoke_args: >- --flowControlTicketOverride=30 --storageEngine=wiredTiger --numClientsPerFixture=10 - --mongodSetParameters="{enableTwoPhaseIndexBuild: false}" name: jstestfuzz_concurrent_sharded_continuous_stepdown_flow_control # jstestfuzz replication continuous stepdown with flow control engaged # @@ -5115,12 +5110,10 @@ tasks: num_tasks: 30 jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_continuous_stepdown - # TODO(SERVER-44896): Remove enableTwoPhaseIndexBuild from server parameters when two phase - # index builds work with this flow control setting. resmoke_args: >- --flowControlTicketOverride=1 --storageEngine=wiredTiger - --mongodSetParameters="{logComponentVerbosity: {command: 2}, enableTwoPhaseIndexBuild: false}" + --mongodSetParameters="{logComponentVerbosity: {command: 2}}" name: jstestfuzz_replication_continuous_stepdown_flow_control ## jstestfuzz concurrent replication continuous stepdown with flow control engaged ## @@ -5135,13 +5128,10 @@ tasks: num_tasks: 15 jstestfuzz_vars: --jsTestsDir ../jstests suite: jstestfuzz_replication_continuous_stepdown - # TODO(SERVER-44896): Remove enableTwoPhaseIndexBuild from server parameters when two phase - # index builds work with this flow control setting. resmoke_args: >- --flowControlTicketOverride=10 --storageEngine=wiredTiger --numClientsPerFixture=10 - --mongodSetParameters="{enableTwoPhaseIndexBuild: false}" name: jstestfuzz_concurrent_replication_continuous_stepdown_flow_control ## jstestfuzz replica set ## diff --git a/src/mongo/db/catalog/multi_index_block.idl b/src/mongo/db/catalog/multi_index_block.idl index 08b183e3560..311a7f701f1 100644 --- a/src/mongo/db/catalog/multi_index_block.idl +++ b/src/mongo/db/catalog/multi_index_block.idl @@ -52,12 +52,12 @@ server_parameters: default: true maxIndexBuildMemoryUsageMegabytes: - description: "Limits the amount of memory that simultaneous foreground index builds on one collection may consume for the duration of the builds" + description: "Limits the amount of memory that simultaneous index builds on one collection may consume for the duration of the builds" set_at: - runtime - startup cpp_varname: maxIndexBuildMemoryUsageMegabytes cpp_vartype: AtomicWord - default: 500 + default: 200 validator: - gte: 100 + gte: 50 diff --git a/src/mongo/db/index_builds_coordinator_mongod.cpp b/src/mongo/db/index_builds_coordinator_mongod.cpp index c67de58a3fc..c386ac34e26 100644 --- a/src/mongo/db/index_builds_coordinator_mongod.cpp +++ b/src/mongo/db/index_builds_coordinator_mongod.cpp @@ -70,7 +70,7 @@ ThreadPool::Options makeDefaultThreadPoolOptions() { // We depend on thread pool sizes being equal between primaries and secondaries. If a secondary // has fewer resources than a primary, index build oplog entries can replicate in an order that // the secondary is unable to fulfill, leading to deadlocks. See SERVER-44250. - options.maxThreads = 10; + options.maxThreads = 3; // Ensure all threads have a client. options.onCreateThread = [](const std::string& threadName) { @@ -86,6 +86,10 @@ IndexBuildsCoordinatorMongod::IndexBuildsCoordinatorMongod() : _threadPool(makeDefaultThreadPoolOptions()) { _threadPool.startup(); } +IndexBuildsCoordinatorMongod::IndexBuildsCoordinatorMongod(ThreadPool::Options options) + : _threadPool(std::move(options)) { + _threadPool.startup(); +} void IndexBuildsCoordinatorMongod::shutdown() { // Stop new scheduling. diff --git a/src/mongo/db/index_builds_coordinator_mongod.h b/src/mongo/db/index_builds_coordinator_mongod.h index 005779a26a2..3e86e06397a 100644 --- a/src/mongo/db/index_builds_coordinator_mongod.h +++ b/src/mongo/db/index_builds_coordinator_mongod.h @@ -55,6 +55,7 @@ public: * Sets up the thread pool. */ IndexBuildsCoordinatorMongod(); + IndexBuildsCoordinatorMongod(ThreadPool::Options options); /** * Shuts down the thread pool, signals interrupt to all index builds, then waits for all of the diff --git a/src/mongo/db/index_builds_coordinator_mongod_test.cpp b/src/mongo/db/index_builds_coordinator_mongod_test.cpp index 8709041d43f..8490b157381 100644 --- a/src/mongo/db/index_builds_coordinator_mongod_test.cpp +++ b/src/mongo/db/index_builds_coordinator_mongod_test.cpp @@ -73,7 +73,15 @@ void IndexBuildsCoordinatorMongodTest::setUp() { createCollection(_testFooNss, _testFooUUID); createCollection(_testBarNss, _testBarUUID); createCollection(_othertestFooNss, _othertestFooUUID); - _indexBuildsCoord = std::make_unique(); + + ThreadPool::Options options; + options.poolName = "IndexBuildsCoordinatorMongod"; + options.minThreads = 0; + options.maxThreads = 5; + options.onCreateThread = [](const std::string& threadName) { + Client::initThread(threadName.c_str()); + }; + _indexBuildsCoord = std::make_unique(options); } void IndexBuildsCoordinatorMongodTest::tearDown() { -- cgit v1.2.1