From 6385224862d53c94235a006547a4eb9e77f990a8 Mon Sep 17 00:00:00 2001 From: Luis Osta Date: Wed, 3 Nov 2021 13:52:02 +0000 Subject: SERVER-57686 Create concurrency test for inserts during reshardCollection --- ...currency_sharded_kill_primary_with_balancer.yml | 3 + ...urrency_sharded_multi_stmt_txn_kill_primary.yml | 3 + ...cy_sharded_multi_stmt_txn_terminate_primary.yml | 3 + ...rency_sharded_multi_stmt_txn_with_stepdowns.yml | 3 + ...ncy_sharded_terminate_primary_with_balancer.yml | 3 + .../suites/concurrency_sharded_with_stepdowns.yml | 3 + ...urrency_sharded_with_stepdowns_and_balancer.yml | 3 + buildscripts/resmokelib/core/programs.py | 3 - .../resmokelib/testing/fixtures/standalone.py | 5 +- .../fsm_workloads/reshard_collection_crud_ops.js | 98 ++++++++++++++++++++++ 10 files changed, 122 insertions(+), 5 deletions(-) create mode 100644 jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml index 804fdb8f720..15be7f0daf8 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml @@ -221,6 +221,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml index 95c712d59dd..b718dd3a0cf 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml @@ -271,6 +271,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml index 5d734d13c1c..5874c4b51bc 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml @@ -272,6 +272,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml index 7f22963a6c8..e93cbeddb6d 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml @@ -245,6 +245,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml index a1dcb438a88..8764a2150c1 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml @@ -217,6 +217,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml index f523a34b9e7..f57b071ba23 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml @@ -208,6 +208,9 @@ executor: configsvr_options: num_nodes: 3 all_nodes_electable: true + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds replset_config_options: settings: catchUpTimeoutMillis: 0 diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml index bbb7a502910..472a2662ba2 100644 --- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml +++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml @@ -215,6 +215,9 @@ executor: replset_config_options: settings: catchUpTimeoutMillis: 0 + mongod_options: + set_parameters: + reshardingMinimumOperationDurationMillis: 30000 # 30 seconds shard_options: all_nodes_electable: true mongod_options: diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py index f9154ff202c..08e224e917a 100644 --- a/buildscripts/resmokelib/core/programs.py +++ b/buildscripts/resmokelib/core/programs.py @@ -211,9 +211,6 @@ def mongo_shell_program( # pylint: disable=too-many-arguments,too-many-branches if config.FLOW_CONTROL is not None: mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL == "on") - # Set the default value for minimum resharding operation duration to 5 seconds. - mongod_set_parameters.setdefault("reshardingMinimumOperationDurationMillis", 5000) - mongos_launcher = shardedcluster.MongosLauncher(fixturelib) # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its # value to a default. diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index d563a313703..136569a5f02 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -253,8 +253,6 @@ class MongodLauncher(object): if "coordinateCommitReturnImmediatelyAfterPersistingDecision" not in suite_set_parameters: suite_set_parameters["coordinateCommitReturnImmediatelyAfterPersistingDecision"] = False - suite_set_parameters["reshardingMinimumOperationDurationMillis"] = 5000 - # There's a periodic background thread that checks for and aborts expired transactions. # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring # and being aborted by the background thread. It defaults to 60 seconds, which is too short to @@ -323,6 +321,9 @@ class MongodLauncher(object): if "configsvr" in mongod_options: shortcut_opts["nojournal"] = False mongod_options["journal"] = "" + suite_set_parameters.setdefault("reshardingMinimumOperationDurationMillis", 5000) + suite_set_parameters.setdefault("reshardingCriticalSectionTimeoutMillis", + 24 * 60 * 60) # 24 hours # Command line options override the YAML configuration. for opt_name in shortcut_opts: diff --git a/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js b/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js new file mode 100644 index 00000000000..28a0fb258a5 --- /dev/null +++ b/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js @@ -0,0 +1,98 @@ +'use strict'; + +/** + * Runs reshardCollection and CRUD operations concurrently. + * + * @tags: [requires_sharding] + */ + +var $config = (function() { + const shardKeys = [ + {a: 1}, + {b: 1}, + ]; + + const data = { + shardKey: shardKeys[0], + currentShardKeyIndex: 0, + }; + + const iterations = 25; + const kTotalWorkingDocuments = 1000; + const kMaxReshardingExecutions = TestData.runningWithShardStepdowns ? 4 : iterations; + + /** + * @summary Takes in a number of documents to create, creates each document. With two properties + * being equal to the index and one counter property. + * @param {number} numDocs + * @returns {Array{Object}} an array of documents to be inserted into the collection. + */ + function createDocuments(numDocs) { + const documents = Array.from({length: numDocs}).map((_, i) => ({a: i, b: i, c: 0})); + return documents; + } + + function executeReshardCommand(db, collName, newShardKey) { + const coll = db.getCollection(collName); + print(`Started Resharding Collection ${coll.getFullName()}. New Shard Key ${ + tojson(newShardKey)}`); + if (TestData.runningWithShardStepdowns) { + assert.commandWorkedOrFailedWithCode( + db.adminCommand({reshardCollection: coll.getFullName(), key: newShardKey}), + [ErrorCodes.SnapshotUnavailable]); + } else { + assert.commandWorked( + db.adminCommand({reshardCollection: coll.getFullName(), key: newShardKey})); + } + print(`Finished Resharding Collection ${coll.getFullName()}. New Shard Key ${ + tojson(newShardKey)}`); + } + + const states = { + insert: function insert(db, collName) { + const coll = db.getCollection(collName); + print(`Inserting documents for collection ${coll.getFullName()}.`); + const totalDocumentsToInsert = 10; + assert.commandWorked(coll.insert(createDocuments(totalDocumentsToInsert))); + print(`Finished Inserting documents.`); + }, + reshardCollection: function reshardCollection(db, collName) { + //'reshardingMinimumOperationDurationMillis' is set to 30 seconds when there are + // stepdowns. So in order to limit the overall time for the test, we limit the number of + // resharding operations to kMaxReshardingExecutions. + const shouldContinueResharding = this.reshardingCount <= kMaxReshardingExecutions; + if (this.tid === 0 && shouldContinueResharding) { + const currentShardKeyIndex = this.currentShardKeyIndex; + const newIndex = (currentShardKeyIndex + 1) % shardKeys.length; + const shardKey = shardKeys[newIndex]; + + executeReshardCommand(db, collName, shardKey); + // If resharding fails with SnapshopUnavailable, then this will be incorrect. But + // its fine since reshardCollection will succeed if the new shard key matches the + // existing one. + this.currentShardKeyIndex = shardKeyIndex; + this.reshardingCount += 1; + } + } + }; + + const transitions = { + reshardCollection: {insert: 1}, + insert: {insert: .5, reshardCollection: .5} + }; + + function setup(db, collName, _cluster) { + const coll = db.getCollection(collName); + assert.commandWorked(coll.insert(createDocuments(kTotalWorkingDocuments))); + } + + return { + threadCount: 20, + iterations: iterations, + startState: 'reshardCollection', + states: states, + transitions: transitions, + setup: setup, + data: data + }; +})(); -- cgit v1.2.1