SERVER-57686 Create concurrency test for inserts during reshardCollection

10 files changed, 122 insertions, 5 deletions

diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
index 804fdb8f720..15be7f0daf8 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_kill_primary_with_balancer.yml
@@ -221,6 +221,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
index 95c712d59dd..b718dd3a0cf 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_kill_primary.yml
@@ -271,6 +271,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
index 5d734d13c1c..5874c4b51bc 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_terminate_primary.yml
@@ -272,6 +272,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
index 7f22963a6c8..e93cbeddb6d 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_multi_stmt_txn_with_stepdowns.yml
@@ -245,6 +245,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
index a1dcb438a88..8764a2150c1 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_terminate_primary_with_balancer.yml
@@ -217,6 +217,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
index f523a34b9e7..f57b071ba23 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns.yml
@@ -208,6 +208,9 @@ executor:
     configsvr_options:
       num_nodes: 3
       all_nodes_electable: true
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
index bbb7a502910..472a2662ba2 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_with_stepdowns_and_balancer.yml
@@ -215,6 +215,9 @@ executor:
       replset_config_options:
         settings:
           catchUpTimeoutMillis: 0
+      mongod_options:
+        set_parameters:
+          reshardingMinimumOperationDurationMillis: 30000 # 30 seconds
     shard_options:
       all_nodes_electable: true
       mongod_options:
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index f9154ff202c..08e224e917a 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -211,9 +211,6 @@ def mongo_shell_program(  # pylint: disable=too-many-arguments,too-many-branches
     if config.FLOW_CONTROL is not None:
         mongod_set_parameters.setdefault("enableFlowControl", config.FLOW_CONTROL == "on")
 
-    # Set the default value for minimum resharding operation duration to 5 seconds.
-    mongod_set_parameters.setdefault("reshardingMinimumOperationDurationMillis", 5000)
-
     mongos_launcher = shardedcluster.MongosLauncher(fixturelib)
     # If the 'logComponentVerbosity' setParameter for mongos was not already specified, we set its
     # value to a default.
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index d563a313703..136569a5f02 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -253,8 +253,6 @@ class MongodLauncher(object):
         if "coordinateCommitReturnImmediatelyAfterPersistingDecision" not in suite_set_parameters:
             suite_set_parameters["coordinateCommitReturnImmediatelyAfterPersistingDecision"] = False
 
-        suite_set_parameters["reshardingMinimumOperationDurationMillis"] = 5000
-
         # There's a periodic background thread that checks for and aborts expired transactions.
         # "transactionLifetimeLimitSeconds" specifies for how long a transaction can run before expiring
         # and being aborted by the background thread. It defaults to 60 seconds, which is too short to
@@ -323,6 +321,9 @@ class MongodLauncher(object):
         if "configsvr" in mongod_options:
             shortcut_opts["nojournal"] = False
             mongod_options["journal"] = ""
+            suite_set_parameters.setdefault("reshardingMinimumOperationDurationMillis", 5000)
+            suite_set_parameters.setdefault("reshardingCriticalSectionTimeoutMillis",
+                                            24 * 60 * 60)  # 24 hours
 
         # Command line options override the YAML configuration.
         for opt_name in shortcut_opts:
diff --git a/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js b/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js
new file mode 100644
index 00000000000..28a0fb258a5
--- /dev/null
+++ b/jstests/concurrency/fsm_workloads/reshard_collection_crud_ops.js
@@ -0,0 +1,98 @@
+'use strict';
+
+/**
+ * Runs reshardCollection and CRUD operations concurrently.
+ *
+ * @tags: [requires_sharding]
+ */
+
+var $config = (function() {
+    const shardKeys = [
+        {a: 1},
+        {b: 1},
+    ];
+
+    const data = {
+        shardKey: shardKeys[0],
+        currentShardKeyIndex: 0,
+    };
+
+    const iterations = 25;
+    const kTotalWorkingDocuments = 1000;
+    const kMaxReshardingExecutions = TestData.runningWithShardStepdowns ? 4 : iterations;
+
+    /**
+     * @summary Takes in a number of documents to create, creates each document. With two properties
+     * being equal to the index and one counter property.
+     * @param {number} numDocs
+     * @returns {Array{Object}} an array of documents to be inserted into the collection.
+     */
+    function createDocuments(numDocs) {
+        const documents = Array.from({length: numDocs}).map((_, i) => ({a: i, b: i, c: 0}));
+        return documents;
+    }
+
+    function executeReshardCommand(db, collName, newShardKey) {
+        const coll = db.getCollection(collName);
+        print(`Started Resharding Collection ${coll.getFullName()}. New Shard Key ${
+            tojson(newShardKey)}`);
+        if (TestData.runningWithShardStepdowns) {
+            assert.commandWorkedOrFailedWithCode(
+                db.adminCommand({reshardCollection: coll.getFullName(), key: newShardKey}),
+                [ErrorCodes.SnapshotUnavailable]);
+        } else {
+            assert.commandWorked(
+                db.adminCommand({reshardCollection: coll.getFullName(), key: newShardKey}));
+        }
+        print(`Finished Resharding Collection ${coll.getFullName()}. New Shard Key ${
+            tojson(newShardKey)}`);
+    }
+
+    const states = {
+        insert: function insert(db, collName) {
+            const coll = db.getCollection(collName);
+            print(`Inserting documents for collection ${coll.getFullName()}.`);
+            const totalDocumentsToInsert = 10;
+            assert.commandWorked(coll.insert(createDocuments(totalDocumentsToInsert)));
+            print(`Finished Inserting documents.`);
+        },
+        reshardCollection: function reshardCollection(db, collName) {
+            //'reshardingMinimumOperationDurationMillis' is set to 30 seconds when there are
+            // stepdowns. So in order to limit the overall time for the test, we limit the number of
+            // resharding operations to kMaxReshardingExecutions.
+            const shouldContinueResharding = this.reshardingCount <= kMaxReshardingExecutions;
+            if (this.tid === 0 && shouldContinueResharding) {
+                const currentShardKeyIndex = this.currentShardKeyIndex;
+                const newIndex = (currentShardKeyIndex + 1) % shardKeys.length;
+                const shardKey = shardKeys[newIndex];
+
+                executeReshardCommand(db, collName, shardKey);
+                // If resharding fails with SnapshopUnavailable, then this will be incorrect. But
+                // its fine since reshardCollection will succeed if the new shard key matches the
+                // existing one.
+                this.currentShardKeyIndex = shardKeyIndex;
+                this.reshardingCount += 1;
+            }
+        }
+    };
+
+    const transitions = {
+        reshardCollection: {insert: 1},
+        insert: {insert: .5, reshardCollection: .5}
+    };
+
+    function setup(db, collName, _cluster) {
+        const coll = db.getCollection(collName);
+        assert.commandWorked(coll.insert(createDocuments(kTotalWorkingDocuments)));
+    }
+
+    return {
+        threadCount: 20,
+        iterations: iterations,
+        startState: 'reshardCollection',
+        states: states,
+        transitions: transitions,
+        setup: setup,
+        data: data
+    };
+})();