SERVER-30676 Add support for stepdown options to the concurrency framework

2 files changed, 64 insertions, 12 deletions

diff --git a/jstests/concurrency/fsm_libs/cluster.js b/jstests/concurrency/fsm_libs/cluster.js
index 110b3ce45e5..931e75a0af1 100644
--- a/jstests/concurrency/fsm_libs/cluster.js
+++ b/jstests/concurrency/fsm_libs/cluster.js
@@ -44,6 +44,9 @@ var Cluster = function(options) {
             'sharded.enableBalancer',
             'sharded.numMongos',
             'sharded.numShards',
+            'sharded.stepdownOptions',
+            'sharded.stepdownOptions.configStepdown',
+            'sharded.stepdownOptions.shardStepdown',
             'teardownFunctions'
         ];
 
@@ -219,6 +222,11 @@ var Cluster = function(options) {
                 shardConfig.rsOptions = {};
             }
 
+            if (this.shouldPerformContinuousStepdowns()) {
+                load('jstests/libs/override_methods/continuous_stepdown.js');
+                ContinuousStepdown.configure(options.sharded.stepdownOptions);
+            }
+
             st = new ShardingTest(shardConfig);
 
             conn = st.s;  // mongos
@@ -230,6 +238,16 @@ var Cluster = function(options) {
                 st.stop();
             };
 
+            if (this.shouldPerformContinuousStepdowns()) {
+                this.startContinuousFailover = function() {
+                    st.startContinuousFailover();
+                };
+
+                this.stopContinuousFailover = function() {
+                    st.stopContinuousFailover({waitForPrimary: true});
+                };
+            }
+
             // Save all mongos and mongod connections
             var i = 0;
             var mongos = st.s0;
@@ -340,7 +358,7 @@ var Cluster = function(options) {
             throw new Error('mongos function must be a function that takes a db as an argument');
         }
         _conns.mongos.forEach(function(mongosConn) {
-            fn(mongosConn.getDB('admin'));
+            fn(mongosConn.getDB('admin'), true /* isMongos */);
         });
     };
 
@@ -474,11 +492,24 @@ var Cluster = function(options) {
     this.validateAllCollections = function validateAllCollections(phase) {
         assert(initialized, 'cluster must be initialized first');
 
-        var _validateCollections = function _validateCollections(db) {
+        const isSteppingDownConfigServers = this.isSteppingDownConfigServers();
+        var _validateCollections = function _validateCollections(db, isMongos = false) {
             // Validate all the collections on each node.
             var res = db.adminCommand({listDatabases: 1});
             assert.commandWorked(res);
             res.databases.forEach(dbInfo => {
+                // Don't perform listCollections on the admin or config database through a mongos
+                // connection when stepping down the config server primary, because both are stored
+                // on the config server, and listCollections may return a not master error if the
+                // mongos is stale.
+                //
+                // TODO SERVER-30949: listCollections through mongos should automatically retry on
+                // NotMaster errors. Once that is true, remove this check.
+                if (isSteppingDownConfigServers && isMongos &&
+                    (dbInfo.name === "admin" || dbInfo.name === "config")) {
+                    return;
+                }
+
                 if (!validateCollections(db.getSiblingDB(dbInfo.name), {full: true})) {
                     throw new Error(phase + ' collection validation failed');
                 }
@@ -592,6 +623,15 @@ var Cluster = function(options) {
 
         return wiredTigerConfigString === 'type=lsm';
     };
+
+    this.shouldPerformContinuousStepdowns = function shouldPerformContinuousStepdowns() {
+        return this.isSharded() && (typeof options.sharded.stepdownOptions !== 'undefined');
+    };
+
+    this.isSteppingDownConfigServers = function isSteppingDownConfigServers() {
+        return this.shouldPerformContinuousStepdowns() &&
+            options.sharded.stepdownOptions.configStepdown;
+    };
 };
 
 /**
diff --git a/jstests/concurrency/fsm_libs/runner.js b/jstests/concurrency/fsm_libs/runner.js
index f620a0b46d9..117d1754419 100644
--- a/jstests/concurrency/fsm_libs/runner.js
+++ b/jstests/concurrency/fsm_libs/runner.js
@@ -537,18 +537,30 @@ var runner = (function() {
                 cleanup.push(workload);
             });
 
+            if (cluster.shouldPerformContinuousStepdowns()) {
+                cluster.startContinuousFailover();
+            }
+
             try {
-                // Start this set of foreground workload threads.
-                threadMgr.spawnAll(cluster, executionOptions);
-                // Allow 20% of foreground threads to fail. This allows the workloads to run on
-                // underpowered test hosts.
-                threadMgr.checkFailed(0.2);
+                try {
+                    // Start this set of foreground workload threads.
+                    threadMgr.spawnAll(cluster, executionOptions);
+                    // Allow 20% of foreground threads to fail. This allows the workloads to run on
+                    // underpowered test hosts.
+                    threadMgr.checkFailed(0.2);
+                } finally {
+                    // Threads must be joined before destruction, so do this
+                    // even in the presence of exceptions.
+                    errors.push(...threadMgr.joinAll().map(
+                        e => new WorkloadFailure(
+                            e.err, e.stack, e.tid, 'Foreground ' + e.workloads.join(' '))));
+                }
             } finally {
-                // Threads must be joined before destruction, so do this
-                // even in the presence of exceptions.
-                errors.push(...threadMgr.joinAll().map(
-                    e => new WorkloadFailure(
-                        e.err, e.stack, e.tid, 'Foreground ' + e.workloads.join(' '))));
+                if (cluster.shouldPerformContinuousStepdowns()) {
+                    // Suspend the stepdown threads prior to calling cleanupWorkload() to avoid
+                    // causing a failover to happen while the data consistency checks are running.
+                    cluster.stopContinuousFailover();
+                }
             }
         } finally {
             // Call each foreground workload's teardown function. After all teardowns have completed