SERVER-31198 Run the concurrency suite with shard stepdowns

3 files changed, 71 insertions, 4 deletions

diff --git a/jstests/libs/override_methods/auto_retry_on_network_error.js b/jstests/libs/override_methods/auto_retry_on_network_error.js
index c8e702fd073..e462e8a47fb 100644
--- a/jstests/libs/override_methods/auto_retry_on_network_error.js
+++ b/jstests/libs/override_methods/auto_retry_on_network_error.js
@@ -213,7 +213,7 @@
             try {
                 let res = clientFunction.apply(mongo, clientFunctionArguments);
 
-                if (isRetryableWriteCmd && canRetryWrites) {
+                if (isRetryableWriteCmd) {
                     // findAndModify can fail during the find stage and return an executor error.
                     if ((cmdName === "findandmodify" || cmdName === "findAndModify") &&
                         isRetryableExecutorCodeAndMessage(res.code, res.errmsg)) {
@@ -265,18 +265,39 @@
                         }
 
                         if (isRetryableExecutorCodeAndMessage(res.code, res.errmsg)) {
-                            // Don't decrement retries for the same reason as above.
                             print("=-=-=-= Retrying because of executor interruption: " + cmdName +
                                   ", retries remaining: " + numRetries);
                             continue;
                         }
+
+                        // listCollections and listIndexes called through mongos may return
+                        // OperationFailed if the request to establish a cursor on the targeted
+                        // shard fails with a network error.
+                        //
+                        // TODO SERVER-30949: Remove this check once those two commands retry on
+                        // retryable errors automatically.
+                        if ((cmdName === "listCollections" || cmdName === "listIndexes") &&
+                            res.code === ErrorCodes.OperationFailed &&
+                            res.hasOwnProperty("errmsg") &&
+                            res.errmsg.indexOf("failed to read command response from shard") >= 0) {
+                            print("=-=-=-= Retrying failed mongos cursor command: " + cmdName +
+                                  ", retries remaining: " + numRetries);
+                            continue;
+                        }
+
+                        // Thrown when an index build is interrupted during its collection scan.
+                        if ((cmdName === "createIndexes" && res.code === 28550)) {
+                            print("=-=-=-= Retrying because of interrupted collection scan: " +
+                                  cmdName + ", retries remaining: " + numRetries);
+                            continue;
+                        }
                     }
 
                     // Swallow safe errors that may come from a retry since the command may have
                     // completed before the connection was closed.
                     if (isAcceptableRetryFailedResponse(cmdName, res)) {
                         print("=-=-=-= Overriding safe failed response for: " + cmdName +
-                              ", retries remaining: " + numRetries);
+                              ", code: " + res.code + ", retries remaining: " + numRetries);
                         res.ok = 1;
                     }
                 }
diff --git a/jstests/libs/override_methods/continuous_stepdown.js b/jstests/libs/override_methods/continuous_stepdown.js
index de9bab2adc6..c2c14753420 100644
--- a/jstests/libs/override_methods/continuous_stepdown.js
+++ b/jstests/libs/override_methods/continuous_stepdown.js
@@ -320,8 +320,13 @@ let ContinuousStepdown;
              * specified by the stepdownOptions object.
              *
              * If waitForPrimary is true, blocks until each replica set has elected a primary.
+             * If waitForMongosRetarget is true, blocks until each mongos has an up to date view of
+             * the cluster.
              */
-            this.stopContinuousFailover = function({waitForPrimary: waitForPrimary = false} = {}) {
+            this.stopContinuousFailover = function({
+                waitForPrimary: waitForPrimary = false,
+                waitForMongosRetarget: waitForMongosRetarget = false
+            } = {}) {
                 if (stepdownOptions.configStepdown) {
                     this.configRS.stopContinuousFailover({waitForPrimary: waitForPrimary});
                 }
@@ -331,6 +336,46 @@ let ContinuousStepdown;
                         rst.test.stopContinuousFailover({waitForPrimary: waitForPrimary});
                     });
                 }
+
+                if (waitForMongosRetarget) {
+                    // Run validate on each collection in each database to ensure mongos can target
+                    // the primary for each shard with data, including the config servers.
+                    this._mongos.forEach(s => {
+                        const res = assert.commandWorked(s.adminCommand({listDatabases: 1}));
+                        res.databases.forEach(dbInfo => {
+                            const startTime = Date.now();
+                            print("Waiting for mongos: " + s.host + " to retarget db: " +
+                                  dbInfo.name);
+
+                            const db = s.getDB(dbInfo.name);
+                            assert.soon(() => {
+                                let collInfo;
+                                try {
+                                    collInfo = db.getCollectionInfos();
+                                } catch (e) {
+                                    if (ErrorCodes.isNotMasterError(e.code)) {
+                                        return false;
+                                    }
+                                    throw e;
+                                }
+
+                                collInfo.forEach(collDoc => {
+                                    const res = db.runCommand({collStats: collDoc["name"]});
+                                    if (ErrorCodes.isNotMasterError(res.code)) {
+                                        return false;
+                                    }
+                                    assert.commandWorked(res);
+                                });
+
+                                return true;
+                            });
+                            const totalTime = Date.now() - startTime;
+                            print("Finished waiting for mongos: " + s.host + " to retarget db: " +
+                                  dbInfo.name + ", in " + totalTime + " ms");
+                        });
+                    });
+                }
+
             };
 
             /**
diff --git a/jstests/libs/override_methods/implicitly_retry_on_database_drop_pending.js b/jstests/libs/override_methods/implicitly_retry_on_database_drop_pending.js
index 83a7e0739ba..8510a0a361e 100644
--- a/jstests/libs/override_methods/implicitly_retry_on_database_drop_pending.js
+++ b/jstests/libs/override_methods/implicitly_retry_on_database_drop_pending.js
@@ -35,6 +35,7 @@
             return func.apply(conn, funcArgs);
         }
 
+        commandObj = Object.assign({}, commandObj);
         const commandName = Object.keys(commandObj)[0];
         let resPrevious;
         let res;