SERVER-27839 Allow for step downs during reconfig in ReplSetTest initiate

(cherry picked from commit a4e1443629b733c7c0fd44dddcd78e884da848bd)

8 files changed, 32 insertions, 23 deletions

diff --git a/jstests/replsets/priority_takeover_one_node_higher_priority.js b/jstests/replsets/priority_takeover_one_node_higher_priority.js
index 88ed5475294..1d9aafb08e7 100644
--- a/jstests/replsets/priority_takeover_one_node_higher_priority.js
+++ b/jstests/replsets/priority_takeover_one_node_higher_priority.js
@@ -33,9 +33,8 @@
         var result = primary.adminCommand({replSetStepDown: stepDownGuardMillis / 1000});
         print('replSetStepDown did not throw exception but returned: ' + tojson(result));
     });
-    assert.neq(-1,
-               tojson(stepDownException).indexOf('error doing query'),
-               'replSetStepDown did not disconnect client');
+    assert(isNetworkError(stepDownException),
+           'replSetStepDown did not disconnect client; failed with ' + tojson(stepDownException));
 
     // Step down primary and wait for node 1 to be promoted to primary.
     replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000);
diff --git a/jstests/replsets/reconfig_without_increased_queues.js b/jstests/replsets/reconfig_without_increased_queues.js
index 39ef7a7e60c..8097b59b72c 100644
--- a/jstests/replsets/reconfig_without_increased_queues.js
+++ b/jstests/replsets/reconfig_without_increased_queues.js
@@ -61,7 +61,7 @@
         try {
             assert.commandWorked(replTest.getPrimary().adminCommand({replSetReconfig: newConfig}));
         } catch (e) {
-            if (tojson(e).indexOf("error doing query: failed") < 0) {
+            if (!isNetworkError(e)) {
                 throw e;
             }
         }
diff --git a/jstests/replsets/request_primary_stepdown.js b/jstests/replsets/request_primary_stepdown.js
index 3e56946397b..b9997e03a23 100644
--- a/jstests/replsets/request_primary_stepdown.js
+++ b/jstests/replsets/request_primary_stepdown.js
@@ -29,9 +29,8 @@
         var result = primary.adminCommand({replSetStepDown: 70, secondaryCatchUpPeriodSecs: 60});
         print('replSetStepDown did not throw exception but returned: ' + tojson(result));
     });
-    assert.neq(-1,
-               tojson(stepDownException).indexOf('error doing query'),
-               'replSetStepDown did not disconnect client');
+    assert(isNetworkError(stepDownException),
+           'replSetStepDown did not disconnect client; failed with ' + tojson(stepDownException));
 
     // Wait for node 1 to be promoted to primary after node 0 stepped down.
     replSet.waitForState(replSet.nodes[1], ReplSetTest.State.PRIMARY, 60 * 1000);
diff --git a/jstests/replsets/rslib.js b/jstests/replsets/rslib.js
index 03ce30de2ab..d310f46cdbb 100644
--- a/jstests/replsets/rslib.js
+++ b/jstests/replsets/rslib.js
@@ -124,7 +124,7 @@ var getLastOpTime;
             assert.commandWorked(admin.runCommand(
                 {replSetReconfig: rs._updateConfigIfNotDurable(config), force: force}));
         } catch (e) {
-            if (tojson(e).indexOf("error doing query: failed") < 0) {
+            if (!isNetworkError(e)) {
                 throw e;
             }
         }
@@ -256,8 +256,7 @@ var getLastOpTime;
             // reInitiate can throw because it tries to run an ismaster command on
             // all secondaries, including the new one that may have already aborted
             const errMsg = tojson(e);
-            if (errMsg.indexOf("error doing query: failed") > -1 ||
-                errMsg.indexOf("socket exception") > -1) {
+            if (isNetworkError(e)) {
                 // Ignore these exceptions, which are indicative of an aborted node
             } else {
                 throw e;
diff --git a/jstests/replsets/stepdown.js b/jstests/replsets/stepdown.js
index 9d7d1f14c11..7979e16e885 100644
--- a/jstests/replsets/stepdown.js
+++ b/jstests/replsets/stepdown.js
@@ -8,11 +8,6 @@
 
 load("jstests/replsets/rslib.js");
 
-// utility to check if an error was due to connection failure.
-var errorWasDueToConnectionFailure = function(error) {
-    return error.message.indexOf("error doing query: failed") >= 0;
-};
-
 var replTest = new ReplSetTest({
     name: 'testSet',
     nodes: {"n0": {rsConfig: {priority: 2}}, "n1": {}, "n2": {rsConfig: {votes: 1, priority: 0}}},
@@ -91,7 +86,7 @@ try {
 } catch (e) {
     // ignore errors due to connection failures as we expect the master to close connections
     // on stepdown
-    if (!errorWasDueToConnectionFailure(e)) {
+    if (!isNetworkError(e)) {
         throw e;
     }
 }
@@ -150,7 +145,7 @@ var currentMaster = replTest.getPrimary();
 try {
     printjson(currentMaster.getDB("admin").runCommand({shutdown: 1, force: true}));
 } catch (e) {
-    if (!errorWasDueToConnectionFailure(e)) {
+    if (!isNetworkError(e)) {
         throw e;
     }
 }
diff --git a/src/mongo/shell/db.js b/src/mongo/shell/db.js
index 839c605e2cb..7134bea4cc6 100644
--- a/src/mongo/shell/db.js
+++ b/src/mongo/shell/db.js
@@ -331,7 +331,7 @@ var DB;
         } catch (e) {
             // we expect the command to not return a response, as the server will shut down
             // immediately.
-            if (e.message.indexOf("error doing query: failed") >= 0) {
+            if (isNetworkError(e)) {
                 print('server should be down...');
                 return;
             }
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 29cee6aae9b..cab0bde19ed 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -725,9 +725,20 @@ var ReplSetTest = function(opts) {
             // if a heartbeat times out during the quorum check. We retry three times to reduce
             // the chance of failing this way.
             assert.retry(() => {
-                const res = master.runCommand(cmd);
-                if (res.ok === 1) {
-                    return true;
+                var res;
+                try {
+                    res = master.runCommand(cmd);
+                    if (res.ok === 1) {
+                        return true;
+                    }
+                } catch (e) {
+                    // reconfig can lead to a stepdown if the primary looks for a majority before
+                    // a majority of nodes have successfully joined the set. If there is a stepdown
+                    // then the reconfig request will be killed and respond with a network error.
+                    if (isNetworkError(e)) {
+                        return true;
+                    }
+                    throw e;
                 }
 
                 assert.commandFailedWithCode(
@@ -816,7 +827,7 @@ var ReplSetTest = function(opts) {
         try {
             assert.commandWorked(this.getPrimary().adminCommand({replSetReconfig: config}));
         } catch (e) {
-            if (tojson(e).indexOf("error doing query: failed") < 0) {
+            if (!isNetworkError(e)) {
                 throw e;
             }
         }
diff --git a/src/mongo/shell/utils.js b/src/mongo/shell/utils.js
index 215d40de685..f9e7bb29451 100644
--- a/src/mongo/shell/utils.js
+++ b/src/mongo/shell/utils.js
@@ -37,6 +37,12 @@ function _getErrorWithCode(codeOrObj, message) {
     return e;
 }
 
+// Checks if a javascript exception is a network error.
+function isNetworkError(error) {
+    return error.message.indexOf("error doing query") >= 0 ||
+        error.message.indexOf("socket exception") >= 0;
+}
+
 // Please consider using bsonWoCompare instead of this as much as possible.
 friendlyEqual = function(a, b) {
     if (a == b)
@@ -1180,7 +1186,7 @@ rs._runCmd = function(c) {
     try {
         res = db.adminCommand(c);
     } catch (e) {
-        if (("" + e).indexOf("error doing query") >= 0) {
+        if (isNetworkError(e)) {
             // closed connection.  reconnect.
             db.getLastErrorObj();
             var o = db.getLastErrorObj();