SERVER-35437 Wait for secondary state after stepdown command in multi_rs.js

5 files changed, 42 insertions, 2 deletions

diff --git a/jstests/multiVersion/3_upgrade_replset.js b/jstests/multiVersion/3_upgrade_replset.js
index bccab5068c7..ea8826c1ad7 100644
--- a/jstests/multiVersion/3_upgrade_replset.js
+++ b/jstests/multiVersion/3_upgrade_replset.js
@@ -58,6 +58,11 @@ rst.upgradeSet({binVersion: "latest"});
 
 jsTest.log("Replica set upgraded.");
 
+// We save a reference to the old primary so that we can call reconnect() on it before
+// joinFindInsert() would attempt to send the node an update operation that signals the parallel
+// shell running the background operations to stop.
+var oldPrimary = primary;
+
 // Wait for primary
 var primary = rst.getPrimary();
 
@@ -73,8 +78,15 @@ jsTest.log("Replica set downgraded.");
 // Allow even more valid writes to go through
 sleep(10 * 1000);
 
+// Since the primary from before the upgrade took place was restarted as part of the
+// upgrade/downgrade process, we explicitly reconnect to it so that sending it an update operation
+// silently fails with an unchecked NotMaster error rather than a network error.
+reconnect(oldPrimary.getDB("admin"));
 joinFindInsert();
 
+// Since the primary from after the upgrade took place was restarted as part of the downgrade
+// process, we explicitly reconnect to it.
+reconnect(primary.getDB("admin"));
 var totalInserts = primary.getCollection(insertNS).find().sort({_id: -1}).next()._id + 1;
 var dataFound = primary.getCollection(insertNS).count();
 
diff --git a/jstests/multiVersion/downgrade_replset.js b/jstests/multiVersion/downgrade_replset.js
index a2f5a5346ba..fb37654f78b 100644
--- a/jstests/multiVersion/downgrade_replset.js
+++ b/jstests/multiVersion/downgrade_replset.js
@@ -58,8 +58,16 @@ jsTest.log("Downgrading replica set..");
 rst.upgradeSet({binVersion: oldVersion, storageEngine: storageEngine});
 jsTest.log("Downgrade complete.");
 
+// We save a reference to the old primary so that we can call reconnect() on it before
+// joinFindInsert() would attempt to send the node an update operation that signals the parallel
+// shell running the background operations to stop.
+var oldPrimary = primary;
 primary = rst.getPrimary();
 printjson(rst.status());
 
+// Since the primary from before the upgrade took place was restarted as part of the
+// upgrade/downgrade process, we explicitly reconnect to it so that sending it an update operation
+// silently fails with an unchecked NotMaster error rather than a network error.
+reconnect(oldPrimary.getDB("admin"));
 joinFindInsert();
 rst.stopSet();
diff --git a/jstests/multiVersion/libs/multi_rs.js b/jstests/multiVersion/libs/multi_rs.js
index 87d5995ef48..d9b55a16229 100644
--- a/jstests/multiVersion/libs/multi_rs.js
+++ b/jstests/multiVersion/libs/multi_rs.js
@@ -25,7 +25,6 @@ ReplSetTest.prototype.upgradeSet = function(options, user, pwd) {
         var node = nodesToUpgrade[i];
         if (node == primary) {
             node = this.stepdown(node);
-            this.waitForState(node, ReplSetTest.State.SECONDARY);
             primary = this.getPrimary();
         }
 
@@ -84,7 +83,10 @@ ReplSetTest.prototype.stepdown = function(nodeId) {
         print('Caught exception after stepDown cmd: ' + tojson(ex));
     }
 
-    return this.reconnect(node);
+    // Wait for the state to be secondary to avoid the race of closing new connections on stepdown.
+    this.waitForState(node, ReplSetTest.State.SECONDARY);
+
+    return node;
 };
 
 ReplSetTest.prototype.reconnect = function(node) {
diff --git a/jstests/multiVersion/minor_version_downgrade_replset.js b/jstests/multiVersion/minor_version_downgrade_replset.js
index 3dae0c0a23f..769bfc22979 100644
--- a/jstests/multiVersion/minor_version_downgrade_replset.js
+++ b/jstests/multiVersion/minor_version_downgrade_replset.js
@@ -58,8 +58,17 @@ jsTest.log("Downgrading replica set..");
 rst.upgradeSet({binVersion: oldVersion, storageEngine: storageEngine});
 jsTest.log("Downgrade complete.");
 
+// We save a reference to the old primary so that we can call reconnect() on it before
+// joinFindInsert() would attempt to send the node an update operation that signals the parallel
+// shell running the background operations to stop.
+var oldPrimary = primary;
+
 primary = rst.getPrimary();
 printjson(rst.status());
 
+// Since the old primary was restarted as part of the downgrade process, we explicitly reconnect
+// to it so that sending it an update operation silently fails with an unchecked NotMaster error
+// rather than a network error.
+reconnect(oldPrimary.getDB("admin"));
 joinFindInsert();
 rst.stopSet();
diff --git a/jstests/multiVersion/minor_version_upgrade_replset.js b/jstests/multiVersion/minor_version_upgrade_replset.js
index 7f784f5c100..bcbdf6e8f3f 100644
--- a/jstests/multiVersion/minor_version_upgrade_replset.js
+++ b/jstests/multiVersion/minor_version_upgrade_replset.js
@@ -59,6 +59,11 @@ rst.upgradeSet({binVersion: "latest"});
 
 jsTest.log("Replica set upgraded.");
 
+// We save a reference to the old primary so that we can call reconnect() on it before
+// joinFindInsert() would attempt to send the node an update operation that signals the parallel
+// shell running the background operations to stop.
+var oldPrimary = primary;
+
 // Wait for primary
 var primary = rst.getPrimary();
 
@@ -67,6 +72,10 @@ printjson(rst.status());
 // Allow more valid writes to go through
 sleep(10 * 1000);
 
+// Since the old primary was restarted as part of the upgrade process, we explicitly reconnect to it
+// so that sending it an update operation silently fails with an unchecked NotMaster error rather
+// than a network error.
+reconnect(oldPrimary.getDB("admin"));
 joinFindInsert();
 
 var totalInserts = primary.getCollection(insertNS).find().sort({_id: -1}).next()._id + 1;