SERVER-31355 Synchronize stepdown in auto_retry_on_network_error.js

Changes the test to wait for all client connections to be closed before
attempting to run commands on the replica set connection object.

Also introduces a failpoint to cause the server to hang after it has
updated its member state to SECONDARY and before it has begun to close
all of its client connections.

1 files changed, 10 insertions, 1 deletions

diff --git a/jstests/noPassthrough/auto_retry_on_network_error.js b/jstests/noPassthrough/auto_retry_on_network_error.js
index 1d8e0c59601..b92a9780b6e 100644
--- a/jstests/noPassthrough/auto_retry_on_network_error.js
+++ b/jstests/noPassthrough/auto_retry_on_network_error.js
@@ -6,6 +6,7 @@
     "use strict";
 
     load("jstests/libs/override_methods/auto_retry_on_network_error.js");
+    load("jstests/replsets/rslib.js");
 
     function stepDownPrimary(rst) {
         // Since we expect the mongo shell's connection to get severed as a result of running the
@@ -19,12 +20,20 @@
             });
             assert(isNetworkError(error),
                    "replSetStepDown did not disconnect client; failed with " + tojson(error));
+
+            // We use the reconnect() function to run a command against the former primary that
+            // acquires the global lock to ensure that it has finished stepping down and has
+            // therefore closed all of its client connections. This ensures commands sent on other
+            // connections to the former primary trigger a network error rather than potentially
+            // returning a "not master" error while the server is in the midst of closing client
+            // connections.
+            reconnect(primary);
         } finally {
             TestData.skipRetryOnNetworkError = false;
         }
     }
 
-    const rst = new ReplSetTest({nodes: 2});
+    const rst = new ReplSetTest({nodes: 1});
     rst.startSet();
     rst.initiate();