SERVER-23877 Make ReplSetTest.awaitLastOpCommitted retry on network errors

1 files changed, 49 insertions, 35 deletions

diff --git a/jstests/libs/override_methods/sharding_continuous_config_stepdown.js b/jstests/libs/override_methods/sharding_continuous_config_stepdown.js
index e111886f1b9..aea3e482961 100644
--- a/jstests/libs/override_methods/sharding_continuous_config_stepdown.js
+++ b/jstests/libs/override_methods/sharding_continuous_config_stepdown.js
@@ -7,6 +7,31 @@
 load('jstests/libs/parallelTester.js');

 load("jstests/replsets/rslib.js");

 

+/**

+ * Executes the specified function and if it fails due to exception, which is related to network

+ * error retries the call once. If the second attempt also fails, simply throws the last

+ * exception.

+ *

+ * Returns the return value of the input call.

+ */

+function retryOnNetworkError(func) {

+    var networkErrorRetriesLeft = 1;

+

+    while (true) {

+        try {

+            return func();

+        } catch (e) {

+            if (e.toString().indexOf("network error") > -1 && networkErrorRetriesLeft > 0) {

+                print("Network error occurred and the call will be retried: " +

+                      tojson({error: e.toString(), stack: e.stack}));

+                networkErrorRetriesLeft--;

+            } else {

+                throw e;

+            }

+        }

+    }

+}

+

 (function() {

     'use strict';

 

@@ -24,58 +49,37 @@ load("jstests/replsets/rslib.js");
 

         /**

          * This function is intended to be called in a separate thread and it continuously steps

-         *down

-         * the current primary for a number of attempts.

+         * down the current primary for a number of attempts.

          *

          * @param {string} seedNode The connection string of a node from which to discover the

-         *primary

-         *                          of the replica set.

+         *      primary of the replica set.

          * @param {CountDownLatch} stopCounter Object, which can be used to stop the thread.

          *

          * @return Object with the following fields:

          *      ok {integer}: 0 if it failed, 1 if it succeeded.

          *      error {string}: Only present if ok == 0. Contains the cause for the error.

          *      stack {string}: Only present if ok == 0. Contains the stack at the time of the

-         *error.

+         *          error.

          */

         function _continuousPrimaryStepdownFn(seedNode, stopCounter) {

             'use strict';

 

+            load('jstests/libs/override_methods/sharding_continuous_config_stepdown.js');

+

             var stepdownDelaySeconds = 10;

 

             print('*** Continuous stepdown thread running with seed node ' + seedNode);

 

-            // The config primary may unexpectedly step down during startup if under heavy load and

-            // too slowly processing heartbeats. When it steps down, it closes all of its

-            // connections.

-            // This can happen during the call to new ReplSetTest, so in order to account for this

-            // and

-            // make the tests stable, retry discovery of the replica set's configuration once

-            // (SERVER-22794).

-            var replSet;

-            var networkErrorRetries = 1;

-            while (networkErrorRetries >= 0) {

-                try {

-                    replSet = new ReplSetTest(seedNode);

-                    break;

-                } catch (e) {

-                    if (((networkErrorRetries--) > 0) &&

-                        (e.toString().indexOf("network error") > -1)) {

-                        print("Error: " + e.toString() + "\nStacktrace: " + e.stack);

-                        print("Stepdown thread's config server connection was closed, retrying.");

-                    } else {

-                        print('*** Continuous stepdown thread failed to connect to the ' +

-                              'config server: ' + tojson(e));

-                        return {

-                            ok: 0,

-                            error: e.toString(),

-                            stack: e.stack

-                        };

-                    }

-                }

-            }

-

             try {

+                // The config primary may unexpectedly step down during startup if under heavy load

+                // and too slowly processing heartbeats. When it steps down, it closes all of its

+                // connections. This can happen during the call to new ReplSetTest, so in order to

+                // account for this and make the tests stable, retry discovery of the replica set's

+                // configuration once (SERVER-22794).

+                var replSet = retryOnNetworkError(function() {

+                    return new ReplSetTest(seedNode);

+                });

+

                 var primary = replSet.getPrimary();

 

                 while (stopCounter.getCount() > 0) {

@@ -120,6 +124,9 @@ load("jstests/replsets/rslib.js");
         var _originalStartSetFn = this.startSet;

         var _originalStopSetFn = this.stopSet;

 

+        // We override these methods to retry on network errors

+        var _originalAwaitLastOpCommitted = this.awaitLastOpCommitted;

+

         // These two manage the scoped failover thread

         var _scopedPrimaryStepdownThread;

         var _scopedPrimaryStepdownThreadStopCounter;

@@ -144,6 +151,13 @@ load("jstests/replsets/rslib.js");
         };

 

         /**

+         * Overrides the awaitLastOpCommitted to retry on network errors.

+         */

+        this.awaitLastOpCommitted = function() {

+            return retryOnNetworkError(_originalAwaitLastOpCommitted.bind(this));

+        };

+

+        /**

          * Spawns a thread to invoke continuousPrimaryStepdownFn. See its comments for more

          * information.

          */