SERVER-61593 Test command waitForMemberState should be interruptible

2 files changed, 41 insertions, 17 deletions

diff --git a/buildscripts/resmokelib/testing/hooks/initialsync.py b/buildscripts/resmokelib/testing/hooks/initialsync.py
index 171787fe900..2da12b17bfe 100644
--- a/buildscripts/resmokelib/testing/hooks/initialsync.py
+++ b/buildscripts/resmokelib/testing/hooks/initialsync.py
@@ -60,6 +60,8 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase):
     """BackgroundInitialSyncTestCase class."""
 
     JS_FILENAME = os.path.join("jstests", "hooks", "run_initial_sync_node_validation.js")
+    INTERRUPTED_DUE_TO_REPL_STATE_CHANGE = 11602
+    INTERRUPTED_DUE_TO_STORAGE_CHANGE = 355
 
     def __init__(  # pylint: disable=too-many-arguments
             self, logger, test_name, description, base_test_name, hook, shell_options=None):
@@ -83,7 +85,17 @@ class BackgroundInitialSyncTestCase(jsfile.DynamicJSTestCase):
                 [("replSetTest", 1), ("waitForMemberState", 2),
                  ("timeoutMillis",
                   fixture_interface.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60 * 1000)])
-            sync_node_conn.admin.command(cmd)
+            while True:
+                try:
+                    sync_node_conn.admin.command(cmd)
+                    break
+                except pymongo.errors.OperationFailure as err:
+                    if (err.code != self.INTERRUPTED_DUE_TO_REPL_STATE_CHANGE
+                            and err.code != self.INTERRUPTED_DUE_TO_STORAGE_CHANGE):
+                        raise
+                msg = ("Interrupted while waiting for node to reach secondary state, retrying: {}"
+                       ).format(err)
+                self.logger.error(msg)
 
         # Check if the initial sync node is in SECONDARY state. If it's been 'n' tests, then it
         # should have waited to be in SECONDARY state and the test should be marked as a failure.
diff --git a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
index 8e70da8a09f..b0ff858c5dc 100644
--- a/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
+++ b/buildscripts/resmokelib/testing/hooks/periodic_kill_secondaries.py
@@ -123,6 +123,9 @@ class PeriodicKillSecondaries(interface.Hook):
 class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
     """PeriodicKillSecondariesTestCase class."""
 
+    INTERRUPTED_DUE_TO_REPL_STATE_CHANGE = 11602
+    INTERRUPTED_DUE_TO_STORAGE_CHANGE = 355
+
     def __init__(  # pylint: disable=too-many-arguments
             self, logger, test_name, description, base_test_name, hook, test_report):
         """Initialize PeriodicKillSecondariesTestCase."""
@@ -434,19 +437,28 @@ class PeriodicKillSecondariesTestCase(interface.DynamicTestCase):
 
     def _await_secondary_state(self, secondary):
         client = secondary.mongo_client()
-        try:
-            client.admin.command(
-                bson.SON([
-                    ("replSetTest", 1),
-                    ("waitForMemberState", 2),  # 2 = SECONDARY
-                    ("timeoutMillis",
-                     fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60 * 1000)
-                ]))
-        except pymongo.errors.OperationFailure as err:
-            self.logger.exception(
-                "mongod on port %d failed to reach state SECONDARY after %d seconds",
-                secondary.port, fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60)
-            raise errors.ServerFailure(
-                "mongod on port {} failed to reach state SECONDARY after {} seconds: {}".format(
-                    secondary.port, fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60,
-                    err.args[0]))
+        while True:
+            try:
+                client.admin.command(
+                    bson.SON([
+                        ("replSetTest", 1),
+                        ("waitForMemberState", 2),  # 2 = SECONDARY
+                        ("timeoutMillis",
+                         fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60 * 1000)
+                    ]))
+                break
+            except pymongo.errors.OperationFailure as err:
+                if (err.code != self.INTERRUPTED_DUE_TO_REPL_STATE_CHANGE
+                        and err.code != self.INTERRUPTED_DUE_TO_STORAGE_CHANGE):
+                    self.logger.exception(
+                        "mongod on port %d failed to reach state SECONDARY after %d seconds",
+                        secondary.port, fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60)
+                    raise errors.ServerFailure(
+                        "mongod on port {} failed to reach state SECONDARY after {} seconds: {}".
+                        format(secondary.port,
+                               fixture.ReplFixture.AWAIT_REPL_TIMEOUT_FOREVER_MINS * 60,
+                               err.args[0]))
+
+                msg = ("Interrupted while waiting for node to reach secondary state, retrying: {}"
+                       ).format(err)
+                self.logger.error(msg)