SERVER-46842 resmoke.py shouldn't run data consistency checks in stepdown suites if a process has crashed

(cherry picked from commit 40801001754b6bdc15bd2f59eae523c59b6ff055)

2 files changed, 49 insertions, 1 deletions

diff --git a/buildscripts/resmokelib/testing/hooks/stepdown.py b/buildscripts/resmokelib/testing/hooks/stepdown.py
index 4cfd09fd52d..f61d070037e 100644
--- a/buildscripts/resmokelib/testing/hooks/stepdown.py
+++ b/buildscripts/resmokelib/testing/hooks/stepdown.py
@@ -439,6 +439,18 @@ class _StepdownThread(threading.Thread):  # pylint: disable=too-many-instance-at
         # Wait for Mongos to retarget the primary for each shard and the config server.
         self._do_wait_for_mongos_retarget()
 
+        # Check that fixtures are still running
+        for rs_fixture in self._rs_fixtures:
+            if not rs_fixture.is_running():
+                raise errors.ServerFailure(
+                    "ReplicaSetFixture with pids {} expected to be running in"
+                    " ContinuousStepdown, but wasn't.".format(rs_fixture.pids()))
+        for mongos_fixture in self._mongos_fixtures:
+            if not mongos_fixture.is_running():
+                raise errors.ServerFailure("MongoSFixture with pids {} expected to be running in"
+                                           " ContinuousStepdown, but wasn't.".format(
+                                               mongos_fixture.pids()))
+
     def resume(self):
         """Resume the thread."""
         self.__lifecycle.mark_test_started()
diff --git a/buildscripts/tests/resmokelib/testing/hooks/test_stepdown.py b/buildscripts/tests/resmokelib/testing/hooks/test_stepdown.py
index 4c4a9d05be7..c700ab60bf7 100644
--- a/buildscripts/tests/resmokelib/testing/hooks/test_stepdown.py
+++ b/buildscripts/tests/resmokelib/testing/hooks/test_stepdown.py
@@ -1,13 +1,15 @@
 """Unit tests for buildscripts/resmokelib/testing/hooks/stepdown.py."""
 
+import logging
 import os
 import unittest
 
 import mock
 
+from buildscripts.resmokelib import errors
 from buildscripts.resmokelib.testing.hooks import stepdown as _stepdown
 
-# pylint: disable=missing-docstring
+# pylint: disable=missing-docstring,protected-access
 
 
 def _get_threading_lock(test_case, MockCondition):  # pylint: disable=invalid-name
@@ -19,6 +21,40 @@ def _get_threading_lock(test_case, MockCondition):  # pylint: disable=invalid-na
     return lock
 
 
+class TestStepdownThread(unittest.TestCase):
+    @mock.patch("buildscripts.resmokelib.testing.fixtures.replicaset.ReplicaSetFixture")
+    @mock.patch("buildscripts.resmokelib.testing.fixtures.shardedcluster.ShardedClusterFixture")
+    def test_pause_throws_error(self, shardcluster_fixture, rs_fixture):
+        stepdown_thread = _stepdown._StepdownThread(
+            logger=logging.getLogger("hook_logger"),
+            mongos_fixtures=[shardcluster_fixture.mongos],
+            rs_fixtures=[rs_fixture],
+            stepdown_interval_secs=8,
+            terminate=False,
+            kill=False,
+            stepdown_lifecycle=_stepdown.FlagBasedStepdownLifecycle(),
+            wait_for_mongos_retarget=False,
+            stepdown_via_heartbeats=True,
+            background_reconfig=False,
+        )
+
+        # doesn't throw error when fixtures are running
+        stepdown_thread.pause()
+
+        # throws error when replica set fixture is not running
+        rs_fixture.is_running.return_value = False
+        try:
+            with self.assertRaises(errors.ServerFailure):
+                stepdown_thread.pause()
+        finally:
+            rs_fixture.is_running.return_value = True
+
+        # throws error when MongoS fixture is not running
+        shardcluster_fixture.mongos.is_running.return_value = False
+        with self.assertRaises(errors.ServerFailure):
+            stepdown_thread.pause()
+
+
 class TestFlagBasedStepdownLifecycle(unittest.TestCase):
     def test_becomes_idle_after_test_finishes(self):
         lifecycle = _stepdown.FlagBasedStepdownLifecycle()