SERVER-40592 Handle exceptions from {setup,teardown}_fixture().

2 files changed, 98 insertions, 5 deletions

diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index 080c498a990..6fb3bd38246 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -84,9 +84,25 @@ class Job(object):  # pylint: disable=too-many-instance-attributes
         will be run before this method returns. If an error occurs
         while destroying the fixture, then the 'teardown_flag' will be set.
         """
-        if setup_flag is not None and not self.setup_fixture():
-            self._interrupt_all_jobs(queue, interrupt_flag)
-            return
+        if setup_flag is not None:
+            try:
+                setup_succeeded = self.setup_fixture()
+            except errors.StopExecution as err:
+                # Something went wrong when setting up the fixture. Perhaps we couldn't get a
+                # test_id from logkeeper for where to put the log output. We don't attempt to run
+                # any tests.
+                self.logger.error(
+                    "Received a StopExecution exception when setting up the fixture: %s.", err)
+                setup_succeeded = False
+            except:  # pylint: disable=bare-except
+                # Something unexpected happened when setting up the fixture. We don't attempt to run
+                # any tests.
+                self.logger.exception("Encountered an error when setting up the fixture.")
+                setup_succeeded = False
+
+            if not setup_succeeded:
+                self._interrupt_all_jobs(queue, interrupt_flag)
+                return
 
         try:
             self._run(queue, interrupt_flag)
@@ -99,8 +115,26 @@ class Job(object):  # pylint: disable=too-many-instance-attributes
             self.logger.exception("Encountered an error during test execution.")
             self._interrupt_all_jobs(queue, interrupt_flag)
 
-        if teardown_flag is not None and not self.teardown_fixture():
-            teardown_flag.set()
+        if teardown_flag is not None:
+            try:
+                teardown_succeeded = self.teardown_fixture()
+            except errors.StopExecution as err:
+                # Something went wrong when tearing down the fixture. Perhaps we couldn't get a
+                # test_id from logkeeper for where to put the log output. We indicate back to the
+                # executor thread that teardown has failed. This likely means resmoke.py is exiting
+                # without having terminated all of the child processes it spawned.
+                self.logger.error(
+                    "Received a StopExecution exception when tearing down the fixture: %s.", err)
+                teardown_succeeded = False
+            except:  # pylint: disable=bare-except
+                # Something unexpected happened when tearing down the fixture. We indicate back to
+                # the executor thread that teardown has failed. This may mean resmoke.py is exiting
+                # without having terminated all of the child processes it spawned.
+                self.logger.exception("Encountered an error when tearing down the fixture.")
+                teardown_succeeded = False
+
+            if not teardown_succeeded:
+                teardown_flag.set()
 
     @staticmethod
     def _get_time():
diff --git a/buildscripts/tests/resmokelib/testing/test_job.py b/buildscripts/tests/resmokelib/testing/test_job.py
index 5ed9064860e..ca204548214 100644
--- a/buildscripts/tests/resmokelib/testing/test_job.py
+++ b/buildscripts/tests/resmokelib/testing/test_job.py
@@ -1,10 +1,12 @@
 """Unit tests for the resmokelib.testing.executor module."""
 
 import logging
+import threading
 import unittest
 
 import mock
 
+from buildscripts.resmokelib import errors
 from buildscripts.resmokelib.testing import job
 from buildscripts.resmokelib.testing import queue_element
 from buildscripts.resmokelib.utils import queue as _queue
@@ -205,3 +207,60 @@ class UnitJob(job.Job):  # pylint: disable=too-many-instance-attributes
         if test.test_name not in self.tests:
             self.tests[test.test_name] = 0
         self.tests[test.test_name] += 1
+
+
+class TestFixtureSetupAndTeardown(unittest.TestCase):
+    """Test cases for error handling around setup_fixture() and teardown_fixture()."""
+
+    def setUp(self):
+        logger = logging.getLogger("job_unittest")
+        self.__job_object = job.Job(job_num=0, logger=logger, fixture=None, hooks=[], report=None,
+                                    archival=None, suite_options=None, test_queue_logger=logger)
+
+        # Initialize the Job instance such that its setup_fixture() and teardown_fixture() methods
+        # always indicate success. The settings for these mocked method will be changed in the
+        # individual test cases below.
+        self.__job_object.setup_fixture = mock.Mock(return_value=True)
+        self.__job_object.teardown_fixture = mock.Mock(return_value=True)
+
+    def __assert_when_run_tests(self, setup_succeeded=True, teardown_succeeded=True):
+        queue = _queue.Queue()
+        interrupt_flag = threading.Event()
+        setup_flag = threading.Event()
+        teardown_flag = threading.Event()
+
+        self.__job_object(queue, interrupt_flag, setup_flag, teardown_flag)
+
+        self.assertEqual(setup_succeeded, not interrupt_flag.is_set())
+        self.assertEqual(teardown_succeeded, not teardown_flag.is_set())
+
+    def test_setup_and_teardown_both_succeed(self):
+        self.__assert_when_run_tests()
+
+    def test_setup_returns_failure(self):
+        self.__job_object.setup_fixture.return_value = False
+        self.__assert_when_run_tests(setup_succeeded=False)
+
+    def test_setup_raises_logging_config_exception(self):
+        self.__job_object.setup_fixture.side_effect = errors.LoggerRuntimeConfigError(
+            "Logging configuration error intentionally raised in unit test")
+        self.__assert_when_run_tests(setup_succeeded=False)
+
+    def test_setup_raises_unexpected_exception(self):
+        self.__job_object.setup_fixture.side_effect = Exception(
+            "Generic error intentionally raised in unit test")
+        self.__assert_when_run_tests(setup_succeeded=False)
+
+    def test_teardown_returns_failure(self):
+        self.__job_object.teardown_fixture.return_value = False
+        self.__assert_when_run_tests(teardown_succeeded=False)
+
+    def test_teardown_raises_logging_config_exception(self):
+        self.__job_object.teardown_fixture.side_effect = errors.LoggerRuntimeConfigError(
+            "Logging configuration error intentionally raised in unit test")
+        self.__assert_when_run_tests(teardown_succeeded=False)
+
+    def test_teardown_raises_unexpected_exception(self):
+        self.__job_object.teardown_fixture.side_effect = Exception(
+            "Generic error intentionally raised in unit test")
+        self.__assert_when_run_tests(teardown_succeeded=False)