SERVER-72860 Python exceptions in create_fixture_table() cause resmoke to incorrectly mark Evergreen tasks as setup failures

3 files changed, 69 insertions, 49 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 94fc0758b1c..c7c9ce53072 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -581,6 +581,10 @@ class _MongoSFixture(interface.Fixture):
 
     def get_node_info(self):
         """Return a list of NodeInfo objects."""
+        if self.mongos is None:
+            self.logger.warning("The mongos fixture has not been set up yet.")
+            return []
+
         info = interface.NodeInfo(full_name=self.logger.full_name, name=self.logger.name,
                                   port=self.port, pid=self.mongos.pid)
         return [info]
diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index bf2d33a024e..341448897e8 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -378,23 +378,35 @@ class FixtureTestCaseManager:
 
         Return True if the teardown was successful, False otherwise.
         """
-
-        # Refresh the fixture table before teardown to capture changes due to
-        # CleanEveryN and stepdown hooks.
-        self.report.logging_prefix = create_fixture_table(self.fixture)
-
-        if abort:
-            test_case = _fixture.FixtureAbortTestCase(self.test_queue_logger, self.fixture,
-                                                      "job{}".format(self.job_num),
-                                                      self.times_set_up)
-            self.times_set_up += 1
-        else:
-            test_case = _fixture.FixtureTeardownTestCase(self.test_queue_logger, self.fixture,
-                                                         "job{}".format(self.job_num))
-
-        test_case(self.report)
-        if self.report.find_test_info(test_case).status != "pass":
-            logger.error("The teardown of %s failed.", self.fixture)
-            return False
-
-        return True
+        try:
+            test_case = None
+
+            if abort:
+                test_case = _fixture.FixtureAbortTestCase(self.test_queue_logger, self.fixture,
+                                                          "job{}".format(self.job_num),
+                                                          self.times_set_up)
+                self.times_set_up += 1
+            else:
+                test_case = _fixture.FixtureTeardownTestCase(self.test_queue_logger, self.fixture,
+                                                             "job{}".format(self.job_num))
+
+            # Refresh the fixture table before teardown to capture changes due to
+            # CleanEveryN and stepdown hooks.
+            self.report.logging_prefix = create_fixture_table(self.fixture)
+            test_case(self.report)
+
+            if self.report.find_test_info(test_case).status != "pass":
+                logger.error("The teardown of %s failed.", self.fixture)
+                return False
+
+            return True
+        finally:
+            # This is a failsafe. In the event that 'teardown_fixture' fails,
+            # any rogue logger handlers will be removed from this fixture.
+            # If not cleaned up, these will trigger 'setup failures' --
+            # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE.
+            if not isinstance(test_case, _fixture.FixtureAbortTestCase):
+                for handler in self.fixture.logger.handlers:
+                    # We ignore the cancellation token returned by close_later() since we always
+                    # want the logs to eventually get flushed.
+                    self.fixture.fixturelib.close_loggers(handler)
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index 5ac5145cdee..176fc4bb241 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -140,35 +140,39 @@ class TestReport(unittest.TestResult):
     def stopTest(self, test):
         """Call after 'test' has run."""
 
-        # check if there are stacktrace files, if so, invoke the symbolizer here.
-        # If there are no stacktrace files for this job, we do not need to invoke the symbolizer at all.
-        # Take a lock to download the debug symbols if it hasn't already been downloaded.
-        # log symbolized output to test.logger.info()
-
-        symbolizer = ResmokeSymbolizer()
-        symbolizer.symbolize_test_logs(test)
-        # symbolization completed
-
-        unittest.TestResult.stopTest(self, test)
-
-        with self._lock:
-            test_info = self.find_test_info(test)
-            test_info.end_time = time.time()
-            test_status = "no failures detected" if test_info.status == "pass" else "failed"
-
-        time_taken = test_info.end_time - test_info.start_time
-        self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken,
-                             test_status)
-
-        # Asynchronously closes the buildlogger test handler to avoid having too many threads open
-        # on 32-bit systems.
-        for handler in test.logger.handlers:
-            # We ignore the cancellation token returned by close_later() since we always want the
-            # logs to eventually get flushed.
-            logging.flush.close_later(handler)
-
-        # Restore the original logger for the test.
-        test.reset_logger()
+        try:
+            # check if there are stacktrace files, if so, invoke the symbolizer here.
+            # If there are no stacktrace files for this job, we do not need to invoke the symbolizer at all.
+            # Take a lock to download the debug symbols if it hasn't already been downloaded.
+            # log symbolized output to test.logger.info()
+
+            symbolizer = ResmokeSymbolizer()
+            symbolizer.symbolize_test_logs(test)
+            # symbolization completed
+
+            unittest.TestResult.stopTest(self, test)
+
+            with self._lock:
+                test_info = self.find_test_info(test)
+                test_info.end_time = time.time()
+                test_status = "no failures detected" if test_info.status == "pass" else "failed"
+
+            time_taken = test_info.end_time - test_info.start_time
+            self.job_logger.info("%s ran in %0.2f seconds: %s.", test.basename(), time_taken,
+                                 test_status)
+
+        finally:
+            # This is a failsafe. In the event that 'stopTest' fails,
+            # any rogue logger handlers will be removed from this test.
+            # If not cleaned up, these will trigger 'setup failures' --
+            # indicated by exiting with LoggerRuntimeConfigError.EXIT_CODE.
+            for handler in test.logger.handlers:
+                # We ignore the cancellation token returned by close_later() since we always want the
+                # logs to eventually get flushed.
+                logging.flush.close_later(handler)
+
+            # Restore the original logger for the test.
+            test.reset_logger()
 
     def addError(self, test, err):
         """Call when a non-failureException was raised during the execution of 'test'."""