SERVER-55249 archive all data on required mainline builders

(cherry picked from commit 39fc7fd24d6098ccd1a8551db75108171ed6370a)

3 files changed, 40 insertions, 47 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 63e951a6d08..3fd220e2006 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -456,6 +456,10 @@ BENCHMARK_REPETITIONS = None
 # S3 Bucket to upload archive files.
 ARCHIVE_BUCKET = "mongodatafiles"
 
+# Force archive all files where appropriate. Eventually we want this to be the default option.
+# For now, only the mainline required builders have this option enabled.
+FORCE_ARCHIVE_ALL_DATA_FILES = False
+
 # Benchmark options set internally by resmoke.py
 BENCHMARK_OUT_FORMAT = "json"
 
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index 40f1719790f..171b9ef5c94 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -202,6 +202,20 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
     # Archival options. Archival is enabled only when running on evergreen.
     if not _config.EVERGREEN_TASK_ID:
         _config.ARCHIVE_FILE = None
+    else:
+        required_builder_names = set([
+            "ubuntu1804-debug-aubsan-lite",
+            "linux-64-debug",
+            "enterprise-windows-required",
+            "enterprise-rhel-62-64-bit",
+            "enterprise-ubuntu-dynamic-1604-clang",
+        ])
+        # Enable archival globally for all required mainline builders.
+        if (_config.EVERGREEN_VARIANT_NAME is not None
+                and _config.EVERGREEN_VARIANT_NAME in required_builder_names
+                and not _config.EVERGREEN_PATCH_BUILD):
+            _config.FORCE_ARCHIVE_ALL_DATA_FILES = True
+
     _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb")
     _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests")
 
diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py
index 0ef70263f82..beef57e5526 100644
--- a/buildscripts/resmokelib/testing/hook_test_archival.py
+++ b/buildscripts/resmokelib/testing/hook_test_archival.py
@@ -37,68 +37,43 @@ class HookTestArchival(object):
             elif archive_config["hooks"]:
                 for hook in hooks:
                     self.hooks.append(hook["class"])
-
         self._tests_repeat = {}
         self._lock = threading.Lock()
 
-    def _should_archive(self, success):
-        """Determine whether archiving should be done."""
-        return config.ARCHIVE_FILE and self.archive_instance \
-            and (not success or self.on_success)
-
-    def _archive_hook(self, logger, result, manager):
+    def archive(self, logger, result, manager):
         """
-        Provide helper to archive hooks.
+        Archive data files for hooks or tests.
 
         :param logger: Where the logging output should be placed.
         :param result: A TestResult named tuple containing the test, hook, and outcome.
         :param manager: FixtureTestCaseManager object for the calling Job.
         """
-        if not result.hook.REGISTERED_NAME in self.hooks:
-            return
-
-        test_name = "{}:{}".format(result.test.short_name(), result.hook.REGISTERED_NAME)
-        self._archive_hook_or_test(logger, test_name, result.test, manager)
 
-    def _archive_test(self, logger, result, manager):
-        """
-        Provide helper to archive tests.
+        success = result.success
+        should_archive = (config.ARCHIVE_FILE and self.archive_instance
+                          and (not success or self.on_success))
 
-        :param logger: Where the logging output should be placed.
-        :param result: A TestResult named tuple containing the test, hook, and outcome.
-        :param manager: FixtureTestCaseManager object for the calling Job.
-
-        """
-        test_name = result.test.test_name
+        if not should_archive:
+            return
 
-        if self.archive_all:
-            test_match = True
+        if result.hook and result.hook.REGISTERED_NAME in self.hooks:
+            test_name = "{}:{}".format(result.test.short_name(), result.hook.REGISTERED_NAME)
+            should_archive = True
         else:
-            test_match = False
-            for arch_test in self.tests:
-                # Ensure that the test_name is in the same format as the arch_test.
-                if os.path.normpath(test_name) == os.path.normpath(arch_test):
-                    test_match = True
-                    break
-
-        if test_match:
+            test_name = result.test.test_name
+            if self.archive_all:
+                should_archive = True
+            else:
+                should_archive = False
+                for arch_test in self.tests:
+                    # Ensure that the test_name is in the same format as the arch_test.
+                    if os.path.normpath(test_name) == os.path.normpath(arch_test):
+                        should_archive = True
+                        break
+
+        if should_archive or config.FORCE_ARCHIVE_ALL_DATA_FILES:
             self._archive_hook_or_test(logger, test_name, result.test, manager)
 
-    def archive(self, logger, result, manager):
-        """
-        Archive data files for hooks or tests.
-
-        :param logger: Where the logging output should be placed.
-        :param result: A TestResult named tuple containing the test, hook, and outcome.
-        :param manager: FixtureTestCaseManager object for the calling Job.
-        """
-        if not self._should_archive(result.success):
-            return
-        if result.hook:
-            self._archive_hook(logger, result, manager)
-        else:
-            self._archive_test(logger, result, manager)
-
     def _archive_hook_or_test(self, logger, test_name, test, manager):
         """Trigger archive of data files for a test or hook."""