SERVER-39284: Handle generated task timeouts with repeated suites

2 files changed, 53 insertions, 15 deletions

diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py
index 9ecd1e10f5d..a0d1893cdcf 100755
--- a/buildscripts/evergreen_generate_resmoke_tasks.py
+++ b/buildscripts/evergreen_generate_resmoke_tasks.py
@@ -51,22 +51,23 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
 """
 
 ConfigOptions = namedtuple("ConfigOptions", [
+    "build_id",
+    "depends_on",
     "fallback_num_sub_suites",
+    "is_patch",
+    "large_distro_name",
     "max_sub_suites",
     "project",
+    "repeat_suites",
     "resmoke_args",
     "resmoke_jobs_max",
-    "target_resmoke_time",
     "run_multiple_jobs",
     "suite",
+    "target_resmoke_time",
     "task",
-    "variant",
     "use_large_distro",
-    "large_distro_name",
     "use_multiversion",
-    "is_patch",
-    "depends_on",
-    "build_id",
+    "variant",
 ])
 
 
@@ -132,11 +133,14 @@ def get_config_options(cmd_line_options, config_file):
     depends_on = split_if_exists(
         read_config.get_config_value("depends_on", cmd_line_options, config_file_data))
     build_id = read_config.get_config_value("build_id", cmd_line_options, config_file_data)
+    repeat_suites = int(
+        read_config.get_config_value("resmoke_repeat_suites", cmd_line_options, config_file_data,
+                                     default=1))
 
-    return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args,
-                         resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task,
-                         variant, use_large_distro, large_distro_name, use_multiversion, is_patch,
-                         depends_on, build_id)
+    return ConfigOptions(build_id, depends_on, fallback_num_sub_suites, is_patch, large_distro_name,
+                         max_sub_suites, project, repeat_suites, resmoke_args, resmoke_jobs_max,
+                         run_multiple_jobs, suite, target_resmoke_time, task, use_large_distro,
+                         use_multiversion, variant)
 
 
 def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites):
@@ -281,9 +285,16 @@ class EvergreenConfigGenerator(object):
         if self.options.use_large_distro and self.options.large_distro_name:
             task_spec.distro(self.options.large_distro_name)
 
+    def _generate_resmoke_args(self, suite_file):
+        resmoke_args = "--suite={0}.yml {1}".format(suite_file, self.options.resmoke_args)
+        if self.options.repeat_suites and "repeatSuites" not in resmoke_args:
+            resmoke_args += " --repeatSuites={0} ".format(self.options.repeat_suites)
+
+        return resmoke_args
+
     def _get_run_tests_vars(self, suite_file):
         variables = {
-            "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args),
+            "resmoke_args": self._generate_resmoke_args(suite_file),
             "run_multiple_jobs": self.options.run_multiple_jobs,
             "task": self.options.task,
         }
@@ -296,14 +307,20 @@ class EvergreenConfigGenerator(object):
 
         return variables
 
-    @staticmethod
-    def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime):
+    def _add_timeout_command(self, commands, max_test_runtime, expected_suite_runtime):
+        repeat_factor = self.options.repeat_suites
         if max_test_runtime or expected_suite_runtime:
             cmd_timeout = CmdTimeoutUpdate()
             if max_test_runtime:
-                cmd_timeout.timeout(calculate_timeout(max_test_runtime, 3))
+                timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor
+                LOGGER.debug("Setting timeout to: %d (max=%d, repeat=%d)", timeout,
+                             max_test_runtime, repeat_factor)
+                cmd_timeout.timeout(timeout)
             if expected_suite_runtime:
-                cmd_timeout.exec_timeout(calculate_timeout(expected_suite_runtime, 3))
+                exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor
+                LOGGER.debug("Setting exec_timeout to: %d (runtime=%d, repeat=%d)", exec_timeout,
+                             expected_suite_runtime, repeat_factor)
+                cmd_timeout.exec_timeout(exec_timeout)
             commands.append(cmd_timeout.validate().resolve())
 
     @staticmethod
@@ -523,6 +540,8 @@ class Main(object):
         parser.add_argument("--is-patch", dest="is_patch", help="Is this part of a patch build.")
         parser.add_argument("--depends-on", dest="depends_on",
                             help="Generate depends on for these tasks.")
+        parser.add_argument("--repeat-suites", dest="resmoke_repeat_suites",
+                            help="Repeat each suite the specified number of times.")
         parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                             help="Enable verbose logging.")
 
diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
index af6fdadf9d1..dbac0cd1802 100644
--- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
+++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
@@ -335,6 +335,7 @@ class EvergreenConfigGeneratorTest(unittest.TestCase):
         options.use_large_distro = None
         options.use_multiversion = False
         options.is_patch = True
+        options.repeat_suites = 1
 
         return options
 
@@ -434,6 +435,24 @@ class EvergreenConfigGeneratorTest(unittest.TestCase):
         cfg_generator._add_dependencies(cfg_mock)
         self.assertEqual(4, cfg_mock.dependency.call_count)
 
+    def test_evg_config_has_timeouts_for_repeated_suites(self):
+        options = self.generate_mock_options()
+        options.repeat_suites = 5
+        suites = self.generate_mock_suites(3)
+
+        config = grt.EvergreenConfigGenerator(suites, options, Mock()).generate_config().to_map()
+
+        self.assertEqual(len(config["tasks"]), len(suites) + 1)
+        command1 = config["tasks"][0]["commands"][2]
+        self.assertIn(" --repeatSuites=5 ", command1["vars"]["resmoke_args"])
+        self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
+        timeout_cmd = config["tasks"][0]["commands"][0]
+        self.assertEqual("timeout.update", timeout_cmd["command"])
+        expected_timeout = grt.calculate_timeout(suites[0].max_runtime, 3) * 5
+        self.assertEqual(expected_timeout, timeout_cmd["params"]["timeout_secs"])
+        expected_exec_timeout = grt.calculate_timeout(suites[0].get_runtime(), 3) * 5
+        self.assertEqual(expected_exec_timeout, timeout_cmd["params"]["exec_timeout_secs"])
+
 
 class NormalizeTestNameTest(unittest.TestCase):
     def test_unix_names(self):