SERVER-39284: Handle generated task timeouts with repeated suites

3 files changed, 156 insertions, 36 deletions

diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py
index 9ecd1e10f5d..a0d1893cdcf 100755
--- a/buildscripts/evergreen_generate_resmoke_tasks.py
+++ b/buildscripts/evergreen_generate_resmoke_tasks.py
@@ -51,22 +51,23 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
 """
 
 ConfigOptions = namedtuple("ConfigOptions", [
+    "build_id",
+    "depends_on",
     "fallback_num_sub_suites",
+    "is_patch",
+    "large_distro_name",
     "max_sub_suites",
     "project",
+    "repeat_suites",
     "resmoke_args",
     "resmoke_jobs_max",
-    "target_resmoke_time",
     "run_multiple_jobs",
     "suite",
+    "target_resmoke_time",
     "task",
-    "variant",
     "use_large_distro",
-    "large_distro_name",
     "use_multiversion",
-    "is_patch",
-    "depends_on",
-    "build_id",
+    "variant",
 ])
 
 
@@ -132,11 +133,14 @@ def get_config_options(cmd_line_options, config_file):
     depends_on = split_if_exists(
         read_config.get_config_value("depends_on", cmd_line_options, config_file_data))
     build_id = read_config.get_config_value("build_id", cmd_line_options, config_file_data)
+    repeat_suites = int(
+        read_config.get_config_value("resmoke_repeat_suites", cmd_line_options, config_file_data,
+                                     default=1))
 
-    return ConfigOptions(fallback_num_sub_suites, max_sub_suites, project, resmoke_args,
-                         resmoke_jobs_max, target_resmoke_time, run_multiple_jobs, suite, task,
-                         variant, use_large_distro, large_distro_name, use_multiversion, is_patch,
-                         depends_on, build_id)
+    return ConfigOptions(build_id, depends_on, fallback_num_sub_suites, is_patch, large_distro_name,
+                         max_sub_suites, project, repeat_suites, resmoke_args, resmoke_jobs_max,
+                         run_multiple_jobs, suite, target_resmoke_time, task, use_large_distro,
+                         use_multiversion, variant)
 
 
 def divide_remaining_tests_among_suites(remaining_tests_runtimes, suites):
@@ -281,9 +285,16 @@ class EvergreenConfigGenerator(object):
         if self.options.use_large_distro and self.options.large_distro_name:
             task_spec.distro(self.options.large_distro_name)
 
+    def _generate_resmoke_args(self, suite_file):
+        resmoke_args = "--suite={0}.yml {1}".format(suite_file, self.options.resmoke_args)
+        if self.options.repeat_suites and "repeatSuites" not in resmoke_args:
+            resmoke_args += " --repeatSuites={0} ".format(self.options.repeat_suites)
+
+        return resmoke_args
+
     def _get_run_tests_vars(self, suite_file):
         variables = {
-            "resmoke_args": "--suites={0}.yml {1}".format(suite_file, self.options.resmoke_args),
+            "resmoke_args": self._generate_resmoke_args(suite_file),
             "run_multiple_jobs": self.options.run_multiple_jobs,
             "task": self.options.task,
         }
@@ -296,14 +307,20 @@ class EvergreenConfigGenerator(object):
 
         return variables
 
-    @staticmethod
-    def _add_timeout_command(commands, max_test_runtime, expected_suite_runtime):
+    def _add_timeout_command(self, commands, max_test_runtime, expected_suite_runtime):
+        repeat_factor = self.options.repeat_suites
         if max_test_runtime or expected_suite_runtime:
             cmd_timeout = CmdTimeoutUpdate()
             if max_test_runtime:
-                cmd_timeout.timeout(calculate_timeout(max_test_runtime, 3))
+                timeout = calculate_timeout(max_test_runtime, 3) * repeat_factor
+                LOGGER.debug("Setting timeout to: %d (max=%d, repeat=%d)", timeout,
+                             max_test_runtime, repeat_factor)
+                cmd_timeout.timeout(timeout)
             if expected_suite_runtime:
-                cmd_timeout.exec_timeout(calculate_timeout(expected_suite_runtime, 3))
+                exec_timeout = calculate_timeout(expected_suite_runtime, 3) * repeat_factor
+                LOGGER.debug("Setting exec_timeout to: %d (runtime=%d, repeat=%d)", exec_timeout,
+                             expected_suite_runtime, repeat_factor)
+                cmd_timeout.exec_timeout(exec_timeout)
             commands.append(cmd_timeout.validate().resolve())
 
     @staticmethod
@@ -523,6 +540,8 @@ class Main(object):
         parser.add_argument("--is-patch", dest="is_patch", help="Is this part of a patch build.")
         parser.add_argument("--depends-on", dest="depends_on",
                             help="Generate depends on for these tasks.")
+        parser.add_argument("--repeat-suites", dest="resmoke_repeat_suites",
+                            help="Repeat each suite the specified number of times.")
         parser.add_argument("--verbose", dest="verbose", action="store_true", default=False,
                             help="Enable verbose logging.")
 
diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
index af6fdadf9d1..dbac0cd1802 100644
--- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
+++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py
@@ -335,6 +335,7 @@ class EvergreenConfigGeneratorTest(unittest.TestCase):
         options.use_large_distro = None
         options.use_multiversion = False
         options.is_patch = True
+        options.repeat_suites = 1
 
         return options
 
@@ -434,6 +435,24 @@ class EvergreenConfigGeneratorTest(unittest.TestCase):
         cfg_generator._add_dependencies(cfg_mock)
         self.assertEqual(4, cfg_mock.dependency.call_count)
 
+    def test_evg_config_has_timeouts_for_repeated_suites(self):
+        options = self.generate_mock_options()
+        options.repeat_suites = 5
+        suites = self.generate_mock_suites(3)
+
+        config = grt.EvergreenConfigGenerator(suites, options, Mock()).generate_config().to_map()
+
+        self.assertEqual(len(config["tasks"]), len(suites) + 1)
+        command1 = config["tasks"][0]["commands"][2]
+        self.assertIn(" --repeatSuites=5 ", command1["vars"]["resmoke_args"])
+        self.assertIn(options.resmoke_args, command1["vars"]["resmoke_args"])
+        timeout_cmd = config["tasks"][0]["commands"][0]
+        self.assertEqual("timeout.update", timeout_cmd["command"])
+        expected_timeout = grt.calculate_timeout(suites[0].max_runtime, 3) * 5
+        self.assertEqual(expected_timeout, timeout_cmd["params"]["timeout_secs"])
+        expected_exec_timeout = grt.calculate_timeout(suites[0].get_runtime(), 3) * 5
+        self.assertEqual(expected_exec_timeout, timeout_cmd["params"]["exec_timeout_secs"])
+
 
 class NormalizeTestNameTest(unittest.TestCase):
     def test_unix_names(self):
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d2f87cb0669..49b0e1e8afb 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -5977,6 +5977,18 @@ tasks:
       resmoke_args: --suites=parallel --storageEngine=wiredTiger
       resmoke_jobs_max: 1
 
+- name: parallel_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: parallel
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+      resmoke_jobs_max: 1
+
 - <<: *task_template
   name: parallel_compatibility
   depends_on:
@@ -6706,6 +6718,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_sharding_default_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_sharding_default_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough
   depends_on:
@@ -6716,6 +6739,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_sharding_100ms_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_sharding_100ms_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough
   depends_on:
@@ -6747,6 +6781,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_sharding_10sec_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_sharding_10sec_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: logical_session_cache_standalone_default_refresh_jscore_passthrough
   depends_on:
@@ -6757,6 +6802,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_standalone_default_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_standalone_default_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough
   depends_on:
@@ -6767,6 +6823,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_standalone_100ms_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_standalone_100ms_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough
   depends_on:
@@ -6798,6 +6865,17 @@ tasks:
     vars:
       resmoke_args: --suites=logical_session_cache_standalone_10sec_refresh_jscore_passthrough --storageEngine=wiredTiger
 
+- name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen
+  depends_on:
+  - name: jsCore
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      task: logical_session_cache_standalone_10sec_refresh_jscore_passthrough
+      depends_on: jsCore
+      resmoke_args: --storageEngine=wiredTiger
+      fallback_num_sub_suites: 1
+
 - <<: *task_template
   name: retryable_writes_jscore_stepdown_passthrough
   depends_on:
@@ -7701,6 +7779,10 @@ buildvariants:
       --excludeWithAnyTags=requires_http_client
       --repeatSuites=10
       --shuffle
+    resmoke_repeat_suites: 10
+    # TODO: There are currently 2 ways of repeating suites, one for regular suites and one for
+    # generated suites. Once everything is converted to generated suites, we should remove the
+    # '--repeatSuites=10' from the test_flags. This will be done in SERVER-38817.
     scons_cache_scope: shared
     tooltags: ""
     build_mongoreplay: true
@@ -7716,19 +7798,19 @@ buildvariants:
   - name: integration_tests_sharded
   - name: jsCore
   - name: jsCore_txns
-  - name: logical_session_cache_replication_100ms_refresh_jscore_passthrough
-  - name: logical_session_cache_replication_1sec_refresh_jscore_passthrough
-  - name: logical_session_cache_replication_10sec_refresh_jscore_passthrough
-  - name: logical_session_cache_replication_default_refresh_jscore_passthrough
-  - name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough
-  - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough
-  - name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough
-  - name: logical_session_cache_sharding_default_refresh_jscore_passthrough
-  - name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough
-  - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough
-  - name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough
-  - name: logical_session_cache_standalone_default_refresh_jscore_passthrough
-  - name: parallel
+  - name: logical_session_cache_replication_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_replication_default_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_sharding_default_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_100ms_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_1sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_10sec_refresh_jscore_passthrough_gen
+  - name: logical_session_cache_standalone_default_refresh_jscore_passthrough_gen
+  - name: parallel_gen
   - name: concurrency
   - name: concurrency_replication
     distros:
@@ -7743,17 +7825,17 @@ buildvariants:
     distros:
     - rhel62-large
   #- name: concurrency_sharded_replication_local_read_write_multi_stmt_txn # Removed until SERVER-38499 is resolved.
-    distros:
-    - rhel62-large
+  #  distros:
+  #  - rhel62-large
   #- name: concurrency_sharded_replication_local_read_write_multi_stmt_txn_with_balancer # Removed until SERVER-38499 is resolved.
-    distros:
-    - rhel62-large
+  #  distros:
+  #  - rhel62-large
   #- name: concurrency_sharded_replication_multi_stmt_txn # Removed until SERVER-38499 is resolved.
-    distros:
-    - rhel62-large
+  #  distros:
+  #  - rhel62-large
   #- name: concurrency_sharded_replication_multi_stmt_txn_with_balancer # Removed until SERVER-38499 is resolved.
-    distros:
-    - rhel62-large
+  #  distros:
+  #  - rhel62-large
   - name: concurrency_sharded_replication_with_balancer
     distros:
     - rhel62-large