SERVER-43797 Add new burn_in_tests_multiversion_gen task

1 files changed, 59 insertions, 19 deletions

diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index 781b7a8fd7e..a034e5f69f6 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -36,9 +36,13 @@ DEFAULT_CONFIG_VALUES["is_sharded"] = False
 TEST_SUITE_DIR = DEFAULT_CONFIG_VALUES["test_suites_dir"]
 CONFIG_FILE = generate_resmoke.CONFIG_FILE
 CONFIG_FORMAT_FN = generate_resmoke.CONFIG_FORMAT_FN
-MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"]
+REPL_MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"]
 SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"]
 
+BURN_IN_TASK = "burn_in_tests_multiversion"
+BURN_IN_CONFIG_KEY = "use_in_multiversion_burn_in_tests"
+PASSTHROUGH_TAG = "multiversion_passthrough"
+
 
 def prepare_directory_for_suite(directory):
     """Ensure that directory exists."""
@@ -46,6 +50,12 @@ def prepare_directory_for_suite(directory):
         os.makedirs(directory)
 
 
+def is_suite_sharded(suite_dir, suite_name):
+    """Return true if a suite uses ShardedClusterFixture."""
+    source_config = generate_resmoke.read_yaml(suite_dir, suite_name + ".yml")
+    return source_config["executor"]["fixture"]["class"] == "ShardedClusterFixture"
+
+
 def update_suite_config_for_multiversion_replset(suite_config):
     """Update suite_config with arguments for multiversion tests using ReplicaSetFixture."""
     suite_config["executor"]["fixture"]["num_nodes"] = 3
@@ -96,17 +106,21 @@ class EvergreenConfigGenerator(object):
         # Strip the "_gen" suffix appended to the name of tasks generated by evergreen.
         self.task = generate_resmoke.remove_gen_suffix(self.options.task)
 
-    def _generate_sub_task(self, mixed_version_config, task_index, suite, num_suites):
+    def _generate_sub_task(self, mixed_version_config, task, task_index, suite, num_suites,
+                           burn_in_test=None):
+        #pylint: disable=too-many-arguments
         """Generate a sub task to be run with the provided suite and  mixed version config."""
 
         # Create a sub task name appended with the task_index and build variant name.
-        task = "{0}_{1}".format(self.task, mixed_version_config)
-        sub_task_name = taskname.name_generated_task(task, task_index, num_suites,
+        task_name = "{0}_{1}".format(task, mixed_version_config)
+        sub_task_name = taskname.name_generated_task(task_name, task_index, num_suites,
                                                      self.options.variant)
         self.task_names.append(sub_task_name)
         self.task_specs.append(TaskSpec(sub_task_name))
         task = self.evg_config.task(sub_task_name)
 
+        gen_task_name = BURN_IN_TASK if burn_in_test is not None else self.task
+
         commands = [CommandDefinition().function("do setup")]
         # Fetch and download the proper mongod binaries before running multiversion tests.
         commands.append(CommandDefinition().function("do multiversion setup"))
@@ -114,11 +128,13 @@ class EvergreenConfigGenerator(object):
         # TODO(SERVER-43306): Remove --dryRun command line option once we start turning on multiversion tests.
         run_tests_vars = {
             "resmoke_args":
-                "{0} --suite={1} --mixedBinVersions={2} --excludeWithAnyTags={3} --dryRun=tests".
+                "{0} --suite={1} --mixedBinVersions={2} --excludeWithAnyTags={3} --dryRun=tests ".
                 format(self.options.resmoke_args, suite, mixed_version_config, exclude_tags),
             "task":
-                self.task,
+                gen_task_name,
         }
+        if burn_in_test is not None:
+            run_tests_vars["resmoke_args"] += burn_in_test
 
         commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars))
         task.dependency(TaskDependency("compile")).commands(commands)
@@ -131,12 +147,32 @@ class EvergreenConfigGenerator(object):
         with open(os.path.join(CONFIG_DIR, task_name + ".json"), "w") as file_handle:
             file_handle.write(self.evg_config.to_json())
 
-    def generate_evg_tasks(self):
+    def create_display_task(self, task_name, task_specs, task_list):
+        """Create the display task definition for the MultiversionConfig object."""
+        dt = DisplayTaskDefinition(task_name).execution_tasks(task_list)\
+            .execution_task("{0}_gen".format(task_name))
+        self.evg_config.variant(self.options.variant).tasks(task_specs).display_task(dt)
+
+    def _generate_burn_in_execution_tasks(self, config, suites, burn_in_test, burn_in_idx):
+        burn_in_prefix = "burn_in_multiversion"
+        task = "{0}:{1}".format(burn_in_prefix, self.task)
+
+        for version_config in config.version_configs:
+            # For burn in tasks, it doesn't matter which generated suite yml to use as all the
+            # yaml configurations are the same.
+            source_suite = os.path.join(CONFIG_DIR, suites[0].name + ".yml")
+            self._generate_sub_task(version_config, task, burn_in_idx, source_suite, 1,
+                                    burn_in_test)
+        return self.evg_config
+
+    def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0):
         """
         Generate evergreen tasks for multiversion tests.
 
         The number of tasks generated equals
-        (the number of configs in MIXED_VERSION_CONFIGS) * (the number of generated suites).
+        (the number of version configs) * (the number of generated suites).
+
+        :param burn_in_test: The test to be run as part of the burn in multiversion suite.
         """
         idx = 0
         # Divide tests into suites based on run-time statistics for the last
@@ -147,32 +183,36 @@ class EvergreenConfigGenerator(object):
         start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
         suites = gen_suites.calculate_suites(start_date, end_date)
         # Render the given suites into yml files that can be used by resmoke.py.
-        if self.options.is_sharded:
+        if is_suite_sharded(TEST_SUITE_DIR, self.options.suite):
             config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
                                         SHARDED_MIXED_VERSION_CONFIGS)
         else:
             config = MultiversionConfig(update_suite_config_for_multiversion_replset,
-                                        MIXED_VERSION_CONFIGS)
+                                        REPL_MIXED_VERSION_CONFIGS)
         config_file_dict = generate_resmoke.render_suite_files(
             suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
+        generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
+
+        if burn_in_test is not None:
+            # Generate the subtasks to run burn_in_test against the appropriate mixed version
+            # configurations. The display task is defined later as part of generating the burn
+            # in tests.
+            self._generate_burn_in_execution_tasks(config, suites, burn_in_test, burn_in_idx)
+            return self.evg_config
+
         for version_config in config.version_configs:
             for suite in suites:
                 # Generate the newly divided test suites
                 source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
-                self._generate_sub_task(version_config, idx, source_suite, len(suites))
+                self._generate_sub_task(version_config, self.task, idx, source_suite, len(suites))
                 idx += 1
 
             # Also generate the misc task.
             misc_suite_name = "{0}_misc".format(self.options.suite)
             source_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
-            self._generate_sub_task(version_config, idx, source_suite, 1)
+            self._generate_sub_task(version_config, self.task, idx, source_suite, 1)
             idx += 1
-
-        generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
-        dt = DisplayTaskDefinition(self.task).execution_tasks(self.task_names)\
-            .execution_task("{0}_gen".format(self.task))
-        self.evg_config.variant(self.options.variant).tasks(self.task_specs).display_task(dt)
-
+        self.create_display_task(self.task, self.task_specs, self.task_names)
         return self.evg_config
 
     def run(self):
@@ -195,7 +235,7 @@ def main(expansion_file, evergreen_config=None):
 
     Tests using ReplicaSetFixture will be generated to use 3 nodes and linear_chain=True.
     Tests using ShardedClusterFixture will be generated to use 2 shards with 2 nodes each.
-    The different binary version configurations tested are stored in MIXED_VERSION_CONFIGS
+    The different binary version configurations tested are stored in REPL_MIXED_VERSION_CONFIGS
     and SHARDED_MIXED_VERSION_CONFIGS.
 
     The `--expansion-file` should contain all the configuration needed to generate the tasks.