SERVER-42879 Generate jstestfuzz replication and sharding multiversion suites

3 files changed, 106 insertions, 24 deletions

diff --git a/buildscripts/evergreen_gen_fuzzer_tests.py b/buildscripts/evergreen_gen_fuzzer_tests.py
index f207d2c4b3d..a33f493d388 100755
--- a/buildscripts/evergreen_gen_fuzzer_tests.py
+++ b/buildscripts/evergreen_gen_fuzzer_tests.py
@@ -13,10 +13,12 @@ from shrub.task import TaskDependency
 from shrub.variant import DisplayTaskDefinition
 from shrub.variant import TaskSpec
 
+import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
 import buildscripts.util.read_config as read_config
 import buildscripts.util.taskname as taskname
 
 CONFIG_DIRECTORY = "generated_resmoke_config"
+TEST_SUITE_DIR = generate_resmoke.DEFAULT_TEST_SUITE_DIR
 
 ConfigOptions = namedtuple("ConfigOptions", [
     "num_files",
@@ -31,6 +33,7 @@ ConfigOptions = namedtuple("ConfigOptions", [
     "should_shuffle",
     "timeout_secs",
     "use_multiversion",
+    "suite",
 ])
 
 
@@ -72,9 +75,11 @@ def _get_config_options(cmd_line_options, config_file):  # pylint: disable=too-m
     use_multiversion = read_config.get_config_value("task_path_suffix", cmd_line_options,
                                                     config_file_data, default=False)
 
+    suite = read_config.get_config_value("suite", cmd_line_options, config_file_data, required=True)
+
     return ConfigOptions(num_files, num_tasks, resmoke_args, npm_command, jstestfuzz_vars, name,
                          variant, continue_on_failure, resmoke_jobs_max, should_shuffle,
-                         timeout_secs, use_multiversion)
+                         timeout_secs, use_multiversion, suite)
 
 
 def _name_task(parent_name, task_index, total_tasks):
@@ -90,20 +95,31 @@ def _name_task(parent_name, task_index, total_tasks):
     return "{0}_{1}".format(parent_name, str(task_index).zfill(index_width))
 
 
-def _generate_evg_tasks(options):
+def _write_fuzzer_yaml(options):
+    """Write the fuzzer yaml to CONFIG_DIRECTORY."""
+    suite_file = options.suite + ".yml"
+    source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file)
+    gen_yml = generate_resmoke.generate_resmoke_suite_config(source_config, suite_file)
+    file_dict = {f"{options.suite}.yml": gen_yml}
+    generate_resmoke.write_file_dict(CONFIG_DIRECTORY, file_dict)
+
+
+def generate_evg_tasks(options, evg_config, task_name_suffix=None, display_task=None):
     """
     Generate an evergreen configuration for fuzzers based on the options given.
 
     :param options: task options.
+    :param evg_config: evergreen configuration.
+    :param task_name_suffix: suffix to be appended to each task name.
+    :param display_task: an existing display task definition to append to.
     :return: An evergreen configuration.
     """
-    evg_config = Configuration()
-
     task_names = []
     task_specs = []
 
     for task_index in range(options.num_tasks):
-        name = taskname.name_generated_task(options.name, task_index, options.num_tasks,
+        task_name = options.name if not task_name_suffix else f"{options.name}_{task_name_suffix}"
+        name = taskname.name_generated_task(task_name, task_index, options.num_tasks,
                                             options.variant)
         task_names.append(name)
         task_specs.append(TaskSpec(name))
@@ -120,21 +136,29 @@ def _generate_evg_tasks(options):
             "npm_command":
                 options.npm_command
         }))
+        # Unix path separators are used because Evergreen only runs this script in unix shells,
+        # even on Windows.
+        suite_arg = f"--suites={CONFIG_DIRECTORY}/{options.suite}.yml"
         run_tests_vars = {
             "continue_on_failure": options.continue_on_failure,
-            "resmoke_args": options.resmoke_args,
+            "resmoke_args": f"{suite_arg} {options.resmoke_args}",
             "resmoke_jobs_max": options.resmoke_jobs_max,
             "should_shuffle": options.should_shuffle,
             "task_path_suffix": options.use_multiversion,
             "timeout_secs": options.timeout_secs,
-        }
+            "task": options.name
+        }  # yapf: disable
 
-        commands.append(CommandDefinition().function("run tests").vars(run_tests_vars))
+        commands.append(CommandDefinition().function("run generated tests").vars(run_tests_vars))
         task.dependency(TaskDependency("compile")).commands(commands)
 
-    dt = DisplayTaskDefinition(options.name).execution_tasks(task_names)\
-        .execution_task("{0}_gen".format(options.name))
-    evg_config.variant(options.variant).tasks(task_specs).display_task(dt)
+    # Create a new DisplayTaskDefinition or append to the one passed in.
+    dt = DisplayTaskDefinition(task_name) if not display_task else display_task
+    dt.execution_tasks(task_names)
+    evg_config.variant(options.variant).tasks(task_specs)
+    if not display_task:
+        dt.execution_task("{0}_gen".format(options.name))
+        evg_config.variant(options.variant).display_task(dt)
 
     return evg_config
 
@@ -165,12 +189,14 @@ def main():
                         help="should_shuffle value for generated tasks.")
     parser.add_argument("--timeout-secs", dest="timeout_secs",
                         help="timeout_secs value for generated tasks.")
+    parser.add_argument("--suite", dest="suite", help="Suite to run using resmoke.")
 
     options = parser.parse_args()
 
     config_options = _get_config_options(options, options.expansion_file)
-
-    evg_config = _generate_evg_tasks(config_options)
+    evg_config = Configuration()
+    _write_fuzzer_yaml(config_options)
+    generate_evg_tasks(config_options, evg_config)
 
     if not os.path.exists(CONFIG_DIRECTORY):
         os.makedirs(CONFIG_DIRECTORY)
diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index 3076a0133d9..90e06d8b511 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -5,6 +5,7 @@ import datetime
 from datetime import timedelta
 import logging
 import os
+import sys
 import tempfile
 
 from collections import namedtuple
@@ -28,6 +29,7 @@ import buildscripts.resmokelib.parser
 import buildscripts.util.read_config as read_config
 import buildscripts.util.taskname as taskname
 import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
+import buildscripts.evergreen_gen_fuzzer_tests as gen_fuzzer
 
 LOGGER = structlog.getLogger(__name__)
 
@@ -40,7 +42,7 @@ LAST_STABLE_MONGO_BINARY = "mongo-4.2"
 
 DEFAULT_CONFIG_VALUES = generate_resmoke.DEFAULT_CONFIG_VALUES
 CONFIG_DIR = DEFAULT_CONFIG_VALUES["generated_config_dir"]
-DEFAULT_CONFIG_VALUES["is_sharded"] = False
+DEFAULT_CONFIG_VALUES["is_jstestfuzz"] = False
 TEST_SUITE_DIR = DEFAULT_CONFIG_VALUES["test_suites_dir"]
 CONFIG_FILE = generate_resmoke.CONFIG_FILE
 CONFIG_FORMAT_FN = generate_resmoke.CONFIG_FORMAT_FN
@@ -50,6 +52,7 @@ SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"]
 BURN_IN_TASK = "burn_in_tests_multiversion"
 BURN_IN_CONFIG_KEY = "use_in_multiversion_burn_in_tests"
 PASSTHROUGH_TAG = "multiversion_passthrough"
+EXCLUDE_TAGS = "requires_fcv_44,multiversion_incompatible"
 
 # The directory in which BACKPORTS_REQUIRED_FILE resides.
 ETC_DIR = "etc"
@@ -57,6 +60,16 @@ BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml"
 BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo"
 
 
+def enable_logging():
+    """Enable INFO level logging."""
+    logging.basicConfig(
+        format="[%(asctime)s - %(name)s - %(levelname)s] %(message)s",
+        level=logging.INFO,
+        stream=sys.stdout,
+    )
+    structlog.configure(logger_factory=structlog.stdlib.LoggerFactory())
+
+
 def prepare_directory_for_suite(directory):
     """Ensure that directory exists."""
     if not os.path.exists(directory):
@@ -173,13 +186,12 @@ class EvergreenConfigGenerator(object):
             # Fetch and download the proper mongod binaries before running multiversion tests.
             CommandDefinition().function("do multiversion setup")
         ]
-        exclude_tags = "requires_fcv_44,multiversion_incompatible"
         # TODO(SERVER-43306): Remove --dryRun command line option once we start turning on
         #  multiversion tests.
         run_tests_vars = {
             "resmoke_args":
                 "{0} --suite={1} --mixedBinVersions={2} --excludeWithAnyTags={3} --dryRun=tests ".
-                format(self.options.resmoke_args, suite, mixed_version_config, exclude_tags),
+                format(self.options.resmoke_args, suite, mixed_version_config, EXCLUDE_TAGS),
             "task":
                 gen_task_name,
         }
@@ -215,6 +227,34 @@ class EvergreenConfigGenerator(object):
                                     burn_in_test)
         return self.evg_config
 
+    def _get_fuzzer_options(self, version_config, suite_file):
+        fuzzer_config = generate_resmoke.ConfigOptions(self.options.config)
+        fuzzer_config.name = f"{self.options.suite}_multiversion"
+        fuzzer_config.num_files = int(self.options.num_files)
+        fuzzer_config.num_tasks = int(self.options.num_tasks)
+        fuzzer_config.resmoke_args = f"{self.options.resmoke_args} "\
+            f"--mixedBinVersions={version_config} --excludeWithAnyTags={EXCLUDE_TAGS}"\
+            f" --suites={CONFIG_DIR}/{suite_file}"
+        return fuzzer_config
+
+    def _generate_fuzzer_tasks(self, config):
+        suite_file = self.options.suite + ".yml"
+        # Update the jstestfuzz yml suite with the proper multiversion configurations.
+        source_config = generate_resmoke.read_yaml(TEST_SUITE_DIR, suite_file)
+        config.update_yaml(source_config)
+        updated_yml = generate_resmoke.generate_resmoke_suite_config(source_config, suite_file)
+        file_dict = {f"{self.options.suite}.yml": updated_yml}
+        dt = DisplayTaskDefinition(self.task)
+
+        for version_config in config.version_configs:
+            fuzzer_config = self._get_fuzzer_options(version_config, suite_file)
+            gen_fuzzer.generate_evg_tasks(fuzzer_config, self.evg_config,
+                                          task_name_suffix=version_config, display_task=dt)
+        generate_resmoke.write_file_dict(CONFIG_DIR, file_dict)
+        dt.execution_task(f"{fuzzer_config.name}_gen")
+        self.evg_config.variant(self.options.variant).display_task(dt)
+        return self.evg_config
+
     def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0):
         # pylint: disable=too-many-locals
         """
@@ -225,6 +265,16 @@ class EvergreenConfigGenerator(object):
 
         :param burn_in_test: The test to be run as part of the burn in multiversion suite.
         """
+        if is_suite_sharded(TEST_SUITE_DIR, self.options.suite):
+            config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
+                                        SHARDED_MIXED_VERSION_CONFIGS)
+        else:
+            config = MultiversionConfig(update_suite_config_for_multiversion_replset,
+                                        REPL_MIXED_VERSION_CONFIGS)
+
+        if self.options.is_jstestfuzz:
+            return self._generate_fuzzer_tasks(config)
+
         # Divide tests into suites based on run-time statistics for the last
         # LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
         # in the misc suite.
@@ -236,12 +286,6 @@ class EvergreenConfigGenerator(object):
         for suite in suites:
             suite.source_name = self.task
         # Render the given suites into yml files that can be used by resmoke.py.
-        if is_suite_sharded(TEST_SUITE_DIR, self.options.suite):
-            config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
-                                        SHARDED_MIXED_VERSION_CONFIGS)
-        else:
-            config = MultiversionConfig(update_suite_config_for_multiversion_replset,
-                                        REPL_MIXED_VERSION_CONFIGS)
         config_file_dict = generate_resmoke.render_suite_files(
             suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
         # Update the base misc suite name to the multiversion name.
@@ -278,6 +322,7 @@ class EvergreenConfigGenerator(object):
         if not generate_resmoke.should_tasks_be_generated(self.evg_api, self.options.task_id):
             LOGGER.info("Not generating configuration due to previous successful generation.")
             return
+
         self.generate_evg_tasks()
         self._write_evergreen_config_to_file(self.task)
 
@@ -331,10 +376,16 @@ def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite):
     last-stable branch to determine which tests should be blacklisted.
     """
 
+    enable_logging()
+
     suite_name = generate_resmoke.remove_gen_suffix(suite)
 
     # Get the backports_required_for_multiversion_tests.yml on the current version branch.
     backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
+    if suite_name not in backports_required_latest:
+        LOGGER.info(f"Generating exclude files not supported for '{suite_name}''.")
+        return
+
     latest_suite_yaml = backports_required_latest[suite_name]
 
     if not latest_suite_yaml:
diff --git a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
index ddcc6f17e00..688ecf39719 100644
--- a/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
+++ b/buildscripts/tests/test_evergreen_gen_fuzzer_tests.py
@@ -3,6 +3,8 @@
 import unittest
 import mock
 
+from shrub.config import Configuration
+
 from buildscripts import evergreen_gen_fuzzer_tests as gft
 
 # pylint: disable=missing-docstring,protected-access
@@ -24,13 +26,15 @@ class TestGenerateEvgTasks(unittest.TestCase):
         options.resmoke_jobs_max = 0
         options.should_shuffle = "false"
         options.timeout_secs = "1800"
+        options.suite = "test_suite"
 
         return options
 
     def test_evg_config_is_created_without_multiversion(self):
+        evg_config = Configuration()
         options = self._create_options_mock()
 
-        config = gft._generate_evg_tasks(options).to_map()
+        config = gft.generate_evg_tasks(options, evg_config).to_map()
 
         self.assertEqual(options.num_tasks, len(config["tasks"]))
 
@@ -50,10 +54,11 @@ class TestGenerateEvgTasks(unittest.TestCase):
         self.assertIn(options.name + "_gen", buildvariant["display_tasks"][0]["execution_tasks"])
 
     def test_evg_config_is_created_with_multiversion(self):
+        evg_config = Configuration()
         options = self._create_options_mock()
         options.use_multiversion = "/data/multiversion"
 
-        config = gft._generate_evg_tasks(options).to_map()
+        config = gft.generate_evg_tasks(options, evg_config).to_map()
 
         self.assertEqual("do multiversion setup", config["tasks"][0]["commands"][1]["func"])
         self.assertEqual("/data/multiversion",