SERVER-43865 Add blacklist logic on multiversion test generation

1 files changed, 139 insertions, 6 deletions

diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index a8c501b39d2..3076a0133d9 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -5,20 +5,26 @@ import datetime
 from datetime import timedelta
 import logging
 import os
+import tempfile
 
 from collections import namedtuple
+from subprocess import check_output
 
+import requests
+import yaml
 import click
 import structlog
-import yaml
 
 from evergreen.api import RetryingEvergreenApi
+from git import Repo
 from shrub.config import Configuration
 from shrub.command import CommandDefinition
 from shrub.task import TaskDependency
 from shrub.variant import DisplayTaskDefinition
 from shrub.variant import TaskSpec
 
+from buildscripts.resmokelib import config as _config
+import buildscripts.resmokelib.parser
 import buildscripts.util.read_config as read_config
 import buildscripts.util.taskname as taskname
 import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
@@ -30,6 +36,8 @@ REQUIRED_CONFIG_KEYS = {
     "use_multiversion"
 }
 
+LAST_STABLE_MONGO_BINARY = "mongo-4.2"
+
 DEFAULT_CONFIG_VALUES = generate_resmoke.DEFAULT_CONFIG_VALUES
 CONFIG_DIR = DEFAULT_CONFIG_VALUES["generated_config_dir"]
 DEFAULT_CONFIG_VALUES["is_sharded"] = False
@@ -43,6 +51,11 @@ BURN_IN_TASK = "burn_in_tests_multiversion"
 BURN_IN_CONFIG_KEY = "use_in_multiversion_burn_in_tests"
 PASSTHROUGH_TAG = "multiversion_passthrough"
 
+# The directory in which BACKPORTS_REQUIRED_FILE resides.
+ETC_DIR = "etc"
+BACKPORTS_REQUIRED_FILE = "backports_required_for_multiversion_tests.yml"
+BACKPORTS_REQUIRED_BASE_URL = "https://raw.githubusercontent.com/mongodb/mongo"
+
 
 def prepare_directory_for_suite(directory):
     """Ensure that directory exists."""
@@ -85,6 +98,39 @@ def update_suite_config_for_multiversion_sharded(suite_config):
     suite_config["executor"]["fixture"]["num_rs_nodes_per_shard"] = 2
 
 
+def get_backports_required_last_stable_hash(task_path_suffix):
+    """Parse the last-stable shell binary to get the commit hash."""
+    last_stable_shell_exec = os.path.join(task_path_suffix, LAST_STABLE_MONGO_BINARY)
+    shell_version = check_output([last_stable_shell_exec, "--version"]).decode('utf-8')
+    last_stable_commit_hash = ""
+    for line in shell_version.splitlines():
+        if "git version" in line:
+            last_stable_commit_hash = line.split(':')[1].strip()
+            break
+    if not last_stable_commit_hash:
+        raise ValueError("Could not find a valid commit hash from the last-stable mongo binary.")
+    return last_stable_commit_hash
+
+
+def get_last_stable_yaml(last_stable_commit_hash, suite_name):
+    """Download BACKPORTS_REQUIRED_FILE from the last stable commit and return the yaml."""
+    LOGGER.info(
+        f"Downloading file from commit hash of last-stable branch {last_stable_commit_hash}")
+    response = requests.get(
+        f'{BACKPORTS_REQUIRED_BASE_URL}/{last_stable_commit_hash}/{ETC_DIR}/{BACKPORTS_REQUIRED_FILE}'
+    )
+    # If the response was successful, no exception will be raised.
+    response.raise_for_status()
+
+    last_stable_file = f"{last_stable_commit_hash}_{BACKPORTS_REQUIRED_FILE}"
+    temp_dir = tempfile.mkdtemp()
+    with open(os.path.join(temp_dir, last_stable_file), "w") as fileh:
+        fileh.write(response.text)
+
+    backports_required_last_stable = generate_resmoke.read_yaml(temp_dir, last_stable_file)
+    return backports_required_last_stable[suite_name]
+
+
 class MultiversionConfig(object):
     """An object containing the configurations to generate and run the multiversion tests with."""
 
@@ -170,6 +216,7 @@ class EvergreenConfigGenerator(object):
         return self.evg_config
 
     def generate_evg_tasks(self, burn_in_test=None, burn_in_idx=0):
+        # pylint: disable=too-many-locals
         """
         Generate evergreen tasks for multiversion tests.
 
@@ -178,7 +225,6 @@ class EvergreenConfigGenerator(object):
 
         :param burn_in_test: The test to be run as part of the burn in multiversion suite.
         """
-        idx = 0
         # Divide tests into suites based on run-time statistics for the last
         # LOOKBACK_DURATION_DAYS. Tests without enough run-time statistics will be placed
         # in the misc suite.
@@ -186,6 +232,9 @@ class EvergreenConfigGenerator(object):
         end_date = datetime.datetime.utcnow().replace(microsecond=0)
         start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
         suites = gen_suites.calculate_suites(start_date, end_date)
+        # Update the base suite names to the multiversion task names.
+        for suite in suites:
+            suite.source_name = self.task
         # Render the given suites into yml files that can be used by resmoke.py.
         if is_suite_sharded(TEST_SUITE_DIR, self.options.suite):
             config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
@@ -195,6 +244,11 @@ class EvergreenConfigGenerator(object):
                                         REPL_MIXED_VERSION_CONFIGS)
         config_file_dict = generate_resmoke.render_suite_files(
             suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
+        # Update the base misc suite name to the multiversion name.
+        base_misc_file = f"{self.options.suite}_misc.yml"
+        misc_suite_name = f"{self.task}_misc.yml"
+        misc_suite = os.path.join(CONFIG_DIR, misc_suite_name)
+        config_file_dict[misc_suite_name] = config_file_dict.pop(base_misc_file)
         generate_resmoke.write_file_dict(CONFIG_DIR, config_file_dict)
 
         if burn_in_test is not None:
@@ -205,6 +259,7 @@ class EvergreenConfigGenerator(object):
             return self.evg_config
 
         for version_config in config.version_configs:
+            idx = 0
             for suite in suites:
                 # Generate the newly divided test suites
                 source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
@@ -213,8 +268,7 @@ class EvergreenConfigGenerator(object):
 
             # Also generate the misc task.
             misc_suite_name = "{0}_misc".format(self.options.suite)
-            source_suite = os.path.join(CONFIG_DIR, misc_suite_name + ".yml")
-            self._generate_sub_task(version_config, self.task, idx, source_suite, 1)
+            self._generate_sub_task(version_config, self.task, idx, misc_suite, 1)
             idx += 1
         self.create_display_task(self.task, self.task_specs, self.task_names)
         return self.evg_config
@@ -228,12 +282,18 @@ class EvergreenConfigGenerator(object):
         self._write_evergreen_config_to_file(self.task)
 
 
-@click.command()
+@click.group()
+def main():
+    """Serve as an entry point for the 'run' and 'generate-exclude-files' commands."""
+    pass
+
+
+@main.command("run")
 @click.option("--expansion-file", type=str, required=True,
               help="Location of expansions file generated by evergreen.")
 @click.option("--evergreen-config", type=str, default=CONFIG_FILE,
               help="Location of evergreen configuration file.")
-def main(expansion_file, evergreen_config=None):
+def run_generate_tasks(expansion_file, evergreen_config=None):
     """
     Create a configuration for generate tasks to create sub suites for the specified resmoke suite.
 
@@ -256,5 +316,78 @@ def main(expansion_file, evergreen_config=None):
     config_generator.run()
 
 
+@main.command("generate-exclude-files")
+@click.option("--suite", type=str, required=True,
+              help="The multiversion suite to generate the exclude_files yaml for.")
+@click.option("--task-path-suffix", type=str, required=True,
+              help="The directory in which multiversion binaries are stored.")
+@click.option("--is-generated-suite", type=bool, required=True,
+              help="Indicates whether the suite yaml to update is generated or static.")
+def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite):
+    """
+    Update the given multiversion suite configuration yaml to exclude tests.
+
+    Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
+    last-stable branch to determine which tests should be blacklisted.
+    """
+
+    suite_name = generate_resmoke.remove_gen_suffix(suite)
+
+    # Get the backports_required_for_multiversion_tests.yml on the current version branch.
+    backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
+    latest_suite_yaml = backports_required_latest[suite_name]
+
+    if not latest_suite_yaml:
+        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
+        return
+
+    # Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable
+    # binary we are running tests against. We do this by using the commit hash from the last-stable
+    # mongo shell executable.
+    last_stable_commit_hash = get_backports_required_last_stable_hash(task_path_suffix)
+
+    # Get the yaml contents under the 'suite_name' key from the last-stable commit.
+    last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash, suite_name)
+    if last_stable_suite_yaml is None:
+        files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml)
+    else:
+        files_to_exclude = set(
+            elem["test_file"] for elem in latest_suite_yaml if elem not in last_stable_suite_yaml)
+
+    if not files_to_exclude:
+        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
+        return
+
+    suite_yaml_dict = {}
+
+    if not is_generated_suite:
+        # Populate the config values to get the resmoke config directory.
+        buildscripts.resmokelib.parser.set_options()
+        suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
+
+        # Update the static suite config with the excluded files and write to disk.
+        file_name = f"{suite_name}.yml"
+        suite_config = generate_resmoke.read_yaml(suites_dir, file_name)
+        suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
+            suite_config, file_name, excludes=list(files_to_exclude))
+    else:
+        # We expect the generated suites to already have been generated in the generated config
+        # directory.
+        for file_name in os.listdir(CONFIG_DIR):
+            suites_dir = CONFIG_DIR
+            # Update the 'exclude_files' for each of the appropriate generated suites.
+            if suite_name in file_name and file_name.endswith('.yml'):
+                suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name)
+                selected_files = suite_config["selector"]["roots"]
+                # Only exclude the files that we want to exclude in the first place and have been
+                # selected to run as part of the generated suite yml.
+                intersection = [test for test in selected_files if test in files_to_exclude]
+                if not intersection:
+                    continue
+                suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
+                    suite_config, file_name, excludes=list(intersection))
+    generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
+
+
 if __name__ == '__main__':
     main()  # pylint: disable=no-value-for-parameter