SERVER-45500 Fix backport exclude logic from multiversion tests for new test files

10 files changed, 73 insertions, 53 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index e1a4e0f03d3..25a86dda769 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -35,7 +35,7 @@ import buildscripts.evergreen_generate_resmoke_tasks as gen_resmoke
 from buildscripts.patch_builds.change_data import find_changed_files
 import buildscripts.resmokelib.parser
 from buildscripts.resmokelib.suitesconfig import create_test_membership_map, get_suites, \
-    get_named_suites_with_root_level_key_and_value
+    get_named_suites_with_root_level_key
 from buildscripts.resmokelib.utils import default_if_none, globstar
 from buildscripts.ciconfig.evergreen import parse_evergreen_file, ResmokeArgs, \
     EvergreenProjectConfig, VariantTask
@@ -72,9 +72,10 @@ SUPPORTED_TEST_KINDS = ("fsm_workload_test", "js_test", "json_schema_test",
 BURN_IN_TESTS_GEN_TASK = "burn_in_tests_gen"
 BURN_IN_TESTS_TASK = "burn_in_tests"
 
-MULTIVERSION_CONFIG_KEY = gen_multiversion.BURN_IN_CONFIG_KEY
+MULTIVERSION_CONFIG_KEY = gen_multiversion.MULTIVERSION_CONFIG_KEY
 MULTIVERSION_TAG = gen_multiversion.PASSTHROUGH_TAG
 BURN_IN_MULTIVERSION_TASK = gen_multiversion.BURN_IN_TASK
+TASK_PATH_SUFFIX = "/data/multiversion"
 
 
 class RepeatConfig(object):
@@ -615,38 +616,44 @@ def create_multiversion_generate_tasks_config(evg_config: Configuration, tests_b
     dt = DisplayTaskDefinition(BURN_IN_MULTIVERSION_TASK)
 
     if tests_by_task:
-        multiversion_suites = get_named_suites_with_root_level_key_and_value(
-            MULTIVERSION_CONFIG_KEY, True)
+        # Get the multiversion suites that will run in as part of burn_in_multiversion.
+        multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
         for suite in multiversion_suites:
             idx = 0
-            if suite not in tests_by_task.keys():
+            if suite["origin"] not in tests_by_task.keys():
                 # Only generate burn in multiversion tasks for suites that would run the detected
                 # changed tests.
                 continue
-            LOGGER.debug("Generating multiversion suite", suite=suite)
+            LOGGER.debug("Generating multiversion suite", suite=suite["multiversion_name"])
 
             # We hardcode the number of fallback sub suites and the target resmoke time here
             # since burn_in_tests cares about individual tests and not entire suites. The config
             # options here are purely used to generate the proper multiversion suites to run
             # tests against.
             config_options = {
-                "suite": suite,
+                "suite": suite["origin"],
                 "fallback_num_sub_suites": 1,
                 "project": generate_config.project,
                 "build_variant": generate_config.build_variant,
                 "task_id": generate_config.task_id,
-                "task_name": suite,
+                "task_name": suite["multiversion_name"],
                 "target_resmoke_time": 60,
             }
             config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
 
             config_generator = gen_multiversion.EvergreenConfigGenerator(
                 evg_api, evg_config, gen_resmoke.ConfigOptions(config_options))
-            test_list = tests_by_task[suite]["tests"]
+            test_list = tests_by_task[suite["origin"]]["tests"]
             for test in test_list:
-                # Generate the multiversion tasks for each test.
-                config_generator.generate_evg_tasks(test, idx)
-                idx += 1
+                # Exclude files that should be blacklisted from multiversion testing.
+                files_to_exclude = gen_multiversion.get_exclude_files(suite["multiversion_name"],
+                                                                      TASK_PATH_SUFFIX)
+                LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test,
+                             suite=suite["multiversion_name"])
+                if test not in files_to_exclude:
+                    # Generate the multiversion tasks for each test.
+                    config_generator.generate_evg_tasks(test, idx)
+                    idx += 1
             dt.execution_tasks(config_generator.task_names)
             evg_config.variant(generate_config.build_variant).tasks(config_generator.task_specs)
 
@@ -815,10 +822,9 @@ def burn_in(repeat_config: RepeatConfig, generate_config: GenerateConfig, resmok
             LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks,
                          tag=MULTIVERSION_TAG)
             # We expect the number of suites with MULTIVERSION_TAG to be the same as in
-            # multiversion_suites. Multiversion passthrough suites must include BURN_IN_CONFIG_KEY
-            # as a root level key and must be set to true.
-            multiversion_suites = get_named_suites_with_root_level_key_and_value(
-                MULTIVERSION_CONFIG_KEY, True)
+            # multiversion_suites. Multiversion passthrough suites must include
+            # MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
+            multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
             assert len(multiversion_tasks) == len(multiversion_suites)
         json_config = create_generate_tasks_file(tests_by_task, generate_config, repeat_config,
                                                  evg_api)
diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index 161dc0ad666..dc161a8351c 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -50,7 +50,7 @@ REPL_MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"]
 SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"]
 
 BURN_IN_TASK = "burn_in_tests_multiversion"
-BURN_IN_CONFIG_KEY = "use_in_multiversion_burn_in_tests"
+MULTIVERSION_CONFIG_KEY = "use_in_multiversion"
 PASSTHROUGH_TAG = "multiversion_passthrough"
 EXCLUDE_TAGS = f"{REQUIRES_FCV_TAG},multiversion_incompatible"
 
@@ -122,6 +122,33 @@ def get_last_stable_yaml(last_stable_commit_hash, suite_name):
     return backports_required_last_stable[suite_name]
 
 
+def get_exclude_files(suite_name, task_path_suffix):
+    """Generate the list of files to exclude based on the BACKPORTS_REQUIRED_FILE."""
+    backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
+    if suite_name not in backports_required_latest:
+        LOGGER.info(f"Generating exclude files not supported for '{suite_name}''.")
+        return set()
+
+    latest_suite_yaml = backports_required_latest[suite_name]
+
+    if not latest_suite_yaml:
+        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
+        return set()
+
+    # Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable
+    # binary we are running tests against. We do this by using the commit hash from the last-stable
+    # mongo shell executable.
+    last_stable_commit_hash = get_backports_required_last_stable_hash(task_path_suffix)
+
+    # Get the yaml contents under the 'suite_name' key from the last-stable commit.
+    last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash, suite_name)
+    if last_stable_suite_yaml is None:
+        return set(elem["test_file"] for elem in latest_suite_yaml)
+    else:
+        return set(
+            elem["test_file"] for elem in latest_suite_yaml if elem not in last_stable_suite_yaml)
+
+
 class EvergreenConfigGenerator(object):
     """Generate evergreen configurations for multiversion tests."""
 
@@ -342,30 +369,7 @@ def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite):
 
     suite_name = generate_resmoke.remove_gen_suffix(suite)
 
-    # Get the backports_required_for_multiversion_tests.yml on the current version branch.
-    backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
-    if suite_name not in backports_required_latest:
-        LOGGER.info(f"Generating exclude files not supported for '{suite_name}''.")
-        return
-
-    latest_suite_yaml = backports_required_latest[suite_name]
-
-    if not latest_suite_yaml:
-        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
-        return
-
-    # Get the state of the backports_required_for_multiversion_tests.yml file for the last-stable
-    # binary we are running tests against. We do this by using the commit hash from the last-stable
-    # mongo shell executable.
-    last_stable_commit_hash = get_backports_required_last_stable_hash(task_path_suffix)
-
-    # Get the yaml contents under the 'suite_name' key from the last-stable commit.
-    last_stable_suite_yaml = get_last_stable_yaml(last_stable_commit_hash, suite_name)
-    if last_stable_suite_yaml is None:
-        files_to_exclude = set(elem["test_file"] for elem in latest_suite_yaml)
-    else:
-        files_to_exclude = set(
-            elem["test_file"] for elem in latest_suite_yaml if elem not in last_stable_suite_yaml)
+    files_to_exclude = get_exclude_files(suite_name, task_path_suffix)
 
     if not files_to_exclude:
         LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
@@ -389,7 +393,16 @@ def generate_exclude_yaml(suite, task_path_suffix, is_generated_suite):
         for file_name in os.listdir(CONFIG_DIR):
             suites_dir = CONFIG_DIR
             # Update the 'exclude_files' for each of the appropriate generated suites.
-            if file_name.endswith('.yml'):
+            if file_name.endswith('misc.yml'):
+                # New tests will be run as part of misc.yml. We want to make sure to properly
+                # exclude these tests if they have been blacklisted.
+                suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name)
+                exclude_files = suite_config["selector"]["exclude_files"]
+                add_to_excludes = [test for test in files_to_exclude if test not in exclude_files]
+                exclude_files += add_to_excludes
+                suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
+                    suite_config, file_name, excludes=list(exclude_files))
+            elif file_name.endswith('.yml'):
                 suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name)
                 selected_files = suite_config["selector"]["roots"]
                 # Only exclude the files that we want to exclude in the first place and have been
diff --git a/buildscripts/resmokeconfig/suites/change_streams.yml b/buildscripts/resmokeconfig/suites/change_streams.yml
index 2e200ecd933..87dbb03a0e0 100644
--- a/buildscripts/resmokeconfig/suites/change_streams.yml
+++ b/buildscripts/resmokeconfig/suites/change_streams.yml
@@ -1,5 +1,5 @@
 test_kind: js_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: change_streams_multiversion
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml b/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml
index 1ba423ae07e..a737ec614bf 100644
--- a/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/change_streams_sharded_collections_passthrough.yml
@@ -1,5 +1,5 @@
 test_kind: js_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: change_streams_sharded_collections_multiversion_passthrough
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
index fed92a7bd2f..f6d2941b116 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_replication.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_replication.yml
@@ -1,5 +1,5 @@
 test_kind: fsm_workload_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: concurrency_replication_multiversion
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
index e089cda1cce..14c9052ad89 100644
--- a/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
+++ b/buildscripts/resmokeconfig/suites/concurrency_sharded_replication.yml
@@ -1,5 +1,5 @@
 test_kind: fsm_workload_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: concurrency_sharded_replication_multiversion
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
index 107ce408367..4173cd32c32 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_jscore_passthrough.yml
@@ -1,5 +1,5 @@
 test_kind: js_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: replica_sets_jscore_multiversion_passthrough
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
index 5c381534c62..4f4ebba5e8c 100644
--- a/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharded_collections_jscore_passthrough.yml
@@ -1,5 +1,5 @@
 test_kind: js_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: sharded_collections_jscore_multiversion_passthrough
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
index d2a74fcd86f..3662fee7850 100644
--- a/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_jscore_passthrough.yml
@@ -1,5 +1,5 @@
 test_kind: js_test
-use_in_multiversion_burn_in_tests: true
+use_in_multiversion: sharding_jscore_multiversion_passthrough
 
 selector:
   roots:
diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index 52a69391a04..ecfef265917 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -19,15 +19,16 @@ def get_named_suites():
     return names
 
 
-def get_named_suites_with_root_level_key_and_value(root_level_key, value):
-    """Return the suites that contain the given root_level_key with value equal to 'value'."""
+def get_named_suites_with_root_level_key(root_level_key):
+    """Return the suites that contain the given root_level_key and their values."""
     all_suite_names = get_named_suites()
     suites_to_return = []
 
     for suite in all_suite_names:
         suite_config = _get_suite_config(suite)
-        if root_level_key in suite_config.keys() and suite_config[root_level_key] == value:
-            suites_to_return.append(suite)
+        if root_level_key in suite_config.keys() and suite_config[root_level_key]:
+            suites_to_return.append(
+                {"origin": suite, "multiversion_name": suite_config[root_level_key]})
     return suites_to_return