SERVER-48048 Use resmoke tag files for multiversion blacklisting

13 files changed, 322 insertions, 219 deletions

diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py
index 613153e13de..ad59089d46b 100644
--- a/buildscripts/burn_in_tests_multiversion.py
+++ b/buildscripts/burn_in_tests_multiversion.py
@@ -74,16 +74,10 @@ def create_multiversion_generate_tasks_config(tests_by_task: Dict, evg_api: Ever
                 evg_api, gen_resmoke.ConfigOptions(config_options))
             test_list = tests_by_task[suite["origin"]]["tests"]
             for test in test_list:
-                # Exclude files that should be blacklisted from multiversion testing.
-                files_to_exclude = gen_multiversion.get_exclude_files(suite["multiversion_name"],
-                                                                      TASK_PATH_SUFFIX)
-                LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test,
-                             suite=suite["multiversion_name"])
-                if test not in files_to_exclude:
-                    # Generate the multiversion tasks for each test.
-                    sub_tasks = config_generator.get_burn_in_tasks(test, idx)
-                    tasks = tasks.union(sub_tasks)
-                    idx += 1
+                # Generate the multiversion tasks for each test.
+                sub_tasks = config_generator.get_burn_in_tasks(test, idx)
+                tasks = tasks.union(sub_tasks)
+                idx += 1
 
     existing_tasks = {ExistingTask(f"{BURN_IN_MULTIVERSION_TASK}_gen")}
     build_variant.display_task(BURN_IN_MULTIVERSION_TASK, tasks,
diff --git a/buildscripts/ciconfig/tags.py b/buildscripts/ciconfig/tags.py
index 62ee15db0ed..2908fa60685 100644
--- a/buildscripts/ciconfig/tags.py
+++ b/buildscripts/ciconfig/tags.py
@@ -6,6 +6,8 @@ from functools import cmp_to_key
 import textwrap
 import yaml
 
+from ..resmokelib.utils import default_if_none
+
 
 # Setup to preserve order in yaml.dump, see https://stackoverflow.com/a/8661021
 def _represent_dict_order(self, data):
@@ -20,13 +22,14 @@ yaml.add_representer(collections.OrderedDict, _represent_dict_order)
 class TagsConfig(object):
     """Represent a test tag configuration file."""
 
-    def __init__(self, raw, cmp_func=None):
+    def __init__(self, raw=None, cmp_func=None):
         """Initialize a TagsConfig from a dict representing the associations between tests and tags.
 
+        'raw' is a dict containing a 'selector' key, whose value is a dict mapping test kinds to tests to tags.
         'cmp_func' can be used to specify a comparison function that will be used when sorting tags.
         """
 
-        self.raw = raw
+        self.raw = default_if_none(raw, {"selector": {}})
         self._conf = self.raw["selector"]
         self._conf_copy = copy.deepcopy(self._conf)
         self._cmp_func = cmp_func
diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index 888b3c00c90..51723f2c5c1 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -8,6 +8,7 @@ import re
 import sys
 import tempfile
 from typing import Optional, List, Set
+from collections import defaultdict
 
 from subprocess import check_output
 
@@ -26,6 +27,7 @@ from buildscripts.util.fileops import write_file_to_dir
 import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
 from buildscripts.evergreen_generate_resmoke_tasks import Suite, ConfigOptions
 import buildscripts.evergreen_gen_fuzzer_tests as gen_fuzzer
+import buildscripts.ciconfig.tags as _tags
 
 # pylint: disable=len-as-condition
 
@@ -49,7 +51,9 @@ BURN_IN_TASK = "burn_in_tests_multiversion"
 MULTIVERSION_CONFIG_KEY = "use_in_multiversion"
 PASSTHROUGH_TAG = "multiversion_passthrough"
 RANDOM_REPLSETS_TAG = "random_multiversion_ds"
-EXCLUDE_TAGS = f"{REQUIRES_FCV_TAG},multiversion_incompatible"
+BACKPORT_REQUIRED_TAG = "backport_required_multiversion"
+EXCLUDE_TAGS = f"{REQUIRES_FCV_TAG},multiversion_incompatible,{BACKPORT_REQUIRED_TAG}"
+EXCLUDE_TAGS_FILE = "multiversion_exclude_tags.yml"
 
 # The directory in which BACKPORTS_REQUIRED_FILE resides.
 ETC_DIR = "etc"
@@ -108,7 +112,7 @@ def get_backports_required_last_lts_hash(task_path_suffix: str):
     raise ValueError("Could not find a valid commit hash from the last-lts mongo binary.")
 
 
-def get_last_lts_yaml(last_lts_commit_hash, suite_name):
+def get_last_lts_yaml(last_lts_commit_hash):
     """Download BACKPORTS_REQUIRED_FILE from the last LTS commit and return the yaml."""
     LOGGER.info(f"Downloading file from commit hash of last-lts branch {last_lts_commit_hash}")
     response = requests.get(
@@ -122,41 +126,15 @@ def get_last_lts_yaml(last_lts_commit_hash, suite_name):
         fileh.write(response.text)
 
     backports_required_last_lts = generate_resmoke.read_yaml(temp_dir, last_lts_file)
-    return backports_required_last_lts[suite_name]
-
-
-def get_exclude_files(suite_name, task_path_suffix):
-    """Generate the list of files to exclude based on the BACKPORTS_REQUIRED_FILE."""
-    backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
-    if suite_name not in backports_required_latest:
-        LOGGER.info(f"Generating exclude files not supported for '{suite_name}''.")
-        return set()
-
-    latest_suite_yaml = backports_required_latest[suite_name]
-
-    if not latest_suite_yaml:
-        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
-        return set()
-
-    # Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts
-    # binary we are running tests against. We do this by using the commit hash from the last-lts
-    # mongo shell executable.
-    last_lts_commit_hash = get_backports_required_last_lts_hash(task_path_suffix)
-
-    # Get the yaml contents under the 'suite_name' key from the last-lts commit.
-    last_lts_suite_yaml = get_last_lts_yaml(last_lts_commit_hash, suite_name)
-    if last_lts_suite_yaml is None:
-        return set(elem["test_file"] for elem in latest_suite_yaml)
-    else:
-        return set(
-            elem["test_file"] for elem in latest_suite_yaml if elem not in last_lts_suite_yaml)
+    return backports_required_last_lts
 
 
 def _generate_resmoke_args(suite_file: str, mixed_version_config: str, is_sharded: bool, options,
                            burn_in_test: Optional[str]) -> str:
-    return (f"{options.resmoke_args} --suite={suite_file} --mixedBinVersions={mixed_version_config}"
-            f" --excludeWithAnyTags={EXCLUDE_TAGS} --originSuite={options.suite} "
-            f" {get_multiversion_resmoke_args(is_sharded)} {burn_in_test if burn_in_test else ''}")
+    return (
+        f"{options.resmoke_args} --suite={suite_file} --mixedBinVersions={mixed_version_config}"
+        f" --excludeWithAnyTags={EXCLUDE_TAGS},{generate_resmoke.remove_gen_suffix(options.task)}_{BACKPORT_REQUIRED_TAG} --tagFile={os.path.join(CONFIG_DIR, EXCLUDE_TAGS_FILE)} --originSuite={options.suite} "
+        f" {get_multiversion_resmoke_args(is_sharded)} {burn_in_test if burn_in_test else ''}")
 
 
 class EvergreenMultiversionConfigGenerator(object):
@@ -368,7 +346,7 @@ class EvergreenMultiversionConfigGenerator(object):
 
 @click.group()
 def main():
-    """Serve as an entry point for the 'run' and 'generate-exclude-files' commands."""
+    """Serve as an entry point for the 'run' and 'generate-exclude-tags' commands."""
     pass
 
 
@@ -398,17 +376,15 @@ def run_generate_tasks(expansion_file: str, evergreen_config: Optional[str] = No
     config_generator.run()
 
 
-@main.command("generate-exclude-files")
-@click.option("--suite", type=str, required=True,
-              help="The multiversion suite to generate the exclude_files yaml for.")
+@main.command("generate-exclude-tags")
 @click.option("--task-path-suffix", type=str, required=True,
               help="The directory in which multiversion binaries are stored.")
-@click.option("--is-generated-suite", type=bool, required=True,
-              help="Indicates whether the suite yaml to update is generated or static.")
-def generate_exclude_yaml(suite: str, task_path_suffix: str, is_generated_suite: bool) -> None:
+@click.option("--output", type=str, default=os.path.join(CONFIG_DIR, EXCLUDE_TAGS_FILE),
+              show_default=True, help="Where to output the generated tags.")
+def generate_exclude_yaml(task_path_suffix: str, output: str) -> None:
     # pylint: disable=too-many-locals
     """
-    Update the given multiversion suite configuration yaml to exclude tests.
+    Create a tag file associating multiversion tests to tags for exclusion.
 
     Compares the BACKPORTS_REQUIRED_FILE on the current branch with the same file on the
     last-lts branch to determine which tests should be blacklisted.
@@ -416,57 +392,47 @@ def generate_exclude_yaml(suite: str, task_path_suffix: str, is_generated_suite:
 
     enable_logging()
 
-    suite_name = generate_resmoke.remove_gen_suffix(suite)
-
-    files_to_exclude = get_exclude_files(suite_name, task_path_suffix)
+    backports_required_latest = generate_resmoke.read_yaml(ETC_DIR, BACKPORTS_REQUIRED_FILE)
 
-    if not files_to_exclude:
-        LOGGER.info(f"No tests need to be excluded from suite '{suite_name}'.")
-        return
+    # Get the state of the backports_required_for_multiversion_tests.yml file for the last-lts
+    # binary we are running tests against. We do this by using the commit hash from the last-lts
+    # mongo shell executable.
+    last_lts_commit_hash = get_backports_required_last_lts_hash(task_path_suffix)
 
-    suite_yaml_dict = {}
+    # Get the yaml contents from the last-lts commit.
+    backports_required_last_lts = get_last_lts_yaml(last_lts_commit_hash)
 
-    if not is_generated_suite:
-        # Populate the config values to get the resmoke config directory.
-        buildscripts.resmokelib.parser.set_run_options()
-        suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
+    def diff(list1, list2):
+        return [elem for elem in (list1 or []) if elem not in (list2 or [])]
 
-        # Update the static suite config with the excluded files and write to disk.
-        file_name = f"{suite_name}.yml"
-        suite_config = generate_resmoke.read_yaml(suites_dir, file_name)
-        suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
-            suite_config, file_name, excludes=list(files_to_exclude))
+    suites_latest = backports_required_latest["suites"] or {}
+    # Check if the changed syntax for etc/backports_required_multiversion.yml has been backported.
+    # This variable and all branches where it's not set can be deleted after backporting the change.
+    change_backported = "all" in backports_required_last_lts.keys()
+    if change_backported:
+        always_exclude = diff(backports_required_latest["all"], backports_required_last_lts["all"])
+        suites_last_lts: defaultdict = defaultdict(list, backports_required_last_lts["suites"])
     else:
-        if not os.path.exists(CONFIG_DIR) or len(os.listdir(CONFIG_DIR)) == 0:
-            LOGGER.info(
-                f"No configuration files exist in '{CONFIG_DIR}'. Skipping exclude file generation")
-            return
-
-        # We expect the generated suites to already have been generated in the generated config
-        # directory.
-        suites_dir = CONFIG_DIR
-        for file_name in os.listdir(suites_dir):
-            # Update the 'exclude_files' for each of the appropriate generated suites.
-            if file_name.endswith('misc.yml'):
-                # New tests will be run as part of misc.yml. We want to make sure to properly
-                # exclude these tests if they have been blacklisted.
-                suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name)
-                exclude_files = suite_config["selector"]["exclude_files"]
-                add_to_excludes = [test for test in files_to_exclude if test not in exclude_files]
-                exclude_files += add_to_excludes
-                suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
-                    suite_config, file_name, excludes=list(exclude_files))
-            elif file_name.endswith('.yml'):
-                suite_config = generate_resmoke.read_yaml(CONFIG_DIR, file_name)
-                selected_files = suite_config["selector"]["roots"]
-                # Only exclude the files that we want to exclude in the first place and have been
-                # selected to run as part of the generated suite yml.
-                intersection = [test for test in selected_files if test in files_to_exclude]
-                if not intersection:
-                    continue
-                suite_yaml_dict[file_name] = generate_resmoke.generate_resmoke_suite_config(
-                    suite_config, file_name, excludes=list(intersection))
-    generate_resmoke.write_file_dict(suites_dir, suite_yaml_dict)
+        always_exclude = backports_required_latest["all"] or []
+        suites_last_lts = defaultdict(list, backports_required_last_lts)
+
+    tags = _tags.TagsConfig()
+
+    # Tag tests that are excluded from every suite.
+    for elem in always_exclude:
+        tags.add_tag("js_test", elem["test_file"], BACKPORT_REQUIRED_TAG)
+
+    # Tag tests that are excluded on a suite-by-suite basis.
+    for suite in suites_latest.keys():
+        test_set = set()
+        for elem in diff(suites_latest[suite], suites_last_lts[suite]):
+            test_set.add(elem["test_file"])
+        for test in test_set:
+            tags.add_tag("js_test", test, f"{suite}_{BACKPORT_REQUIRED_TAG}")
+
+    LOGGER.info(f"Writing exclude tags to {output}.")
+    tags.write_file(filename=output,
+                    preamble="Tag file that specifies exclusions from multiversion suites.")
 
 
 if __name__ == '__main__':
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
index 86764bcde91..7326444b9d6 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
@@ -6,6 +6,8 @@ selector:
   exclude_with_any_tags:
   - multiversion_incompatible
   - requires_fcv_47
+  - backport_required_multiversion
+  - replica_sets_multiversion_backport_required_multiversion
   exclude_files:
   - jstests/replsets/initial_sync_rename_collection.js
   - jstests/replsets/initial_sync_drop_collection.js
@@ -14,6 +16,11 @@ selector:
   - jstests/replsets/invalidate_sessions_on_stepdown.js
   - jstests/replsets/rollback_via_refetch_anomaly.js
   - jstests/replsets/initial_sync_fails_unclean_restart.js
+
+  # This tag file can be created using ./buildscripts/evergreen_gen_multiversion_tests.py
+  # or downloaded from the evergreen task.
+  tag_file: multiversion_exclude_tags.yml
+
 executor:
   config:
     shell_options:
diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
index 9f0bcf755de..c679643cb00 100644
--- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
@@ -8,6 +8,16 @@ selector:
   exclude_with_any_tags:
   - multiversion_incompatible
   - requires_fcv_47
+  - backport_required_multiversion
+  - replica_sets_multiversion_backport_required_multiversion
+  exclude_files:
+  # TODO SERVER-46040: Enable when SERVER-46040 is backported
+  - jstests/sharding/drop_indexes_with_stale_config_error.js
+
+  # This tag file can be created using ./buildscripts/evergreen_gen_multiversion_tests.py
+  # or downloaded from the evergreen task.
+  tag_file: multiversion_exclude_tags.yml
+
 executor:
   config:
     shell_options:
diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py
index 115229e23a6..f789c24b049 100644
--- a/buildscripts/resmokelib/selector.py
+++ b/buildscripts/resmokelib/selector.py
@@ -114,15 +114,16 @@ class TestFileExplorer(object):
         return program.returncode, stdout.decode("utf-8"), stderr.decode("utf-8")
 
     @staticmethod
-    def parse_tag_file(test_kind):
+    def parse_tag_file(test_kind, tag_file=None, tagged_tests=None):
         """Parse the tag file and return a dict of tagged tests.
 
         The resulting dict will have as a key the filename and the
         value a list of tags, i.e., {'file1.js': ['tag1', 'tag2'], 'file2.js': ['tag2', 'tag3']}.
         """
-        tagged_tests = collections.defaultdict(list)
-        if config.TAG_FILE:
-            tags_conf = _tags.TagsConfig.from_file(config.TAG_FILE)
+        if tagged_tests is None:
+            tagged_tests = collections.defaultdict(list)
+        if tag_file and os.path.exists(tag_file):
+            tags_conf = _tags.TagsConfig.from_file(tag_file)
             tagged_roots = tags_conf.get_test_patterns(test_kind)
             for tagged_root in tagged_roots:
                 # Multiple tests could be returned for a set of tags.
@@ -336,7 +337,8 @@ class _SelectorConfig(object):
 
     def __init__(  # pylint: disable=too-many-arguments
             self, root=None, roots=None, include_files=None, exclude_files=None, include_tags=None,
-            exclude_tags=None, include_with_any_tags=None, exclude_with_any_tags=None):
+            exclude_tags=None, include_with_any_tags=None, exclude_with_any_tags=None,
+            tag_file=None):
         """Initialize the _SelectorConfig from the configuration elements.
 
         Args:
@@ -350,6 +352,7 @@ class _SelectorConfig(object):
                 selected tests must not match. Incompatible with 'include_tags'.
             include_with_any_tags: a list of tags. All selected tests must have at least one them.
             exclude_with_any_tags: a list of tags. No selected tests can have any of them.
+            tag_file: filename of a tag file associating tests to tags.
         """
         # Incompatible arguments check.
         if root and roots:
@@ -358,6 +361,7 @@ class _SelectorConfig(object):
             raise ValueError("include_tags and exclude_tags cannot be specified at the same time")
         self.root = root
         self.roots = roots
+        self.tag_file = tag_file
         self.include_files = utils.default_if_none(include_files, [])
         self.exclude_files = utils.default_if_none(exclude_files, [])
         include_with_any_tags = self.__merge_lists(include_with_any_tags,
@@ -457,12 +461,12 @@ class _JSTestSelectorConfig(_SelectorConfig):
 
     def __init__(  # pylint: disable=too-many-arguments
             self, roots=None, include_files=None, exclude_files=None, include_with_any_tags=None,
-            exclude_with_any_tags=None, include_tags=None, exclude_tags=None):
-        _SelectorConfig.__init__(self, roots=roots, include_files=include_files,
-                                 exclude_files=exclude_files,
-                                 include_with_any_tags=include_with_any_tags,
-                                 exclude_with_any_tags=exclude_with_any_tags,
-                                 include_tags=include_tags, exclude_tags=exclude_tags)
+            exclude_with_any_tags=None, include_tags=None, exclude_tags=None, tag_file=None):
+        _SelectorConfig.__init__(
+            self, roots=roots, include_files=include_files, exclude_files=exclude_files,
+            include_with_any_tags=include_with_any_tags,
+            exclude_with_any_tags=exclude_with_any_tags, include_tags=include_tags,
+            exclude_tags=exclude_tags, tag_file=tag_file)
 
 
 class _JSTestSelector(_Selector):
@@ -470,7 +474,12 @@ class _JSTestSelector(_Selector):
 
     def __init__(self, test_file_explorer):
         _Selector.__init__(self, test_file_explorer)
-        self._tags = self._test_file_explorer.parse_tag_file("js_test")
+        self._tags = self._test_file_explorer.parse_tag_file("js_test", config.TAG_FILE)
+
+    def select(self, selector_config):
+        self._tags = self._test_file_explorer.parse_tag_file("js_test", selector_config.tag_file,
+                                                             self._tags)
+        return _Selector.select(self, selector_config)
 
     def get_tags(self, test_file):
         """Return tags from test_file."""
diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py
index f5d77acee77..09f3722643a 100644
--- a/buildscripts/tests/resmokelib/test_selector.py
+++ b/buildscripts/tests/resmokelib/test_selector.py
@@ -4,6 +4,7 @@ import fnmatch
 import os.path
 import sys
 import unittest
+import collections
 
 import buildscripts.resmokelib.config
 import buildscripts.resmokelib.parser as parser
@@ -12,6 +13,8 @@ import buildscripts.resmokelib.utils.globstar as globstar
 
 # pylint: disable=missing-docstring,protected-access
 
+FIXTURE_PREFIX = "buildscripts/tests/selftest_fixtures"
+
 
 class TestExpressions(unittest.TestCase):
     """Unit tests for the tag matching expressions."""
@@ -104,6 +107,26 @@ class TestTestFileExplorer(unittest.TestCase):
         self.assertTrue(self.test_file_explorer.fnmatchcase("directory/file.js", pattern))
         self.assertFalse(self.test_file_explorer.fnmatchcase("other/file.js", pattern))
 
+    def test_parse_tag_file(self):
+        tests = (os.path.join(FIXTURE_PREFIX, "one.js"), os.path.join(FIXTURE_PREFIX, "two.js"),
+                 os.path.join(FIXTURE_PREFIX, "three.js"))
+        expected = collections.defaultdict(list)
+        expected[tests[0]] = ["tag1", "tag2", "tag3"]
+        expected[tests[1]] = ["tag1", "tag2"]
+
+        tags = self.test_file_explorer.parse_tag_file("js_test",
+                                                      os.path.join(FIXTURE_PREFIX, "tag_file1.yml"))
+        # defaultdict isn't == comparable
+        for test in tests:
+            self.assertEqual(tags[test], expected[test])
+
+        expected[tests[1]] = ["tag1", "tag2", "tag4"]
+        tags = self.test_file_explorer.parse_tag_file("js_test",
+                                                      os.path.join(FIXTURE_PREFIX, "tag_file2.yml"),
+                                                      tags)
+        for test in tests:
+            self.assertEqual(tags[test], expected[test])
+
 
 class MockTestFileExplorer(object):
     """Component giving access to mock test files data."""
@@ -150,7 +173,7 @@ class MockTestFileExplorer(object):
     def list_dbtests(self, binary):  # pylint: disable=no-self-use,unused-argument
         return ["dbtestA", "dbtestB", "dbtestC"]
 
-    def parse_tag_file(self, test_kind):
+    def parse_tag_file(self, test_kind, tag_file=None, tagged_tests=None):  # pylint: disable=unused-argument
         if test_kind == "js_test":
             return self.jstest_tag_file
         return None
diff --git a/buildscripts/tests/selftest_fixtures/one.js b/buildscripts/tests/selftest_fixtures/one.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/one.js
diff --git a/buildscripts/tests/selftest_fixtures/tag_file1.yml b/buildscripts/tests/selftest_fixtures/tag_file1.yml
new file mode 100644
index 00000000000..043ce804256
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/tag_file1.yml
@@ -0,0 +1,9 @@
+selector:
+  js_test:
+    buildscripts/tests/selftest_fixtures/one.js:
+    - tag1
+    - tag2
+    - tag3
+    buildscripts/tests/selftest_fixtures/two.js:
+    - tag1
+    - tag2
diff --git a/buildscripts/tests/selftest_fixtures/tag_file2.yml b/buildscripts/tests/selftest_fixtures/tag_file2.yml
new file mode 100644
index 00000000000..381f64390a7
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/tag_file2.yml
@@ -0,0 +1,4 @@
+selector:
+  js_test:
+    buildscripts/tests/selftest_fixtures/two.js:
+    - tag4
diff --git a/buildscripts/tests/selftest_fixtures/three.js b/buildscripts/tests/selftest_fixtures/three.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/three.js
diff --git a/buildscripts/tests/selftest_fixtures/two.js b/buildscripts/tests/selftest_fixtures/two.js
new file mode 100644
index 00000000000..e69de29bb2d
--- /dev/null
+++ b/buildscripts/tests/selftest_fixtures/two.js
diff --git a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
index 4fdba6e44f9..662ba311d4d 100644
--- a/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
+++ b/buildscripts/tests/test_evergreen_gen_multiversion_tests.py
@@ -14,7 +14,7 @@ from buildscripts import evergreen_gen_multiversion_tests as under_test
 from buildscripts.evergreen_generate_resmoke_tasks import read_yaml
 import buildscripts.evergreen_generate_resmoke_tasks as generate_resmoke
 
-# pylint: disable=missing-docstring
+# pylint: disable=missing-docstring, no-self-use
 
 
 class TestRun(unittest.TestCase):
@@ -70,119 +70,197 @@ class TestRun(unittest.TestCase):
                 os.remove(evg_conf.name)
 
 
-class TestGenerateExcludeFiles(unittest.TestCase):
+class TestGenerateExcludeYaml(unittest.TestCase):
     def setUp(self):
         self._tmpdir = TemporaryDirectory()
-        under_test.CONFIG_DIR = self._tmpdir.name
 
     def tearDown(self):
         if self._tmpdir is not None:
             self._tmpdir.cleanup()
-        under_test.CONFIG_DIR = generate_resmoke.DEFAULT_CONFIG_VALUES["generated_config_dir"]
 
-    def test_missing_dir_okay(self):
-        self._tmpdir.cleanup()
-        self._tmpdir = None
-        self.assertFalse(os.path.exists(under_test.CONFIG_DIR))
-
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=test', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-
-    def test_empty_dir_okay(self):
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=test', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-        self.assertEqual(len(os.listdir(under_test.CONFIG_DIR)), 0)
-
-    @patch('buildscripts.evergreen_gen_multiversion_tests.get_exclude_files')
-    def test_adds_exclude_file(self, get_exclude_files):
-        get_exclude_files.return_value = set()
-        get_exclude_files.return_value.add('jstests/core/count_plan_summary.js')
-        with open(self._tmpdir.name + "/sharding_jscore_passthrough_00.yml", mode='w') as fh:
-            fh.write(CONF)
-
-        runner = CliRunner()
-        result = runner.invoke(
-            under_test.generate_exclude_yaml,
-            ['--suite=test', '--task-path-suffix=/data/multiversion', '--is-generated-suite=true'])
-        self.assertEqual(result.exit_code, 0, result)
-        self.assertEqual(get_exclude_files.call_count, 1)
-        new_conf = read_yaml(self._tmpdir.name, "sharding_jscore_passthrough_00.yml")
-        self.assertEqual(new_conf["selector"]["exclude_files"],
-                         ["jstests/core/count_plan_summary.js"])
-
-
-CONF = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
-# This file was generated by /data/mci/4d9a5adb3744dad3d44d93e5ff2a441f/src/buildscripts/evergreen_generate_resmoke_tasks.py from
-# sharded_collections_jscore_passthrough.
-executor:
-  archive:
-    hooks:
-    - CheckReplDBHash
-    - ValidateCollections
-  config:
-    shell_options:
-      eval: load("jstests/libs/override_methods/implicitly_shard_accessed_collections.js")
-      readMode: commands
-  fixture:
-    class: ShardedClusterFixture
-    enable_balancer: false
-    mongod_options:
-      set_parameters:
-        enableTestCommands: 1
-    mongos_options:
-      set_parameters:
-        enableTestCommands: 1
-    num_shards: 2
-  hooks:
-  - class: CheckReplDBHash
-  - class: ValidateCollections
-  - class: CleanEveryN
-    n: 20
-selector:
-  exclude_with_any_tags:
-  - assumes_against_mongod_not_mongos
-  - assumes_no_implicit_collection_creation_after_drop
-  - assumes_no_implicit_index_creation
-  - assumes_unsharded_collection
-  - cannot_create_unique_index_when_using_hashed_shard_key
-  - requires_profiling
-  roots:
-  - jstests/core/count_plan_summary.js
-  - jstests/core/json_schema/unique_items.js
-  - jstests/core/arrayfind9.js
-  - jstests/core/sort_numeric.js
-  - jstests/core/remove4.js
-  - jstests/core/aggregation_accepts_write_concern.js
-  - jstests/core/geo3.js
-  - jstests/core/minmax_edge.js
-  - jstests/core/json_schema/additional_properties.js
-  - jstests/core/update_min_max_examples.js
-  - jstests/core/where2.js
-  - jstests/core/geo_s2meridian.js
-  - jstests/core/bittest.js
-  - jstests/core/orj.js
-  - jstests/core/find_dedup.js
-  - jstests/core/hashed_index_queries.js
-  - jstests/core/geo_circle2a.js
-  - jstests/core/js4.js
-  - jstests/core/index_create_with_nul_in_name.js
-  - jstests/core/count_hint.js
-  - jstests/core/sorta.js
-  - jstests/core/orf.js
-  - jstests/core/geo_s2within.js
-  - jstests/core/json_schema/required.js
-  - jstests/core/pop_server_13516.js
-  - jstests/core/updatel.js
-  - jstests/core/geo_s2descindex.js
-test_kind: js_test
-use_in_multiversion: sharded_collections_jscore_multiversion_passthrough
-"""
+    def assert_contents(self, expected):
+        actual = read_yaml(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)
+        self.assertEqual(actual, expected)
+
+    def patch_and_run(self, latest, last_lts):
+        """
+        Helper to patch and run the test.
+        """
+        mock_multiversion_methods = {
+            'get_backports_required_last_lts_hash': MagicMock(),
+            'get_last_lts_yaml': MagicMock(return_value=last_lts)
+        }
+
+        with patch.multiple('buildscripts.evergreen_gen_multiversion_tests',
+                            **mock_multiversion_methods):
+            with patch('buildscripts.evergreen_generate_resmoke_tasks.read_yaml',
+                       return_value=latest) as mock_read_yaml:
+
+                output = os.path.join(self._tmpdir.name, under_test.EXCLUDE_TAGS_FILE)
+                runner = CliRunner()
+                result = runner.invoke(
+                    under_test.generate_exclude_yaml,
+                    [f"--output={output}", '--task-path-suffix=/data/multiversion'])
+
+                self.assertEqual(result.exit_code, 0, result)
+                mock_read_yaml.assert_called_once()
+                mock_multiversion_methods[
+                    'get_backports_required_last_lts_hash'].assert_called_once()
+                mock_multiversion_methods['get_last_lts_yaml'].assert_called_once()
+
+    def test_create_yaml_suite1(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {'jstests/fake_file1.js': ['suite1_backport_required_multiversion']}
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_create_yaml_suite1_and_suite2(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}],
+                'suite2': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': [
+                        'suite1_backport_required_multiversion',
+                        'suite2_backport_required_multiversion'
+                    ]
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_both_all_are_none(self):
+        latest_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {'jstests/fake_file1.js': ['suite1_backport_required_multiversion']}
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_old_all_is_none(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                           {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'all': None, 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': ['suite1_backport_required_multiversion'],
+                    'jstests/fake_file0.js': ['backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    def test_create_yaml_suite1_and_all(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'},
+                    {'ticket': 'fake_ticket4', 'test_file': 'jstests/fake_file4.js'}], 'suites': {
+                        'suite1': [{'ticket': 'fake_ticket1', 'test_file': 'jstests/fake_file1.js'},
+                                   {'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+                    }
+        }
+
+        last_lts_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+            }
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file1.js': ['suite1_backport_required_multiversion'],
+                    'jstests/fake_file4.js': ['backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
+    # Can delete after backporting the changed yml syntax.
+    def test_not_backported(self):
+        latest_yaml = {
+            'all': [{'ticket': 'fake_ticket0', 'test_file': 'jstests/fake_file0.js'}], 'suites': {
+                'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'},
+                           {'ticket': 'fake_ticket3', 'test_file': 'jstests/fake_file3.js'}]
+            }
+        }
+
+        last_lts_yaml = {
+            'suite1': [{'ticket': 'fake_ticket2', 'test_file': 'jstests/fake_file2.js'}]
+        }
+
+        expected = {
+            'selector': {
+                'js_test': {
+                    'jstests/fake_file0.js': ['backport_required_multiversion'],
+                    'jstests/fake_file3.js': ['suite1_backport_required_multiversion']
+                }
+            }
+        }
+
+        self.patch_and_run(latest_yaml, last_lts_yaml)
+        self.assert_contents(expected)
+
 
 EXPANSIONS = """task: t
 build_variant: bv