SERVER-55857 remove NAMED_SUITES from resmoke config

26 files changed, 210 insertions, 161 deletions

diff --git a/.gitignore b/.gitignore
index acb29394d1c..7c2303af13a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -212,6 +212,9 @@ default.profraw
 /corpora
 /corpora-merged
 
+# Linter and formatter cache
+.mypy_cache/
+
 # RPM build temps
 /distsrc.tar
 /selinux/tmp
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index e7ca49ce25a..67a77fc7151 100755
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -351,7 +351,7 @@ def create_task_list_for_tests(changed_tests: Set[str], build_variant: str,
     if not exclude_tasks:
         exclude_tasks = []
 
-    suites = get_suites(suite_files=SUITE_FILES, test_files=changed_tests)
+    suites = get_suites(suite_names_or_paths=SUITE_FILES, test_files=changed_tests)
     LOGGER.debug("Found suites to run", suites=suites)
 
     tests_by_executor = create_executor_list(suites, exclude_suites)
diff --git a/buildscripts/burn_in_tests_multiversion.py b/buildscripts/burn_in_tests_multiversion.py
index 0092b5c4a73..779f060835b 100644
--- a/buildscripts/burn_in_tests_multiversion.py
+++ b/buildscripts/burn_in_tests_multiversion.py
@@ -18,12 +18,11 @@ from buildscripts.burn_in_tests import EVERGREEN_FILE, \
 from buildscripts.ciconfig.evergreen import parse_evergreen_file, EvergreenProjectConfig
 from buildscripts.evergreen_burn_in_tests import GenerateConfig, DEFAULT_PROJECT, EvergreenFileChangeDetector
 from buildscripts.task_generation.constants import CONFIG_FILE
-from buildscripts.resmokelib.suitesconfig import get_named_suites_with_root_level_key
+from buildscripts.resmokelib.suitesconfig import burn_in_multiversion_suites
 from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder
 from buildscripts.task_generation.gen_config import GenerationConfiguration
 from buildscripts.task_generation.generated_config import GeneratedConfiguration
 from buildscripts.task_generation.multiversion_util import MultiversionUtilService
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
 from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters
 from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \
     FallbackStrategy, round_robin_fallback
@@ -34,7 +33,6 @@ from buildscripts.util.cmdutils import enable_logging
 structlog.configure(logger_factory=LoggerFactory())
 LOGGER = structlog.getLogger(__name__)
 
-MULTIVERSION_CONFIG_KEY = "use_in_multiversion"
 MULTIVERSION_PASSTHROUGH_TAG = "multiversion_passthrough"
 BURN_IN_MULTIVERSION_TASK = "burn_in_tests_multiversion"
 DEFAULT_CONFIG_DIR = "generated_resmoke_config"
@@ -130,8 +128,7 @@ class MultiversionBurnInOrchestrator:
         tasks = set()
         if tests_by_task:
             # Get the multiversion suites that will run in as part of burn_in_multiversion.
-            multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
-            for suite in multiversion_suites:
+            for suite in burn_in_multiversion_suites():
                 task_name = suite["origin"]
                 if task_name not in tests_by_task.keys():
                     # Only generate burn in multiversion tasks for suites that would run the
@@ -264,8 +261,6 @@ def main(build_variant, run_build_variant, distro, project, generate_tasks_file,
         binder.bind(GenTaskOptions, gen_task_options)
         binder.bind(EvergreenApi, evg_api)
         binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file())
-        binder.bind(ResmokeProxyConfig,
-                    ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
         binder.bind(EvergreenFileChangeDetector, EvergreenFileChangeDetector(task_id, evg_api))
         binder.bind(BurnInConfig, burn_in_config)
 
diff --git a/buildscripts/evergreen_gen_build_variant.py b/buildscripts/evergreen_gen_build_variant.py
index cd383b62e87..5625e420ccc 100644
--- a/buildscripts/evergreen_gen_build_variant.py
+++ b/buildscripts/evergreen_gen_build_variant.py
@@ -22,7 +22,6 @@ from buildscripts.task_generation.gen_config import GenerationConfiguration
 from buildscripts.task_generation.gen_task_validation import GenTaskValidationService
 from buildscripts.task_generation.multiversion_util import MultiversionUtilService, \
     SHARDED_MIXED_VERSION_CONFIGS, REPL_MIXED_VERSION_CONFIGS
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
 from buildscripts.task_generation.suite_split import SuiteSplitConfig, SuiteSplitParameters
 from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \
     greedy_division, round_robin_fallback
@@ -453,8 +452,6 @@ def main(expansion_file: str, evg_api_config: str, evg_project_config: str, outp
         binder.bind(EvergreenApi, RetryingEvergreenApi.get_api(config_file=evg_api_config))
         binder.bind(EvergreenProjectConfig, parse_evergreen_file(evg_project_config))
         binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file())
-        binder.bind(ResmokeProxyConfig,
-                    ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
 
     inject.configure(dependencies)
 
diff --git a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml
index cd070f88195..ee8d51b7c23 100644
--- a/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml
+++ b/buildscripts/resmokeconfig/matrix_suites/mappings/multiversion.yml
@@ -1,33 +1,41 @@
 # TODO SERVER-55857: check for history conflicts in overrides outside of explicitly set keys.
 # TODO SERVER-55857: use the generated suite if --originSuite is used (to indicate that a generated suite exists).
-# TODO SERVER-55857: ensure listSuites work with matrix suites.
-- suite_name: change_streams_multiversion_passthrough
+# TODO: move sharding and replica sets multiversion
+# TODO: Remove use_in_multiversion used by burn in multiversion and run all tasks from here instead.
+- suite_name: change_streams_multiversion
   base_suite: change_streams
   overrides:
     - "multiversion.replica_fixture"
-- suite_name: change_streams_sharded_collections_multiversion_passthrough
+    - "multiversion.test_kind"
+- suite_name: change_streams_sharded_collections_multiversion
   base_suite: change_streams_sharded_collections_passthrough
   overrides:
     - "multiversion.sharded_fixture"
-- suite_name: concurrency_replication_multiversion_passthrough
+    - "multiversion.test_kind"
+- suite_name: concurrency_replication_multiversion
   base_suite: concurrency_replication
   overrides:
     - "multiversion.replica_fixture"
-- suite_name: multiversion_sanity_check_passthrough
+    - "multiversion.test_kind"
+- suite_name: multiversion_sanity_check
   base_suite: replica_sets_jscore_passthrough
   overrides:
     - "multiversion.replica_fixture"
     - "multiversion.multiversion_sanity_check_selector"
-- suite_name: concurrency_sharded_replication_multiversion_passthrough
+    - "multiversion.test_kind"
+- suite_name: concurrency_sharded_replication_multiversion
   base_suite: concurrency_sharded_replication
   overrides:
     - "multiversion.sharded_fixture"
     - "multiversion.concurrency_sharded_replication_multiversion_mongod_options"
-- suite_name: sharding_jscore_multiversion_passthrough
+    - "multiversion.test_kind"
+- suite_name: sharding_jscore_multiversion
   base_suite: sharding_jscore_passthrough
   overrides:
     - "multiversion.sharded_fixture"
-- suite_name: sharded_collections_jscore_multiversion_passthrough
+    - "multiversion.test_kind"
+- suite_name: sharded_collections_jscore_multiversion
   base_suite: sharded_collections_jscore_passthrough
   overrides:
     - "multiversion.sharded_fixture"
+    - "multiversion.test_kind"
diff --git a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml
index ab316474592..0fedfaa3c82 100644
--- a/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml
+++ b/buildscripts/resmokeconfig/matrix_suites/overrides/multiversion.yml
@@ -12,6 +12,9 @@
         num_shards: 2
         num_rs_nodes_per_shard: 2
 
+- name: test_kind
+  value:
+    test_kind: all_versions_js_test
 
 ### Suite-specific overrides ###
 - name: multiversion_sanity_check_selector
diff --git a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
index a83ebb1e5e4..68903a8ee0f 100644
--- a/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/replica_sets_multiversion.yml
@@ -1,4 +1,4 @@
-test_kind: js_test
+test_kind: all_versions_js_test
 
 selector:
   roots:
diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
index 3ae8023351f..faf96dca13b 100644
--- a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
+++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
@@ -1,4 +1,4 @@
-test_kind: js_test
+test_kind: all_versions_js_test
 
 selector:
   roots:
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index ad50a565a00..cb14c081ee7 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -567,7 +567,6 @@ EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST,
 
 # Where to look for logging and suite configuration files
 CONFIG_DIR = None
-NAMED_SUITES = None
 LOGGER_DIR = None
 
 # Generated logging config for the current invocation.
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index fc5d352a922..06fe4b479d3 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -375,8 +375,6 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
 
     configure_tests(config.pop("test_files"), config.pop("replay_file"))
 
-    _config.NAMED_SUITES = SuiteFinder.get_named_suites(_config.CONFIG_DIR)
-
     _config.LOGGER_DIR = os.path.join(_config.CONFIG_DIR, "loggers")
 
     shuffle = config.pop("shuffle")
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index 4b5882432e8..c7edacd52cb 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -10,9 +10,6 @@ import shlex
 import sys
 import time
 import shutil
-import tempfile
-import requests
-import dateutil.parser
 
 import curatorbin
 import pkg_resources
@@ -40,6 +37,7 @@ from buildscripts.resmokelib.run import generate_multiversion_exclude_tags
 from buildscripts.resmokelib.run import runtime_recorder
 from buildscripts.resmokelib.run import list_tags
 from buildscripts.resmokelib.run.runtime_recorder import compare_start_time
+from buildscripts.resmokelib.suitesconfig import get_suite_files
 
 _INTERNAL_OPTIONS_TITLE = "Internal Options"
 _MONGODB_SERVER_OPTIONS_TITLE = "MongoDB Server Options"
@@ -153,11 +151,16 @@ class TestRunner(Subcommand):  # pylint: disable=too-many-instance-attributes
                                       suite_names)
 
     def list_tags(self):
-        """List the tags and its documentation available in the suites."""
+        """
+        List the tags and its documentation available in the suites.
+
+        Note: this currently ignores composed/matrix suites as it's not obvious the suite
+        a particular tag applies to.
+        """
         tag_docs = {}
         out_tag_names = []
-        for suite_name in suitesconfig.get_named_suites():
-            suite_file = config.NAMED_SUITES.get(suite_name, "")
+        for suite_name, suite_file in get_suite_files():
+            # Matrix suites are ignored.
             tags_blocks = list_tags.get_tags_blocks(suite_file)
 
             for tags_block in tags_blocks:
diff --git a/buildscripts/resmokelib/selector.py b/buildscripts/resmokelib/selector.py
index 8ce2f3a8761..238d305262f 100644
--- a/buildscripts/resmokelib/selector.py
+++ b/buildscripts/resmokelib/selector.py
@@ -721,6 +721,7 @@ _SELECTOR_REGISTRY = {
     "parallel_fsm_workload_test": (_MultiJSTestSelectorConfig, _MultiJSTestSelector),
     "json_schema_test": (_FileBasedSelectorConfig, _Selector),
     "js_test": (_JSTestSelectorConfig, _JSTestSelector),
+    "all_versions_js_test": (_JSTestSelectorConfig, _JSTestSelector),
     "mql_model_haskell_test": (_FileBasedSelectorConfig, _Selector),
     "mql_model_mongod_test": (_FileBasedSelectorConfig, _Selector),
     "multi_stmt_txn_passthrough": (_JSTestSelectorConfig, _JSTestSelector),
diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index d3b3bd47af3..b98b111928b 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -1,9 +1,9 @@
 """Module for retrieving the configuration of resmoke.py test suites."""
-
 import collections
-import optparse
 import os
 
+from typing import List, Dict
+
 import buildscripts.resmokelib.utils.filesystem as fs
 from buildscripts.resmokelib import config as _config
 from buildscripts.resmokelib import errors
@@ -11,27 +11,41 @@ from buildscripts.resmokelib import utils
 from buildscripts.resmokelib.testing import suite as _suite
 from buildscripts.resmokelib.utils import load_yaml_file
 
+SuiteName = str
+
+_NAMED_SUITES = None
+
+
+def get_named_suites() -> List[SuiteName]:
+    """Return a list of the suites names."""
+    global _NAMED_SUITES  # pylint: disable=global-statement
+
+    if _NAMED_SUITES is None:
+        # Skip "with_*server" and "no_server" because they do not define any test files to run.
+        executor_only = {"with_server", "with_external_server", "no_server"}
+
+        explicit_suite_names = [
+            name for name in ExplicitSuiteConfig.get_named_suites() if name not in executor_only
+        ]
+        composed_suite_names = MatrixSuiteConfig.get_named_suites()
+        _NAMED_SUITES = explicit_suite_names + composed_suite_names
+        _NAMED_SUITES.sort()
+    return _NAMED_SUITES
 
-def get_named_suites():
-    """Return a sorted list of the suites names."""
-    # Skip "with_*server" and "no_server" because they do not define any test files to run.
-    executor_only = {"with_server", "with_external_server", "no_server"}
-    names = [name for name in _config.NAMED_SUITES.keys() if name not in executor_only]
-    names += MatrixSuiteConfig.get_all_suite_names()
-    names.sort()
-    return names
 
+def get_suite_files() -> List[str]:
+    """Get the physical files defining these suites for parsing comments."""
+    return ExplicitSuiteConfig.get_suite_files() + MatrixSuiteConfig.get_suite_files()
 
-def get_named_suites_with_root_level_key(root_level_key):
-    """Return the suites that contain the given root_level_key and their values."""
-    all_suite_names = get_named_suites()
+
+def burn_in_multiversion_suites():
+    """Return the suites that should run in burn_in_multiversion."""
     suites_to_return = []
 
-    for suite in all_suite_names:
-        suite_config = _get_suite_config(suite)
-        if root_level_key in suite_config.keys() and suite_config[root_level_key]:
-            suites_to_return.append(
-                {"origin": suite, "multiversion_name": suite_config[root_level_key]})
+    for suite_name in get_named_suites():
+        multiversion_name = get_suite(suite_name).burn_in_multiversion_task_name()
+        if multiversion_name is not None:
+            suites_to_return.append({"origin": suite_name, "multiversion_name": multiversion_name})
     return suites_to_return
 
 
@@ -50,13 +64,11 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
         test_kind = frozenset(test_kind)
 
     test_membership = collections.defaultdict(list)
-    suite_names = get_named_suites()
-    for suite_name in suite_names:
+    for suite_name in get_named_suites():
         try:
-            suite_config = _get_suite_config(suite_name)
-            if test_kind and suite_config.get("test_kind") not in test_kind:
+            suite = get_suite(suite_name)
+            if test_kind and suite.get_test_kind_config() not in test_kind:
                 continue
-            suite = _suite.Suite(suite_name, suite_config)
         except IOError as err:
             # We ignore errors from missing files referenced in the test suite's "selector"
             # section. Certain test suites (e.g. unittests.yml) have a dedicated text file to
@@ -74,11 +86,11 @@ def create_test_membership_map(fail_on_missing_selector=False, test_kind=None):
     return test_membership
 
 
-def get_suites(suite_files, test_files):
+def get_suites(suite_names_or_paths, test_files):
     """Retrieve the Suite instances based on suite configuration files and override parameters.
 
     Args:
-        suite_files: A list of file paths pointing to suite YAML configuration files. For the suites
+        suite_names_or_paths: A list of file paths pointing to suite YAML configuration files. For the suites
             defined in 'buildscripts/resmokeconfig/suites/' and matrix suites, a shorthand name consisting
             of the filename without the extension can be used.
         test_files: A list of file paths pointing to test files overriding the roots for the suites.
@@ -94,7 +106,7 @@ def get_suites(suite_files, test_files):
         suite_roots = _make_suite_roots(test_files)
 
     suites = []
-    for suite_filename in suite_files:
+    for suite_filename in suite_names_or_paths:
         suite_config = _get_suite_config(suite_filename)
         if suite_roots:
             # Override the suite's default test files with those passed in from the command line.
@@ -104,51 +116,79 @@ def get_suites(suite_files, test_files):
     return suites
 
 
-def get_suite(suite_file):
-    """Retrieve the Suite instance corresponding to a suite configuration file."""
-    suite_config = _get_suite_config(suite_file)
-    return _suite.Suite(suite_file, suite_config)
-
-
 def _make_suite_roots(files):
     return {"selector": {"roots": files}}
 
 
-def _get_suite_config(suite_path):
+def _get_suite_config(suite_name_or_path):
     """Attempt to read YAML configuration from 'suite_path' for the suite."""
-    return SuiteFinder.get_config_obj(suite_path)
+    return SuiteFinder.get_config_obj(suite_name_or_path)
 
 
-class SuiteConfigInterface(object):
+class SuiteConfigInterface:
     """Interface for suite configs."""
 
-    def __init__(self, yaml_path=None):
-        """Initialize the suite config interface."""
-        self.yaml_path = yaml_path
+    @classmethod
+    def get_config_obj(cls, suite_name):
+        """Get the config object given the suite name, which can be a path."""
+        pass
+
+    @classmethod
+    def get_named_suites(cls):
+        """Populate the named suites by scanning `config_dir`."""
+        pass
+
+    @classmethod
+    def get_suite_files(cls):
+        """Get the physical files defining these suites for parsing comments."""
+        pass
 
 
 class ExplicitSuiteConfig(SuiteConfigInterface):
     """Class for storing the resmoke.py suite YAML configuration."""
 
-    @staticmethod
-    def get_config_obj(pathname):
+    @classmethod
+    def get_config_obj(cls, suite_name):
         """Get the suite config object in the given file."""
         # Named executors or suites are specified as the basename of the file, without the .yml
         # extension.
-        if not fs.is_yaml_file(pathname) and not os.path.dirname(pathname):
-            if pathname not in _config.NAMED_SUITES:  # pylint: disable=unsupported-membership-test
-                # Expand 'pathname' to full path.
+        if not fs.is_yaml_file(suite_name) and not os.path.dirname(suite_name):
+            named_suites = cls.get_named_suites()
+            if suite_name not in named_suites:  # pylint: disable=unsupported-membership-test
                 return None
-            pathname = _config.NAMED_SUITES[pathname]  # pylint: disable=unsubscriptable-object
+            suite_name = named_suites[suite_name]  # pylint: disable=unsubscriptable-object
 
-        if not fs.is_yaml_file(pathname) or not os.path.isfile(pathname):
-            raise optparse.OptionValueError("Expected a suite YAML config, but got '%s'" % pathname)
-        return utils.load_yaml_file(pathname)
+        if not fs.is_yaml_file(suite_name) or not os.path.isfile(suite_name):
+            raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name)
+        return utils.load_yaml_file(suite_name)
+
+    @classmethod
+    def get_named_suites(cls) -> Dict[str, str]:
+        """Populate the named suites by scanning config_dir/suites."""
+        named_suites = {}
+
+        suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
+        root = os.path.abspath(suites_dir)
+        files = os.listdir(root)
+        for filename in files:
+            (short_name, ext) = os.path.splitext(filename)
+            if ext in (".yml", ".yaml"):
+                pathname = os.path.join(root, filename)
+                named_suites[short_name] = pathname
+
+        return named_suites
+
+    @classmethod
+    def get_suite_files(cls):
+        """Get the suite files."""
+        return cls.get_named_suites()
 
 
 class MatrixSuiteConfig(SuiteConfigInterface):
     """Class for storing the resmoke.py suite YAML configuration."""
 
+    _all_mappings = None
+
     @staticmethod
     def get_all_yamls(target_dir):
         """Get all YAML files in the given directory."""
@@ -162,8 +202,7 @@ class MatrixSuiteConfig(SuiteConfigInterface):
                 pathname = os.path.join(root, filename)
 
                 if not fs.is_yaml_file(pathname) or not os.path.isfile(pathname):
-                    raise optparse.OptionValueError(
-                        "Expected a suite YAML config, but got '%s'" % pathname)
+                    raise ValueError("Expected a suite YAML config, but got '%s'" % pathname)
                 all_files[short_name] = load_yaml_file(pathname)
         return all_files
 
@@ -172,10 +211,10 @@ class MatrixSuiteConfig(SuiteConfigInterface):
         return os.path.join(_config.CONFIG_DIR, "matrix_suites")
 
     @classmethod
-    def get_config_obj(cls, suite_path):
+    def get_config_obj(cls, suite_name):
         """Get the suite config object in the given file."""
         suites_dir = cls._get_suites_dir()
-        matrix_suite = cls.parse_mappings_file(suites_dir, suite_path)
+        matrix_suite = cls.parse_mappings_file(suites_dir, suite_name)
         if not matrix_suite:
             return None
 
@@ -228,32 +267,40 @@ class MatrixSuiteConfig(SuiteConfigInterface):
         """Get the mapping object for a given suite name and directory to search for suite mappings."""
         all_matrix_suites = cls.get_all_mappings(suites_dir)
 
-        if suite_name in all_matrix_suites:
-            return all_matrix_suites[suite_name]
+        if suite_name in all_matrix_suites.keys():
+            return all_matrix_suites[suite_name]  # pylint: disable=unsubscriptable-object
         return None
 
     @classmethod
-    def get_all_suite_names(cls):
+    def get_named_suites(cls):
         """Get a list of all suite names."""
         suites_dir = cls._get_suites_dir()
         all_mappings = cls.get_all_mappings(suites_dir)
-        return all_mappings.keys()
+        return list(all_mappings.keys())
+
+    @classmethod
+    def get_suite_files(cls):
+        """Get the physical files defining these suites for parsing comments."""
+        mappings_dir = os.path.join(cls._get_suites_dir(), "mappings")
+        return cls.get_all_yamls(mappings_dir)
 
     @classmethod
-    def get_all_mappings(cls, suites_dir):
+    def get_all_mappings(cls, suites_dir) -> Dict[str, str]:
         """Get a dictionary of all suite mapping files keyed by the suite name."""
-        mappings_dir = os.path.join(suites_dir, "mappings")
-        mappings_files = cls.get_all_yamls(mappings_dir)
-
-        all_matrix_suites = {}
-        for _, suite_config_file in mappings_files.items():
-            for suite_config in suite_config_file:
-                if "suite_name" in suite_config and "base_suite" in suite_config:
-                    all_matrix_suites[suite_config["suite_name"]] = suite_config
-                else:
-                    raise ValueError("Invalid suite configuration, missing required keys. ",
-                                     suite_config)
-        return all_matrix_suites
+        if cls._all_mappings is None:
+            mappings_dir = os.path.join(suites_dir, "mappings")
+            mappings_files = cls.get_all_yamls(mappings_dir)
+
+            all_mappings = {}
+            for _, suite_config_file in mappings_files.items():
+                for suite_config in suite_config_file:
+                    if "suite_name" in suite_config and "base_suite" in suite_config:
+                        all_mappings[suite_config["suite_name"]] = suite_config
+                    else:
+                        raise ValueError("Invalid suite configuration, missing required keys. ",
+                                         suite_config)
+            cls._all_mappings = all_mappings
+        return cls._all_mappings
 
     @classmethod
     def merge_dicts(cls, dict1, dict2):
@@ -288,19 +335,8 @@ class SuiteFinder(object):
 
         return matrix_suite or explicit_suite
 
-    @staticmethod
-    def get_named_suites(config_dir):
-        """Populate the named suites by scanning config_dir/suites."""
-        named_suites = {}
-
-        suites_dir = os.path.join(config_dir, "suites")
-        root = os.path.abspath(suites_dir)
-        files = os.listdir(root)
-        for filename in files:
-            (short_name, ext) = os.path.splitext(filename)
-            if ext in (".yml", ".yaml"):
-                pathname = os.path.join(root, filename)
-                # TODO: store named suite in an object
-                named_suites[short_name] = pathname
 
-        return named_suites
+def get_suite(suite_name_or_path) -> _suite.Suite:
+    """Retrieve the Suite instance corresponding to a suite configuration file."""
+    suite_config = _get_suite_config(suite_name_or_path)
+    return _suite.Suite(suite_name_or_path, suite_config)
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index e02dcde1ae3..dc53d3785cb 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -36,6 +36,8 @@ EXIT_CODE_MAP = {
     3221225725: "Stack Overflow",
 }
 
+MULTIVERSION_CONFIG_KEY = "use_in_multiversion"
+
 
 def translate_exit_code(exit_code):
     """
@@ -71,7 +73,8 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         self._suite_options = suite_options
 
         self.test_kind = self.get_test_kind_config()
-        self.tests, self.excluded = self._get_tests_for_kind(self.test_kind)
+        self._tests = None
+        self._excluded = None
 
         self.return_code = None  # Set by the executor.
 
@@ -90,6 +93,20 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         """Create a string representation of object for debugging."""
         return f"{self.test_kind}:{self._suite_name}"
 
+    @property
+    def tests(self):
+        """Get the tests."""
+        if self._tests is None:
+            self._tests, self._excluded = self._get_tests_for_kind(self.test_kind)
+        return self._tests
+
+    @property
+    def excluded(self):
+        """Get the excluded."""
+        if self._excluded is None:
+            self._tests, self._excluded = self._get_tests_for_kind(self.test_kind)
+        return self._excluded
+
     def _get_tests_for_kind(self, test_kind):
         """Return the tests to run based on the 'test_kind'-specific filtering policy."""
         selector_config = self.get_selector_config()
@@ -151,6 +168,10 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         """Return the "test_kind" section of the YAML configuration."""
         return self._suite_config["test_kind"]
 
+    def burn_in_multiversion_task_name(self):
+        """Check if the suite should run in burn_in_multiversion by using a special root-level key."""
+        return self._suite_config.get(MULTIVERSION_CONFIG_KEY)
+
     @property
     def options(self):
         """Get the options."""
diff --git a/buildscripts/resmokelib/testing/testcases/jstest.py b/buildscripts/resmokelib/testing/testcases/jstest.py
index 2edd7c133dd..ccd5b4db5cd 100644
--- a/buildscripts/resmokelib/testing/testcases/jstest.py
+++ b/buildscripts/resmokelib/testing/testcases/jstest.py
@@ -275,3 +275,14 @@ class JSTestCase(interface.TestCase, interface.UndoDBUtilsMixin):
                 f"Mongo shell exited with code {return_code} while running jstest {self.basename()}."
                 " Further test execution may be unsafe.")
             raise self.propagate_error  # pylint: disable=raising-bad-type
+
+
+class AllVersionsJSTestCase(JSTestCase):
+    """
+    Alias for JSTestCase for multiversion passthrough suites.
+
+    It run with all combinations of versions of replica sets and sharded clusters.
+    The distinct name is picked up by task generation.
+    """
+
+    REGISTERED_NAME = "all_versions_js_test"
diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py
index a535705cb98..a49ba2494c7 100644
--- a/buildscripts/selected_tests.py
+++ b/buildscripts/selected_tests.py
@@ -26,7 +26,6 @@ from buildscripts.patch_builds.selected_tests.selected_tests_client import Selec
 from buildscripts.task_generation.evg_config_builder import EvgConfigBuilder
 from buildscripts.task_generation.gen_config import GenerationConfiguration
 from buildscripts.task_generation.generated_config import GeneratedConfiguration
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
 from buildscripts.task_generation.suite_split import SuiteSplitParameters, SuiteSplitConfig
 from buildscripts.task_generation.suite_split_strategies import SplitStrategy, FallbackStrategy, \
     greedy_division, round_robin_fallback
@@ -503,8 +502,6 @@ def main(
         binder.bind(FallbackStrategy, round_robin_fallback)
         binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options())
         binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file())
-        binder.bind(ResmokeProxyConfig,
-                    ResmokeProxyConfig(resmoke_suite_dir=DEFAULT_TEST_SUITE_DIR))
 
     inject.configure(dependencies)
 
diff --git a/buildscripts/task_generation/resmoke_proxy.py b/buildscripts/task_generation/resmoke_proxy.py
index 487dd2a05cd..820e63fb284 100644
--- a/buildscripts/task_generation/resmoke_proxy.py
+++ b/buildscripts/task_generation/resmoke_proxy.py
@@ -8,9 +8,8 @@ import structlog
 import yaml
 
 import buildscripts.resmokelib.parser as _parser
-import buildscripts.resmokelib.suitesconfig as suitesconfig
+import buildscripts.resmokelib.suitesconfig as _suiteconfig
 from buildscripts.task_generation.generated_config import GeneratedFile
-from buildscripts.util.fileops import read_yaml_file
 
 if TYPE_CHECKING:
     from buildscripts.task_generation.suite_split import GeneratedSuite, SubSuite
@@ -23,29 +22,14 @@ HEADER_TEMPLATE = """# DO NOT EDIT THIS FILE. All manual edits will be lost.
 """
 
 
-class ResmokeProxyConfig(NamedTuple):
-    """
-    Configuration for resmoke proxy.
-
-    resmoke_suite_dir: Directory that contains resmoke suite configurations.
-    """
-
-    resmoke_suite_dir: str
-
-
 class ResmokeProxyService:
     """A service to proxy requests to resmoke."""
 
     @inject.autoparams()
-    def __init__(self, proxy_config: ResmokeProxyConfig) -> None:
-        """
-        Initialize the service.
-
-        :param proxy_config: Configuration for the proxy.
-        """
+    def __init__(self) -> None:
+        """Initialize the service."""
         _parser.set_run_options()
-        self.suitesconfig = suitesconfig
-        self.resmoke_suite_dir = proxy_config.resmoke_suite_dir
+        self._suite_config = _suiteconfig
 
     def list_tests(self, suite_name: str) -> List[str]:
         """
@@ -54,9 +38,9 @@ class ResmokeProxyService:
         :param suite_name: Name of suite to query.
         :return: List of test names that belong to the suite.
         """
-        suite_config = self.suitesconfig.get_suite(suite_name)
+        suite = self._suite_config.get_suite(suite_name)
         test_list = []
-        for tests in suite_config.tests:
+        for tests in suite.tests:
             # `tests` could return individual tests or lists of tests, we need to handle both.
             if isinstance(tests, list):
                 test_list.extend(tests)
@@ -72,7 +56,7 @@ class ResmokeProxyService:
         :param suite_name: Name of suite to read.
         :return: Configuration of specified suite.
         """
-        return read_yaml_file(os.path.join(self.resmoke_suite_dir, f"{suite_name}.yml"))
+        return self._suite_config.SuiteFinder.get_config_obj(suite_name)
 
     def render_suite_files(self, suites: List["SubSuite"], suite_name: str,
                            generated_suite_filename: str, test_list: List[str],
@@ -92,7 +76,7 @@ class ResmokeProxyService:
         :return: Dictionary of rendered resmoke config files.
         """
         # pylint: disable=too-many-arguments
-        source_config = self.read_suite_config(suite_name)
+        source_config = self._suite_config.SuiteFinder.get_config_obj(suite_name)
         suite_configs = [
             GeneratedFile(file_name=f"{suite.sub_suite_config_file(i)}.yml",
                           content=sub_suite.generate_resmoke_config(source_config))
diff --git a/buildscripts/task_generation/task_types/multiversion_tasks.py b/buildscripts/task_generation/task_types/multiversion_tasks.py
index a783cb743ea..0cb0fd17742 100644
--- a/buildscripts/task_generation/task_types/multiversion_tasks.py
+++ b/buildscripts/task_generation/task_types/multiversion_tasks.py
@@ -129,6 +129,7 @@ class MultiversionGenTaskService:
         :param params: Parameters for how tasks should be generated.
         :return: Arguments to pass to resmoke to run the generated task.
         """
+
         tag_file_location = self.gen_task_options.generated_file_location(EXCLUDE_TAGS_FILE)
 
         return (
diff --git a/buildscripts/tests/resmokelib/test_suitesconfig.py b/buildscripts/tests/resmokelib/test_suitesconfig.py
index 31f65e40612..44d01c31ec1 100644
--- a/buildscripts/tests/resmokelib/test_suitesconfig.py
+++ b/buildscripts/tests/resmokelib/test_suitesconfig.py
@@ -23,31 +23,29 @@ class TestSuitesConfig(unittest.TestCase):
 
         membership_map = suitesconfig.create_test_membership_map(test_kind="nonexistent_test")
         self.assertEqual(membership_map, {})
-        self.assertEqual(mock_suite_class.call_count, 0)
+        self.assertEqual(mock_suite_class.call_count, 2)
 
-    @mock.patch(RESMOKELIB + ".testing.suite.Suite")
+    @mock.patch(RESMOKELIB + ".testing.suite.Suite.tests")
     @mock.patch(RESMOKELIB + ".suitesconfig.get_named_suites")
     def test_multiple_suites_matching_single_test_kind(self, mock_get_named_suites,
-                                                       mock_suite_class):
+                                                       mock_suite_get_tests):
         all_suites = ["core", "replica_sets_jscore_passthrough"]
         mock_get_named_suites.return_value = all_suites
 
-        mock_suite_class.return_value.tests = ["test1", "test2"]
+        mock_suite_get_tests.__get__ = mock.Mock(return_value=["test1", "test2"])
 
         membership_map = suitesconfig.create_test_membership_map(test_kind="js_test")
         self.assertEqual(membership_map, dict(test1=all_suites, test2=all_suites))
-        self.assertEqual(mock_suite_class.call_count, 2)
 
-    @mock.patch(RESMOKELIB + ".testing.suite.Suite")
+    @mock.patch(RESMOKELIB + ".testing.suite.Suite.tests")
     @mock.patch(RESMOKELIB + ".suitesconfig.get_named_suites")
     def test_multiple_suites_matching_multiple_test_kinds(self, mock_get_named_suites,
-                                                          mock_suite_class):
+                                                          mock_suite_get_tests):
         all_suites = ["core", "concurrency"]
         mock_get_named_suites.return_value = all_suites
 
-        mock_suite_class.return_value.tests = ["test1", "test2"]
+        mock_suite_get_tests.__get__ = mock.Mock(return_value=["test1", "test2"])
 
         membership_map = suitesconfig.create_test_membership_map(
             test_kind=("fsm_workload_test", "js_test"))
         self.assertEqual(membership_map, dict(test1=all_suites, test2=all_suites))
-        self.assertEqual(mock_suite_class.call_count, 2)
diff --git a/buildscripts/tests/task_generation/test_resmoke_proxy.py b/buildscripts/tests/task_generation/test_resmoke_proxy.py
index 4752cf4659e..d2f6ba61572 100644
--- a/buildscripts/tests/task_generation/test_resmoke_proxy.py
+++ b/buildscripts/tests/task_generation/test_resmoke_proxy.py
@@ -12,9 +12,9 @@ class TestResmokeProxy(unittest.TestCase):
         mock_suite = MagicMock(
             tests=["test0", "test1", ["test2a", "tests2b", "test2c"], "test3", ["test4a"]])
 
-        resmoke_proxy = under_test.ResmokeProxyService(under_test.ResmokeProxyConfig("suites_dir"))
-        resmoke_proxy.suitesconfig = MagicMock()
-        resmoke_proxy.suitesconfig.get_suite.return_value = mock_suite
+        resmoke_proxy = under_test.ResmokeProxyService()
+        resmoke_proxy._suite_config = MagicMock()
+        resmoke_proxy._suite_config.get_suite.return_value = mock_suite
 
         test_list = resmoke_proxy.list_tests("some suite")
 
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index 009749f5fc3..b8ea84a7d43 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -311,7 +311,7 @@ class CreateExecutorList(unittest.TestCase):
         mock_get_named_suites.return_value = ["dbtest"]
 
         under_test.create_executor_list([], [])
-        self.assertEqual(mock_suite_class.call_count, 0)
+        self.assertEqual(mock_suite_class.call_count, 1)
 
 
 def create_variant_task_mock(task_name, suite_name, distro="distro"):
diff --git a/buildscripts/tests/test_burn_in_tests_multiversion.py b/buildscripts/tests/test_burn_in_tests_multiversion.py
index 9e1ac0d2e0b..eb7ec81aa89 100644
--- a/buildscripts/tests/test_burn_in_tests_multiversion.py
+++ b/buildscripts/tests/test_burn_in_tests_multiversion.py
@@ -20,7 +20,6 @@ from buildscripts.evergreen_burn_in_tests import EvergreenFileChangeDetector
 from buildscripts.task_generation.gen_config import GenerationConfiguration
 from buildscripts.task_generation.multiversion_util import REPL_MIXED_VERSION_CONFIGS, \
     SHARDED_MIXED_VERSION_CONFIGS
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
 from buildscripts.task_generation.suite_split import SuiteSplitConfig
 from buildscripts.task_generation.suite_split_strategies import greedy_division, SplitStrategy, \
     FallbackStrategy, round_robin_fallback
@@ -165,8 +164,6 @@ def configure_dependencies(evg_api, split_config):
         binder.bind(GenTaskOptions, gen_task_options)
         binder.bind(EvergreenApi, evg_api)
         binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file())
-        binder.bind(ResmokeProxyConfig,
-                    ResmokeProxyConfig(resmoke_suite_dir=under_test.DEFAULT_TEST_SUITE_DIR))
         binder.bind(EvergreenFileChangeDetector, None)
         binder.bind(EvergreenProjectConfig, MagicMock())
         binder.bind(
diff --git a/buildscripts/tests/test_evergreen_burn_in_tests.py b/buildscripts/tests/test_evergreen_burn_in_tests.py
index 87624813cab..f2dabd0b295 100644
--- a/buildscripts/tests/test_evergreen_burn_in_tests.py
+++ b/buildscripts/tests/test_evergreen_burn_in_tests.py
@@ -98,7 +98,6 @@ class TestAcceptance(unittest.TestCase):
         # introduce failures and require this test to be updated.
         # You can see the test file it is using below. This test is used in the 'auth' and
         # 'auth_audit' test suites. It needs to be in at least one of those for the test to pass.
-        _config.NAMED_SUITES = None
         variant = "enterprise-rhel-80-64-bit"
         repos = [mock_changed_git_files(["jstests/auth/auth1.js"])]
         repeat_config = under_test.RepeatConfig()
diff --git a/buildscripts/tests/test_selected_tests.py b/buildscripts/tests/test_selected_tests.py
index 377ee59bc6b..8bd5f17ebb4 100644
--- a/buildscripts/tests/test_selected_tests.py
+++ b/buildscripts/tests/test_selected_tests.py
@@ -17,7 +17,6 @@ from buildscripts.patch_builds.selected_tests.selected_tests_client import Selec
     TaskMapping
 from buildscripts.selected_tests import EvgExpansions
 from buildscripts.task_generation.gen_config import GenerationConfiguration
-from buildscripts.task_generation.resmoke_proxy import ResmokeProxyConfig
 from buildscripts.task_generation.suite_split import SuiteSplitConfig
 from buildscripts.task_generation.suite_split_strategies import SplitStrategy, greedy_division, \
     FallbackStrategy, round_robin_fallback
@@ -59,7 +58,6 @@ def configure_dependencies(evg_api, evg_expansions, evg_project_config, selected
         binder.bind(GenTaskOptions, evg_expansions.build_gen_task_options())
         binder.bind(EvergreenApi, evg_api)
         binder.bind(GenerationConfiguration, GenerationConfiguration.from_yaml_file())
-        binder.bind(ResmokeProxyConfig, ResmokeProxyConfig(resmoke_suite_dir=test_suites_dir))
         binder.bind(SelectedTestsClient, selected_test_client)
 
     inject.clear_and_configure(dependencies)
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d706fbd9de4..0f0b22597f6 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -5800,7 +5800,6 @@ tasks:
       suite: retryable_writes_downgrade_passthrough
       resmoke_args: --storageEngine=wiredTiger
       require_multiversion: true
-      implicit_multiversion: true
 
 - <<: *gen_task_template
   name: sharded_retryable_writes_downgrade_passthrough_gen
diff --git a/evergreen/implicit_multiversions_tasks_generate.sh b/evergreen/implicit_multiversions_tasks_generate.sh
index da452d69ab0..f39e003604c 100644
--- a/evergreen/implicit_multiversions_tasks_generate.sh
+++ b/evergreen/implicit_multiversions_tasks_generate.sh
@@ -8,6 +8,7 @@ set -o errexit
 activate_venv
 PATH="$PATH:/data/multiversion"
 
+# TODO: remove this file and generate tags at execution time for suites defined in multiversion/
 if [ -n "${require_multiversion}" ]; then
   $python buildscripts/resmoke.py generate-multiversion-exclude-tags --oldBinVersion=last_continuous
 fi