SERVER-60927 run non-fuzzer multiversion suites against last-lts and last-continuous

6 files changed, 73 insertions, 66 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index de93f1840f1..21b37a91f35 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -108,7 +108,7 @@ DEFAULTS = {
     "transport_layer": None,
     "user_friendly_output": None,
     "mixed_bin_versions": None,
-    "old_bin_version": "last_continuous",
+    "old_bin_version": None,
     "linear_chain": None,
     "num_replset_nodes": None,
     "num_shards": None,
@@ -275,6 +275,9 @@ ARCHIVE_LIMIT_MB = None
 # The limit number of tests to archive for an Evergreen task.
 ARCHIVE_LIMIT_TESTS = None
 
+# Whether to back up data when restarting a process.
+BACKUP_ON_RESTART_DIR = None
+
 # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the
 # mongo shell.
 BASE_PORT = None
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
index d32a1e1e758..50da8c1b0f0 100644
--- a/buildscripts/resmokelib/configure_resmoke.py
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -162,10 +162,11 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
         if _config.RUN_ALL_FEATURE_FLAG_TESTS:
             _config.RUN_ALL_FEATURE_FLAGS = True
 
-        all_feature_flags = []
+        all_ff = []
         enabled_feature_flags = []
         try:
-            all_feature_flags = open(ALL_FEATURE_FLAG_FILE).read().split()
+            with open(ALL_FEATURE_FLAG_FILE) as fd:
+                all_ff = fd.read().split()
         except FileNotFoundError:
             # If we ask resmoke to run with all feature flags, the feature flags file
             # needs to exist.
@@ -173,7 +174,7 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
                 raise
 
         if _config.RUN_ALL_FEATURE_FLAGS:
-            enabled_feature_flags = all_feature_flags[:]
+            enabled_feature_flags = all_ff[:]
 
         # Specify additional feature flags from the command line.
         # Set running all feature flag tests to True if this options is specified.
@@ -181,7 +182,7 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
         if additional_feature_flags is not None:
             enabled_feature_flags.extend(additional_feature_flags)
 
-        return enabled_feature_flags, all_feature_flags
+        return enabled_feature_flags, all_ff
 
     _config.ENABLED_FEATURE_FLAGS, all_feature_flags = setup_feature_flags()
     not_enabled_feature_flags = list(set(all_feature_flags) - set(_config.ENABLED_FEATURE_FLAGS))
@@ -246,10 +247,6 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements,too-many
     _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
 
     mongod_set_parameters = config.pop("mongod_set_parameters")
-    # TODO: This should eventually be migrated entirely to _builder.py
-    if _config.ENABLED_FEATURE_FLAGS and not _config.MIXED_BIN_VERSIONS:
-        feature_flag_dict = {ff: "true" for ff in _config.ENABLED_FEATURE_FLAGS}
-        mongod_set_parameters.append(str(feature_flag_dict))
 
     _config.MONGOD_SET_PARAMETERS = _merge_set_params(mongod_set_parameters)
     _config.FUZZ_MONGOD_CONFIGS = config.pop("fuzz_mongod_configs")
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index 8a421e6342e..f9154ff202c 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -136,10 +136,15 @@ def mongo_shell_program(  # pylint: disable=too-many-arguments,too-many-branches
     else:
         test_name = None
 
+    # the Shell fixtures uses hyphen-delimited versions (e.g. last-lts) while resmoke.py
+    # uses underscore (e.g. last_lts). resmoke's version is needed as it's part of the task name.
+    shell_mixed_version = (config.MULTIVERSION_BIN_VERSION or "").replace("_", "-")
+
     shortcut_opts = {
         "backupOnRestartDir": (config.BACKUP_ON_RESTART_DIR, None),
         "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
         "mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""),
+        "multiversionBinVersion": (shell_mixed_version, ""),
         "noJournal": (config.NO_JOURNAL, False),
         "storageEngine": (config.STORAGE_ENGINE, ""),
         "storageEngineCacheSizeGB": (config.STORAGE_ENGINE_CACHE_SIZE, ""),
@@ -172,15 +177,21 @@ def mongo_shell_program(  # pylint: disable=too-many-arguments,too-many-branches
     mongos_set_parameters = test_data.get("setParametersMongos", {}).copy()
     mongocryptd_set_parameters = test_data.get("setParametersMongocryptd", {}).copy()
 
+    feature_flag_dict = {}
+    if config.ENABLED_FEATURE_FLAGS is not None:
+        feature_flag_dict = {ff: "true" for ff in config.ENABLED_FEATURE_FLAGS}
+
     # Propagate additional setParameters to mongod processes spawned by the mongo shell. Command
     # line options to resmoke.py override the YAML configuration.
     if config.MONGOD_SET_PARAMETERS is not None:
         mongod_set_parameters.update(utils.load_yaml(config.MONGOD_SET_PARAMETERS))
+        mongod_set_parameters.update(feature_flag_dict)
 
     # Propagate additional setParameters to mongos processes spawned by the mongo shell. Command
     # line options to resmoke.py override the YAML configuration.
     if config.MONGOS_SET_PARAMETERS is not None:
         mongos_set_parameters.update(utils.load_yaml(config.MONGOS_SET_PARAMETERS))
+        mongos_set_parameters.update(feature_flag_dict)
 
     # Propagate additional setParameters to mongocryptd processes spawned by the mongo shell.
     # Command line options to resmoke.py override the YAML configuration.
diff --git a/buildscripts/resmokelib/run/__init__.py b/buildscripts/resmokelib/run/__init__.py
index 0d9a3432ea4..303b4a267d6 100644
--- a/buildscripts/resmokelib/run/__init__.py
+++ b/buildscripts/resmokelib/run/__init__.py
@@ -801,19 +801,6 @@ class RunPlugin(PluginInterface):
             " to run a particular test under a particular suite configuration.")
 
         parser.add_argument(
-            "--mixedBinVersions", type=str, dest="mixed_bin_versions",
-            metavar="version1-version2-..-versionN",
-            help="Runs the test with the provided replica set"
-            " binary version configuration. Specify 'old-new' to configure a replica set with a"
-            " 'last-lts' version primary and 'latest' version secondary. For a sharded cluster"
-            " with two shards and two replica set nodes each, specify 'old-new-old-new'.")
-
-        parser.add_argument(
-            "--oldBinVersion", type=str, dest="old_bin_version",
-            choices=config.MultiversionOptions.all_options(),
-            help="Choose the multiversion binary version as last-lts or last-continuous.")
-
-        parser.add_argument(
             "--linearChain", action="store", dest="linear_chain", choices=("on", "off"),
             metavar="ON|OFF", help="Enable or disable linear chaining for tests using "
             "ReplicaSetFixture.")
diff --git a/buildscripts/resmokelib/suitesconfig.py b/buildscripts/resmokelib/suitesconfig.py
index 709a1373d65..63353d56b3c 100644
--- a/buildscripts/resmokelib/suitesconfig.py
+++ b/buildscripts/resmokelib/suitesconfig.py
@@ -140,36 +140,40 @@ class SuiteConfigInterface:
 class ExplicitSuiteConfig(SuiteConfigInterface):
     """Class for storing the resmoke.py suite YAML configuration."""
 
+    _named_suites = {}
+
     @classmethod
     def get_config_obj(cls, suite_name):
         """Get the suite config object in the given file."""
-        # Named executors or suites are specified as the basename of the file, without the .yml
-        # extension.
-        if not fs.is_yaml_file(suite_name) and not os.path.dirname(suite_name):
-            named_suites = cls.get_named_suites()
-            if suite_name not in named_suites:  # pylint: disable=unsupported-membership-test
-                return None
-            suite_name = named_suites[suite_name]  # pylint: disable=unsubscriptable-object
-
-        if not fs.is_yaml_file(suite_name) or not os.path.isfile(suite_name):
-            raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name)
-        return utils.load_yaml_file(suite_name)
+        if suite_name in cls.get_named_suites():
+            # Check if is a named suite first for efficiency.
+            suite_path = cls.get_named_suites()[suite_name]
+        elif fs.is_yaml_file(suite_name):
+            # Check if is a path to a YAML file.
+            if os.path.isfile(suite_name):
+                suite_path = suite_name
+            else:
+                raise ValueError("Expected a suite YAML config, but got '%s'" % suite_name)
+        else:
+            # Not an explicit suite, return None.
+            return None
+
+        return utils.load_yaml_file(suite_path)
 
     @classmethod
     def get_named_suites(cls) -> Dict[str, str]:
         """Populate the named suites by scanning config_dir/suites."""
-        named_suites = {}
-
-        suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
-        root = os.path.abspath(suites_dir)
-        files = os.listdir(root)
-        for filename in files:
-            (short_name, ext) = os.path.splitext(filename)
-            if ext in (".yml", ".yaml"):
-                pathname = os.path.join(root, filename)
-                named_suites[short_name] = pathname
-
-        return named_suites
+        if not cls._named_suites:
+            suites_dir = os.path.join(_config.CONFIG_DIR, "suites")
+            root = os.path.abspath(suites_dir)
+            files = os.listdir(root)
+            for filename in files:
+                (short_name, ext) = os.path.splitext(filename)
+                if ext in (".yml", ".yaml"):
+                    pathname = os.path.join(root, filename)
+                    cls._named_suites[short_name] = pathname
+
+        return cls._named_suites
 
     @classmethod
     def get_suite_files(cls):
@@ -180,7 +184,8 @@ class ExplicitSuiteConfig(SuiteConfigInterface):
 class MatrixSuiteConfig(SuiteConfigInterface):
     """Class for storing the resmoke.py suite YAML configuration."""
 
-    _all_mappings = None
+    _all_mappings = {}
+    _all_overrides = {}
 
     @staticmethod
     def get_all_yamls(target_dir):
@@ -240,20 +245,19 @@ class MatrixSuiteConfig(SuiteConfigInterface):
     @classmethod
     def parse_override_file(cls, suites_dir):
         """Get a dictionary of all overrides in a given directory keyed by the suite name."""
-        overrides_dir = os.path.join(suites_dir, "overrides")
-        overrides_files = cls.get_all_yamls(overrides_dir)
-
-        all_overrides = {}
-        for filename, override_config_file in overrides_files.items():
-            for override_config in override_config_file:
-                if "name" in override_config and "value" in override_config:
-                    all_overrides[f"{filename}.{override_config['name']}"] = override_config[
-                        "value"]
-                else:
-                    raise ValueError("Invalid override configuration, missing required keys. ",
-                                     override_config)
-
-        return all_overrides
+        if not cls._all_overrides:
+            overrides_dir = os.path.join(suites_dir, "overrides")
+            overrides_files = cls.get_all_yamls(overrides_dir)
+
+            for filename, override_config_file in overrides_files.items():
+                for override_config in override_config_file:
+                    if "name" in override_config and "value" in override_config:
+                        cls._all_overrides[
+                            f"{filename}.{override_config['name']}"] = override_config["value"]
+                    else:
+                        raise ValueError("Invalid override configuration, missing required keys. ",
+                                         override_config)
+        return cls._all_overrides
 
     @classmethod
     def parse_mappings_file(cls, suites_dir, suite_name):
@@ -280,19 +284,17 @@ class MatrixSuiteConfig(SuiteConfigInterface):
     @classmethod
     def get_all_mappings(cls, suites_dir) -> Dict[str, str]:
         """Get a dictionary of all suite mapping files keyed by the suite name."""
-        if cls._all_mappings is None:
+        if not cls._all_mappings:
             mappings_dir = os.path.join(suites_dir, "mappings")
             mappings_files = cls.get_all_yamls(mappings_dir)
 
-            all_mappings = {}
             for _, suite_config_file in mappings_files.items():
                 for suite_config in suite_config_file:
                     if "suite_name" in suite_config and "base_suite" in suite_config:
-                        all_mappings[suite_config["suite_name"]] = suite_config
+                        cls._all_mappings[suite_config["suite_name"]] = suite_config
                     else:
                         raise ValueError("Invalid suite configuration, missing required keys. ",
                                          suite_config)
-            cls._all_mappings = all_mappings
         return cls._all_mappings
 
     @classmethod
diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py
index 19bc0c103c0..fc4215b807e 100644
--- a/buildscripts/resmokelib/testing/fixtures/_builder.py
+++ b/buildscripts/resmokelib/testing/fixtures/_builder.py
@@ -73,10 +73,13 @@ class ReplSetBuilder(FixtureBuilder):
         # We have the same code in configure_resmoke.py to split config.MIXED_BIN_VERSIONS,
         # but here it is for the case, when it comes from resmoke suite definition
         if isinstance(mixed_bin_versions, str):
-            mixed_bin_versions = mixed_bin_versions.split("-")
+            mixed_bin_versions = mixed_bin_versions.split("_")
         if config.MIXED_BIN_VERSIONS is None:
             config.MIXED_BIN_VERSIONS = mixed_bin_versions
+
         old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION)
+        if config.MULTIVERSION_BIN_VERSION is None:
+            config.MULTIVERSION_BIN_VERSION = old_bin_version
 
         # We also hijack the num_nodes because we need it here.
         num_nodes = kwargs.pop("num_nodes", 2)
@@ -283,10 +286,14 @@ class ShardedClusterBuilder(FixtureBuilder):
         # We have the same code in configure_resmoke.py to split config.MIXED_BIN_VERSIONS,
         # but here it is for the case, when it comes from resmoke suite definition
         if isinstance(mixed_bin_versions, str):
-            mixed_bin_versions = mixed_bin_versions.split("-")
+            mixed_bin_versions = mixed_bin_versions.split("_")
         if config.MIXED_BIN_VERSIONS is None:
             config.MIXED_BIN_VERSIONS = mixed_bin_versions
+
         old_bin_version = kwargs.pop("old_bin_version", config.MULTIVERSION_BIN_VERSION)
+        if config.MULTIVERSION_BIN_VERSION is None:
+            config.MULTIVERSION_BIN_VERSION = old_bin_version
+
         is_multiversion = mixed_bin_versions is not None
 
         num_shards = kwargs.pop("num_shards", 1)