SERVER-54622 Retrieve back-branch fixture files to assemble multiversion

remove task_path_suffix from evergreen_gen_multiversion_tasks.py

replication works

fix fixture API adherence test

generate mongos port on startup

fix sharding and reformat

8 files changed, 324 insertions, 112 deletions

diff --git a/buildscripts/resmokelib/testing/fixtures/__init__.py b/buildscripts/resmokelib/testing/fixtures/__init__.py
index ea85dadbc7c..49293d4fe4a 100644
--- a/buildscripts/resmokelib/testing/fixtures/__init__.py
+++ b/buildscripts/resmokelib/testing/fixtures/__init__.py
@@ -2,7 +2,7 @@
 
 from buildscripts.resmokelib.testing.fixtures.external import ExternalFixture as _ExternalFixture
 from buildscripts.resmokelib.testing.fixtures.interface import NoOpFixture as _NoOpFixture
-from buildscripts.resmokelib.testing.fixtures.interface import make_fixture
+from buildscripts.resmokelib.testing.fixtures._builder import make_fixture
 from buildscripts.resmokelib.utils import autoloader as _autoloader
 
 EXTERNAL_FIXTURE_CLASS = _ExternalFixture.REGISTERED_NAME
diff --git a/buildscripts/resmokelib/testing/fixtures/_builder.py b/buildscripts/resmokelib/testing/fixtures/_builder.py
new file mode 100644
index 00000000000..685791a84ca
--- /dev/null
+++ b/buildscripts/resmokelib/testing/fixtures/_builder.py
@@ -0,0 +1,190 @@
+"""Utilities for constructing fixtures that may span multiple versions."""
+import io
+import os
+import threading
+from abc import ABC, abstractmethod
+from git import Repo
+
+import buildscripts.resmokelib.utils.registry as registry
+import buildscripts.resmokelib.config as config
+from buildscripts.resmokelib import errors, multiversionconstants
+from buildscripts.resmokelib.utils import default_if_none
+from buildscripts.resmokelib.utils import autoloader
+from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
+from buildscripts.resmokelib.testing.fixtures.interface import _FIXTURES
+
+MONGO_REPO_LOCATION = "."
+FIXTURE_DIR = "buildscripts/resmokelib/testing/fixtures"
+RETRIEVE_DIR = "build/multiversionfixtures"
+RETRIEVE_LOCK = threading.Lock()
+
+USE_LEGACY_MULTIVERSION = True
+
+_BUILDERS = {}  # type: ignore
+
+
+def make_fixture(class_name, logger, job_num, *args, **kwargs):
+    """Provide factory function for creating Fixture instances."""
+
+    fixturelib = FixtureLib()
+
+    if class_name in _BUILDERS:
+        builder = _BUILDERS[class_name]()
+        return builder.build_fixture(logger, job_num, fixturelib, *args, **kwargs)
+
+    if class_name not in _FIXTURES:
+        raise ValueError("Unknown fixture class '%s'" % class_name)
+    return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
+
+
+class FixtureBuilder(ABC, metaclass=registry.make_registry_metaclass(_BUILDERS, type(ABC))):  # pylint: disable=invalid-metaclass
+    """
+    ABC for fixture builders.
+
+    If any fixture has special logic for assembling different components
+    (e.g. for multiversion), define a builder to handle it.
+    """
+
+    # For any subclass, set a REGISTERED_NAME corresponding to the fixture the class builds.
+    REGISTERED_NAME = "Builder"
+
+    @abstractmethod
+    def build_fixture(self, logger, job_num, fixturelib, *args, **kwargs):
+        """Abstract method to build a replica set."""
+        return
+
+
+class ReplSetBuilder(FixtureBuilder):
+    """Builder class for fixtures support replication."""
+
+    REGISTERED_NAME = "ReplicaSetFixture"
+
+    def build_fixture(self, logger, job_num, fixturelib, *args, **kwargs):  # pylint: disable=too-many-locals
+        """Build a replica set."""
+        # We hijack the mixed_bin_versions passed to the fixture.
+        mixed_bin_versions = kwargs.pop("mixed_bin_versions", config.MIXED_BIN_VERSIONS)
+        if USE_LEGACY_MULTIVERSION:
+            # We mark the use of the legacy multiversion system by allowing
+            # access to mixed_bin_versions.
+            kwargs["mixed_bin_versions"] = mixed_bin_versions
+
+        # We also hijack the num_nodes because we need it here.
+        num_nodes = kwargs.pop("num_nodes", 2)
+        num_replset_nodes = config.NUM_REPLSET_NODES
+        num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
+        kwargs["num_nodes"] = num_nodes
+
+        replset_config_options = kwargs.get("replset_config_options", {})
+        mongod_executable = default_if_none(
+            kwargs.get("mongod_executable"), config.MONGOD_EXECUTABLE,
+            config.DEFAULT_MONGOD_EXECUTABLE)
+        kwargs["mongod_executable"] = mongod_executable
+        num_nodes = kwargs["num_nodes"]
+        latest_mongod = mongod_executable
+        latest_class = "MongoDFixture"
+        executables = []
+        classes = []
+        fcv = None
+
+        lts_class_suffix = "_last_lts"
+
+        if mixed_bin_versions is None:
+            executables = [latest_mongod for x in range(num_nodes)]
+            classes = [latest_class for x in range(num_nodes)]
+        else:
+
+            is_config_svr = "configsvr" in replset_config_options and replset_config_options[
+                "configsvr"]
+            if USE_LEGACY_MULTIVERSION:
+                executables = [
+                    latest_mongod if (x == "new") else multiversionconstants.LAST_LTS_MONGOD_BINARY
+                    for x in mixed_bin_versions
+                ]
+                classes = [latest_class for x in range(num_nodes)]
+            else:
+                load_version(version_path_suffix=lts_class_suffix,
+                             shell_path=multiversionconstants.LAST_LTS_MONGO_BINARY)
+
+                if not is_config_svr:
+                    executables = [
+                        latest_mongod if
+                        (x == "new") else multiversionconstants.LAST_LTS_MONGOD_BINARY
+                        for x in mixed_bin_versions
+                    ]
+                    classes = [
+                        latest_class if (x == "new") else f"{latest_class}{lts_class_suffix}"
+                        for x in mixed_bin_versions
+                    ]
+            if is_config_svr:
+                # Our documented recommended path for upgrading shards lets us assume that config
+                # server nodes will always be fully upgraded before shard nodes.
+                executables = [latest_mongod, latest_mongod]
+                classes = [latest_class, latest_class]
+
+            num_versions = len(mixed_bin_versions)
+            fcv = multiversionconstants.LAST_LTS_FCV
+
+            if num_versions != num_nodes and not is_config_svr:
+                msg = (("The number of binary versions specified: {} do not match the number of"\
+                        " nodes in the replica set: {}.")).format(num_versions, num_nodes)
+                raise errors.ServerFailure(msg)
+
+        replset = _FIXTURES[self.REGISTERED_NAME](logger, job_num, fixturelib, *args, **kwargs)
+
+        replset.set_fcv(fcv)
+        for i in range(replset.num_nodes):
+            node = self._new_mongod(replset, i, executables[i], classes[i])
+            replset.install_mongod(node)
+
+        if replset.start_initial_sync_node:
+            if not replset.initial_sync_node:
+                replset.initial_sync_node_idx = replset.num_nodes
+                # TODO: This adds the linear chain and steady state param now, is that ok?
+                replset.initial_sync_node = self._new_mongod(replset, replset.initial_sync_node_idx,
+                                                             latest_mongod, latest_class)
+
+        return replset
+
+    @classmethod
+    def _new_mongod(cls, replset, index, executable, mongod_class):  # TODO Not a class method
+        """Return a standalone.MongoDFixture configured to be used as replica-set member."""
+        mongod_logger = replset.get_logger_for_mongod(index)
+        mongod_options = replset.get_options_for_mongod(index)
+
+        return make_fixture(mongod_class, mongod_logger, replset.job_num,
+                            mongod_executable=executable, mongod_options=mongod_options,
+                            preserve_dbpath=replset.preserve_dbpath)
+
+
+def load_version(version_path_suffix=None, shell_path=None):
+    """Load the last_lts fixtures."""
+    with RETRIEVE_LOCK, registry.suffix(version_path_suffix):
+        # Only one thread needs to retrieve the fixtures.
+        retrieve_dir = os.path.relpath(os.path.join(RETRIEVE_DIR, version_path_suffix))
+        if not os.path.exists(retrieve_dir):
+            try:
+                # Avoud circular import
+                import buildscripts.evergreen_gen_multiversion_tests as gen_tests
+                commit = gen_tests.get_backports_required_hash_for_shell_version(
+                    mongo_shell_path=shell_path)
+            except FileNotFoundError as err:
+                print("Error running the mongo shell, please ensure it's in your $PATH: ", err)
+                raise
+            retrieve_fixtures(retrieve_dir, commit)
+
+        package_name = retrieve_dir.replace('/', '.')
+        autoloader.load_all_modules(name=package_name, path=[retrieve_dir])  # type: ignore
+
+
+def retrieve_fixtures(directory, commit):
+    """Populate a directory with the fixture files corresponding to a commit."""
+    repo = Repo(MONGO_REPO_LOCATION)
+    real_commit = repo.commit(commit)
+    tree = real_commit.tree / FIXTURE_DIR
+
+    os.makedirs(directory, exist_ok=True)
+
+    for blob in tree.blobs:
+        output = os.path.join(directory, blob.name)
+        with io.BytesIO(blob.data_stream.read()) as retrieved, open(output, "w") as file:
+            file.write(retrieved.read().decode("utf-8"))
diff --git a/buildscripts/resmokelib/testing/fixtures/fixturelib.py b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
index 7ce01b6cbee..e163340f3a6 100644
--- a/buildscripts/resmokelib/testing/fixtures/fixturelib.py
+++ b/buildscripts/resmokelib/testing/fixtures/fixturelib.py
@@ -5,8 +5,9 @@ from buildscripts.resmokelib import core
 from buildscripts.resmokelib import errors
 from buildscripts.resmokelib import utils
 from buildscripts.resmokelib import logging
+from buildscripts.resmokelib.core import network
 from buildscripts.resmokelib.utils.history import make_historic as _make_historic
-from buildscripts.resmokelib.testing.fixtures import interface
+from buildscripts.resmokelib.testing.fixtures import _builder
 from buildscripts.resmokelib.multiversionconstants import LAST_LTS_MONGOD_BINARY, LAST_LTS_MONGOS_BINARY
 
 
@@ -39,6 +40,10 @@ class FixtureLib(object):
     # Programs #
     ############
 
+    def make_fixture(self, class_name, logger, job_num, *args, **kwargs):
+        """Build fixtures by calling builder API."""
+        return _builder.make_fixture(class_name, logger, job_num, *args, **kwargs)
+
     def mongod_program(self, logger, job_num, executable, process_kwargs, mongod_options):  # pylint: disable=too-many-arguments
         """
         Return a Process instance that starts mongod arguments constructed from 'mongod_options'.
@@ -86,6 +91,10 @@ class FixtureLib(object):
         """Return an objects whose attributes are fixture config values."""
         return _FixtureConfig()
 
+    def get_next_port(self, job_num):
+        """Return the next available port that fixture can use."""
+        return network.PortAllocator.next_fixture_port(job_num)
+
 
 class _FixtureConfig(object):  # pylint: disable=too-many-instance-attributes
     """Class that stores fixture configuration info."""
@@ -120,3 +129,6 @@ class _FixtureConfig(object):  # pylint: disable=too-many-instance-attributes
         self.MONGOS_SET_PARAMETERS = config.MONGOS_SET_PARAMETERS
         self.DBPATH_PREFIX = config.DBPATH_PREFIX
         self.DEFAULT_DBPATH_PREFIX = config.DEFAULT_DBPATH_PREFIX
+        # Config servers have to be upgraded first. We hardcode the value here since there's
+        # no way to set it on the command line.
+        self.CONFIG_SVR_MIXED_BIN_VERSIONS = ["new", "new"]
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 4237ca803b1..86def51322a 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -11,14 +11,49 @@ import pymongo.errors
 
 import buildscripts.resmokelib.multiversionconstants as multiversion
 import buildscripts.resmokelib.utils.registry as registry
-from buildscripts.resmokelib.testing.fixtures.fixturelib import FixtureLib
 
-_FIXTURES = {}  # type: ignore
+_VERSIONS = {}  # type: ignore
 
-# Represents the version of the API presented by interface.py,
+# FIXTURE_API_VERSION versions the API presented by interface.py,
 # registry.py, and fixturelib.py. Increment this when making
-# possibly-breaking changes to them.
-FIXTURE_API_VERSION = 0.1
+# changes to them. Follow semantic versioning so that back-branch
+# fixtures are compatible when new features are added.
+
+
+# Note for multiversion fixtures: The formal API version here describes what
+# features a fixture can expect resmokelib to provide (e.g. new_fixture_node_logger
+# in fixturelib). It also provides a definition of _minimum_ functionality
+# (defined in the Fixture base class) that resmoke at large can expect from a fixture.
+# It is possible for fixtures to define additional public members beyond the minimum
+# in the Fixture base class, for use in hooks, builders, etc. (e.g. the initial
+# sync node in ReplicaSetFixture). These form an informal API of their own, which has
+# less of a need to be formalized because we expect major changes to it to occur on the
+# current master, allowing backwards-compatibility. On the other hand, the
+# interface.py and fixturelib API establishes forward-compatibility of fixture files.
+# If the informal API becomes heavily used and needs forward-compatibility,
+# consider adding it to the formal API.
+class APIVersion(object, metaclass=registry.make_registry_metaclass(_VERSIONS)):  # pylint: disable=invalid-metaclass
+    """Class storing fixture API version info."""
+
+    REGISTERED_NAME = "APIVersion"
+
+    FIXTURE_API_VERSION = "0.1.0"
+
+    @classmethod
+    def check_api_version(cls, actual):
+        """Check that we are compatible with the actual API version."""
+
+        def to_major(version):
+            return int(version.split(".")[0])
+
+        def to_minor(version):
+            return int(version.split(".")[1])
+
+        expected = cls.FIXTURE_API_VERSION
+        return to_major(expected) == to_major(actual) and to_minor(expected) <= to_minor(actual)
+
+
+_FIXTURES = {}  # type: ignore
 
 
 class TeardownMode(Enum):
@@ -34,16 +69,6 @@ class TeardownMode(Enum):
     ABORT = 6
 
 
-def make_fixture(class_name, logger, job_num, *args, **kwargs):
-    """Provide factory function for creating Fixture instances."""
-
-    fixturelib = FixtureLib()
-
-    if class_name not in _FIXTURES:
-        raise ValueError("Unknown fixture class '%s'" % class_name)
-    return _FIXTURES[class_name](logger, job_num, fixturelib, *args, **kwargs)
-
-
 class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):  # pylint: disable=invalid-metaclass
     """Base class for all fixtures."""
 
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index e9771f0925f..7c8ac94bc6b 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -48,19 +48,20 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self.use_replica_set_connection_string = use_replica_set_connection_string
         self.default_read_concern = default_read_concern
         self.default_write_concern = default_write_concern
-        self.mixed_bin_versions = self.fixturelib.default_if_none(mixed_bin_versions,
-                                                                  self.config.MIXED_BIN_VERSIONS)
-        self.mixed_bin_versions_config = self.mixed_bin_versions
+        self.mixed_bin_versions = mixed_bin_versions
         self.shard_logging_prefix = shard_logging_prefix
         self.replicaset_logging_prefix = replicaset_logging_prefix
+        self.num_nodes = num_nodes
+        # Used by the enhanced multiversion system to signify multiversion mode.
+        # None implies no multiversion run.
+        self.fcv = None
 
         # Use the values given from the command line if they exist for linear_chain and num_nodes.
         linear_chain_option = self.fixturelib.default_if_none(self.config.LINEAR_CHAIN,
                                                               linear_chain)
         self.linear_chain = linear_chain_option if linear_chain_option else linear_chain
-        num_replset_nodes = self.config.NUM_REPLSET_NODES
-        self.num_nodes = num_replset_nodes if num_replset_nodes else num_nodes
 
+        # Legacy multiversion line
         if self.mixed_bin_versions is not None:
             mongod_executable = self.fixturelib.default_if_none(
                 self.mongod_executable, self.config.MONGOD_EXECUTABLE,
@@ -110,43 +111,29 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
             self._dbpath_prefix = os.path.join(self._dbpath_prefix, self.config.FIXTURE_SUBDIR)
 
         self.nodes = []
-        self.replset_name = None
+        self.replset_name = self.mongod_options.setdefault("replSet", "rs")
         self.initial_sync_node = None
         self.initial_sync_node_idx = -1
 
     def setup(self):  # pylint: disable=too-many-branches,too-many-statements,too-many-locals
         """Set up the replica set."""
-        self.replset_name = self.mongod_options.get("replSet", "rs")
-        if not self.nodes:
-            for i in range(self.num_nodes):
-                node = self._new_mongod(i, self.replset_name)
-                self.nodes.append(node)
 
+        # Version-agnostic options for mongod/s can be set here.
+        # Version-specific options should be set in get_version_specific_options_for_mongod()
+        # to avoid options for old versions being applied to new Replicaset fixtures.
         for i in range(self.num_nodes):
-            steady_state_constraint_param = "oplogApplicationEnforcesSteadyStateConstraints"
-            # TODO (SERVER-52985): Set steady state constraint parameters on last-lts nodes.
-            if (steady_state_constraint_param not in self.nodes[i].mongod_options["set_parameters"]
-                    and self.mixed_bin_versions is not None
-                    and self.mixed_bin_versions[i] == "new"):
-                self.nodes[i].mongod_options["set_parameters"][steady_state_constraint_param] = True
-            if self.linear_chain and i > 0:
-                self.nodes[i].mongod_options["set_parameters"][
-                    "failpoint.forceSyncSourceCandidate"] = self.fixturelib.make_historic({
-                        "mode": "alwaysOn",
-                        "data": {"hostAndPort": self.nodes[i - 1].get_internal_connection_string()}
-                    })
             self.nodes[i].setup()
 
-        if self.start_initial_sync_node:
-            if not self.initial_sync_node:
-                self.initial_sync_node_idx = len(self.nodes)
-                self.initial_sync_node = self._new_mongod(self.initial_sync_node_idx,
-                                                          self.replset_name)
+        if self.initial_sync_node:
             self.initial_sync_node.setup()
             self.initial_sync_node.await_ready()
 
+        # Legacy multiversion line
+        # TODO (SERVER-57255): Don't delete steady state constraint options when backporting to 5.0.
         if self.mixed_bin_versions:
             for i in range(self.num_nodes):
+                print("node[i] version: " + self.nodes[i].mongod_executable +
+                      "mixed_bin_version[i]: " + self.mixed_bin_versions[i])
                 if self.nodes[i].mongod_executable != self.mixed_bin_versions[i]:
                     msg = (f"Executable of node{i}: {self.nodes[i].mongod_executable} does not "
                            f"match the executable assigned by mixedBinVersions: "
@@ -213,23 +200,13 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self._initiate_repl_set(client, repl_config)
         self._await_primary()
 
-        if self.mixed_bin_versions is not None:
-            if self.mixed_bin_versions[0] == "new":
-                fcv_response = client.admin.command(
-                    {"getParameter": 1, "featureCompatibilityVersion": 1})
-                fcv = fcv_response["featureCompatibilityVersion"]["version"]
-                if fcv != ReplicaSetFixture._LATEST_FCV:
-                    msg = (("Server returned FCV{} when we expected FCV{}.").format(
-                        fcv, ReplicaSetFixture._LATEST_FCV))
-                    raise self.fixturelib.ServerFailure(msg)
-
+        if self.fcv is not None:
             # Initiating a replica set with a single node will use "latest" FCV. This will
             # cause IncompatibleServerVersion errors if additional "last-lts" binary version
             # nodes are subsequently added to the set, since such nodes cannot set their FCV to
             # "latest". Therefore, we make sure the primary is "last-lts" FCV before adding in
             # nodes of different binary versions to the replica set.
-            client.admin.command(
-                {"setFeatureCompatibilityVersion": ReplicaSetFixture._LAST_LTS_FCV})
+            client.admin.command({"setFeatureCompatibilityVersion": self.fcv})
 
         if self.nodes[1:]:
             # Wait to connect to each of the secondaries before running the replSetReconfig
@@ -581,21 +558,32 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         """Return initial sync node from the replica set."""
         return self.initial_sync_node
 
-    def _new_mongod(self, index, replset_name):
-        """Return a standalone.MongoDFixture configured to be used as replica-set member."""
-        mongod_executable = (self.mongod_executable
-                             if self.mixed_bin_versions is None else self.mixed_bin_versions[index])
-        mongod_logger = self._get_logger_for_mongod(index)
+    def set_fcv(self, fcv):
+        """Set the fcv used by this fixtures."""
+        self.fcv = fcv
+
+    def install_mongod(self, mongod):
+        """Install a mongod node. Called by a builder."""
+        self.nodes.append(mongod)
+
+    def get_options_for_mongod(self, index):
+        """Return options that may be passed to a mongod."""
         mongod_options = self.mongod_options.copy()
-        mongod_options["replSet"] = replset_name
-        mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "node{}".format(index))
-        mongod_options["set_parameters"] = mongod_options.get("set_parameters", {}).copy()
 
-        return interface.make_fixture(
-            "MongoDFixture", mongod_logger, self.job_num, mongod_executable=mongod_executable,
-            mongod_options=mongod_options, preserve_dbpath=self.preserve_dbpath)
+        mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "node{}".format(index))
+        mongod_options["set_parameters"] = mongod_options.get("set_parameters",
+                                                              self.fixturelib.make_historic(
+                                                                  {})).copy()
+
+        if self.linear_chain and index > 0:
+            self.mongod_options["set_parameters"][
+                "failpoint.forceSyncSourceCandidate"] = self.fixturelib.make_historic({
+                    "mode": "alwaysOn",
+                    "data": {"hostAndPort": self.nodes[index - 1].get_internal_connection_string()}
+                })
+        return mongod_options
 
-    def _get_logger_for_mongod(self, index):
+    def get_logger_for_mongod(self, index):
         """Return a new logging.Logger instance.
 
         The instance is used as the primary, secondary, or initial sync member of a replica-set.
@@ -624,9 +612,6 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
     def get_internal_connection_string(self):
         """Return the internal connection string."""
-        if self.replset_name is None:
-            raise ValueError("Must call setup() before calling get_internal_connection_string()")
-
         conn_strs = [node.get_internal_connection_string() for node in self.nodes]
         if self.initial_sync_node:
             conn_strs.append(self.initial_sync_node.get_internal_connection_string())
@@ -643,9 +628,6 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
 
     def get_driver_connection_url(self):
         """Return the driver connection URL."""
-        if self.replset_name is None:
-            raise ValueError("Must call setup() before calling get_driver_connection_url()")
-
         if self.use_replica_set_connection_string:
             # We use a replica set connection string when all nodes are electable because we
             # anticipate the client will want to gracefully handle any failovers.
@@ -658,6 +640,10 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
             # electable because we want the client to error out if a stepdown occurs.
             return self.nodes[0].get_driver_connection_url()
 
+    def write_historic(self, obj):
+        """Convert the obj to a record to track history."""
+        self.fixturelib.make_historic(obj)
+
 
 def get_last_optime(client, fixturelib):
     """Get the latest optime.
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 10fbfadd697..738f271176e 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -294,11 +294,16 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "config")
         mongod_options["replSet"] = ShardedClusterFixture._CONFIGSVR_REPLSET_NAME
         mongod_options["storageEngine"] = "wiredTiger"
+        config_svr_mixed_bin_version = None
+        if self.mixed_bin_versions is not None:
+            config_svr_mixed_bin_version = self.fixturelib.get_config(
+            ).CONFIG_SVR_MIXED_BIN_VERSIONS
 
-        return interface.make_fixture(
+        return self.fixturelib.make_fixture(
             "ReplicaSetFixture", mongod_logger, self.job_num, mongod_options=mongod_options,
             mongod_executable=self.mongod_executable, preserve_dbpath=preserve_dbpath,
-            num_nodes=num_nodes, auth_options=auth_options, mixed_bin_versions=None,
+            num_nodes=num_nodes, auth_options=auth_options,
+            mixed_bin_versions=config_svr_mixed_bin_version,
             replset_config_options=replset_config_options,
             shard_logging_prefix=shard_logging_prefix, **configsvr_options)
 
@@ -331,7 +336,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
         mongod_options["replSet"] = ShardedClusterFixture._SHARD_REPLSET_NAME_PREFIX + str(index)
 
-        return interface.make_fixture(
+        return self.fixturelib.make_fixture(
             "ReplicaSetFixture", mongod_logger, self.job_num,
             mongod_executable=self.mongod_executable, mongod_options=mongod_options,
             preserve_dbpath=preserve_dbpath, num_nodes=num_rs_nodes_per_shard,
@@ -402,7 +407,9 @@ class _MongoSFixture(interface.Fixture):
             self.fixturelib.default_if_none(mongos_options, {})).copy()
 
         self.mongos = None
-        self.port = None
+        self.port = fixturelib.get_next_port(job_num)
+        self.mongos_options["port"] = self.port
+
         self._dbpath_prefix = dbpath_prefix
 
     def setup(self):
@@ -413,9 +420,9 @@ class _MongoSFixture(interface.Fixture):
             self.mongos_options["logappend"] = ""
 
         launcher = MongosLauncher(self.fixturelib)
-        mongos, self.port = launcher.launch_mongos_program(self.logger, self.job_num,
-                                                           executable=self.mongos_executable,
-                                                           mongos_options=self.mongos_options)
+        mongos, _ = launcher.launch_mongos_program(self.logger, self.job_num,
+                                                   executable=self.mongos_executable,
+                                                   mongos_options=self.mongos_options)
         self.mongos_options["port"] = self.port
         try:
             self.logger.info("Starting mongos on port %d...\n%s", self.port, mongos.as_command())
@@ -507,9 +514,6 @@ class _MongoSFixture(interface.Fixture):
 
     def get_internal_connection_string(self):
         """Return the internal connection string."""
-        if self.mongos is None:
-            raise ValueError("Must call setup() before calling get_internal_connection_string()")
-
         return "localhost:%d" % self.port
 
     def get_driver_connection_url(self):
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index de53305ce9a..9dcf1a31389 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -19,7 +19,6 @@ class MongoDFixture(interface.Fixture):
             self, logger, job_num, fixturelib, mongod_executable=None, mongod_options=None,
             dbpath_prefix=None, preserve_dbpath=False):
         """Initialize MongoDFixture with different options for the mongod process."""
-
         interface.Fixture.__init__(self, logger, job_num, fixturelib, dbpath_prefix=dbpath_prefix)
         self.mongod_options = self.fixturelib.make_historic(
             self.fixturelib.default_if_none(mongod_options, {}))
@@ -49,7 +48,8 @@ class MongoDFixture(interface.Fixture):
             self.preserve_dbpath = preserve_dbpath
 
         self.mongod = None
-        self.port = None
+        self.port = fixturelib.get_next_port(job_num)
+        self.mongod_options["port"] = self.port
 
     def setup(self):
         """Set up the mongod."""
@@ -63,10 +63,11 @@ class MongoDFixture(interface.Fixture):
             pass
 
         launcher = MongodLauncher(self.fixturelib)
-        mongod, self.port = launcher.launch_mongod_program(self.logger, self.job_num,
-                                                           executable=self.mongod_executable,
-                                                           mongod_options=self.mongod_options)
-        self.mongod_options["port"] = self.port
+        # Second return val is the port, which we ignore because we explicitly created the port above.
+        # The port is used to set other mongod_option's here: https://github.com/mongodb/mongo/blob/532a6a8ae7b8e7ab5939e900759c00794862963d/buildscripts/resmokelib/testing/fixtures/replicaset.py#L136
+        mongod, _ = launcher.launch_mongod_program(self.logger, self.job_num,
+                                                   executable=self.mongod_executable,
+                                                   mongod_options=self.mongod_options)
         try:
             self.logger.info("Starting mongod on port %d...\n%s", self.port, mongod.as_command())
             mongod.start()
@@ -166,9 +167,6 @@ class MongoDFixture(interface.Fixture):
 
     def get_internal_connection_string(self):
         """Return the internal connection string."""
-        if self.mongod is None:
-            raise ValueError("Must call setup() before calling get_internal_connection_string()")
-
         return "localhost:%d" % self.port
 
     def get_driver_connection_url(self):
diff --git a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
index fd0dd9c5f2d..9431c2116b4 100644
--- a/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
+++ b/buildscripts/resmokelib/testing/fixtures/tenant_migration.py
@@ -49,21 +49,6 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
 
         self.replica_sets = []
 
-        # The ReplicaSetFixture for the replica set that starts out owning the data (i.e. the
-        # replica set that driver should connect to when running commands).
-        self.replica_set_with_tenant = None
-
-    def pids(self):
-        """:return: pids owned by this fixture if any."""
-        out = []
-        for replica_set in self.replica_sets:
-            out.extend(replica_set.pids())
-        if not out:
-            self.logger.debug('No replica sets when gathering multi replicaset fixture pids.')
-        return out
-
-    def setup(self):
-        """Set up the replica sets."""
         if not self.replica_sets:
             for i in range(self.num_replica_sets):
                 rs_name = f"rs{i}"
@@ -73,7 +58,7 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
                 mongod_options["replSet"] = rs_name
 
                 self.replica_sets.append(
-                    interface.make_fixture(
+                    self.fixturelib.make_fixture(
                         "ReplicaSetFixture", self.logger, self.job_num,
                         mongod_options=mongod_options, preserve_dbpath=self.preserve_dbpath,
                         num_nodes=self.num_nodes_per_replica_set, auth_options=self.auth_options,
@@ -83,9 +68,21 @@ class TenantMigrationFixture(interface.Fixture):  # pylint: disable=too-many-ins
                         use_replica_set_connection_string=self.use_replica_set_connection_string,
                         all_nodes_electable=self.all_nodes_electable))
 
-            self.replica_set_with_tenant = self.replica_sets[0]
+        # The ReplicaSetFixture for the replica set that starts out owning the data (i.e. the
+        # replica set that driver should connect to when running commands).
+        self.replica_set_with_tenant = self.replica_sets[0]
 
-        # Start up each of the replica sets
+    def pids(self):
+        """:return: pids owned by this fixture if any."""
+        out = []
+        for replica_set in self.replica_sets:
+            out.extend(replica_set.pids())
+        if not out:
+            self.logger.debug('No replica sets when gathering multi replicaset fixture pids.')
+        return out
+
+    def setup(self):
+        """Set up the replica sets."""
         for replica_set in self.replica_sets:
             replica_set.setup()
             self._create_tenant_migration_donor_and_recipient_roles(replica_set)