SERVER-42761 Allow resmoke and the shell to initiate a replica set with "latest" version primary and "last-stable" secondaries

5 files changed, 81 insertions, 30 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 3c31cd81ead..4e9012202e4 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -94,6 +94,7 @@ DEFAULTS = {
     "storage_engine_cache_size_gb": None,
     "tag_file": None,
     "transport_layer": None,
+    "mixed_bin_versions": None,
 
     # Evergreen options.
     "build_id": None,
@@ -379,6 +380,9 @@ STAGGER_JOBS = None
 # If set to true, it enables read concern majority. Else, read concern majority is disabled.
 MAJORITY_READ_CONCERN = None
 
+# Specifies the binary versions of each node we should run for a replica set.
+MIXED_BIN_VERSIONS = None
+
 # If set to "on", it enables flow control. If set to "off", it disables flow control. If left as
 # None, the server's default will determine whether flow control is enabled.
 FLOW_CONTROL = None
diff --git a/buildscripts/resmokelib/core/programs.py b/buildscripts/resmokelib/core/programs.py
index a000f34e8d3..f995badf353 100644
--- a/buildscripts/resmokelib/core/programs.py
+++ b/buildscripts/resmokelib/core/programs.py
@@ -244,6 +244,7 @@ def mongo_shell_program(  # pylint: disable=too-many-branches,too-many-locals,to
         test_name = None
     shortcut_opts = {
         "enableMajorityReadConcern": (config.MAJORITY_READ_CONCERN, True),
+        "mixedBinVersions": (config.MIXED_BIN_VERSIONS, ""),
         "noJournal": (config.NO_JOURNAL, False),
         "serviceExecutor": (config.SERVICE_EXECUTOR, ""),
         "storageEngine": (config.STORAGE_ENGINE, ""),
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index d37bb06108f..c9c425ea279 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -281,6 +281,12 @@ def _make_parser():  # pylint: disable=too-many-statements
         help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
         " to run a particular test under a particular suite configuration.")
 
+    parser.add_option(
+        "--mixedBinVersions", type="string", dest="mixed_bin_versions",
+        metavar="version1-version2-..-versionN", help="Runs the test with the provided replica set"
+        " binary version configuration. Specify 'old-new' to configure a replica set with a"
+        " 'last-stable' version primary and 'latest' version secondary.")
+
     evergreen_options = optparse.OptionGroup(
         parser, title=_EVERGREEN_OPTIONS_TITLE,
         description=("Options used to propagate information about the Evergreen task running this"
@@ -524,6 +530,12 @@ def _validate_config(parser):
     if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS:
         parser.error("Cannot specify --repeatTests and --repeatTestsSecs")
 
+    if _config.MIXED_BIN_VERSIONS is not None:
+        for version in _config.MIXED_BIN_VERSIONS:
+            if version not in set(['old', 'new']):
+                parser.error("Must specify binary versions as 'old' or 'new' in format"
+                             " 'version1-version2'")
+
 
 def validate_benchmark_options():
     """Error out early if any options are incompatible with benchmark test suites.
@@ -579,6 +591,9 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements
     _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable"))
     _config.JOBS = config.pop("jobs")
     _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on"
+    _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions")
+    if _config.MIXED_BIN_VERSIONS is not None:
+        _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-")
     _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
     _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
     _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 9c449dbc432..b72cfb89dc5 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -22,17 +22,21 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
     # Error response codes copied from mongo/base/error_codes.err.
     _NODE_NOT_FOUND = 74
 
+    _LAST_STABLE_FCV = "4.2"
+    _LATEST_FCV = "4.4"
+
+    _LAST_STABLE_BIN_VERSION = "4.2"
+
     def __init__(  # pylint: disable=too-many-arguments, too-many-locals
-            self, logger, job_num, mongod_executable=None, mongod_options=None, dbpath_prefix=None,
-            preserve_dbpath=False, num_nodes=2, start_initial_sync_node=False,
-            write_concern_majority_journal_default=None, auth_options=None,
-            replset_config_options=None, voting_secondaries=None, all_nodes_electable=False,
-            use_replica_set_connection_string=None, linear_chain=False):
+            self, logger, job_num, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False,
+            num_nodes=2, start_initial_sync_node=False, write_concern_majority_journal_default=None,
+            auth_options=None, replset_config_options=None, voting_secondaries=None,
+            all_nodes_electable=False, use_replica_set_connection_string=None, linear_chain=False,
+            mixed_bin_versions=None):
         """Initialize ReplicaSetFixture."""
 
         interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
-        self.mongod_executable = mongod_executable
         self.mongod_options = utils.default_if_none(mongod_options, {})
         self.preserve_dbpath = preserve_dbpath
         self.num_nodes = num_nodes
@@ -44,6 +48,22 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self.all_nodes_electable = all_nodes_electable
         self.use_replica_set_connection_string = use_replica_set_connection_string
         self.linear_chain = linear_chain
+        self.mixed_bin_versions = utils.default_if_none(mixed_bin_versions,
+                                                        config.MIXED_BIN_VERSIONS)
+        if self.mixed_bin_versions is not None:
+            mongod_executable = utils.default_if_none(config.MONGOD_EXECUTABLE,
+                                                      config.DEFAULT_MONGOD_EXECUTABLE)
+            latest_mongod = mongod_executable
+            last_stable_mongod = mongod_executable + "-" \
+                                + ReplicaSetFixture._LAST_STABLE_BIN_VERSION
+            self.mixed_bin_versions = [
+                latest_mongod if x == "new" else last_stable_mongod for x in self.mixed_bin_versions
+            ]
+            num_versions = len(self.mixed_bin_versions)
+            if num_versions != num_nodes:
+                msg = (("The number of binary versions: {} do not match the number of nodes: "\
+                        "{}.")).format(num_versions, num_nodes)
+                raise errors.ServerFailure(msg)
 
         # If voting_secondaries has not been set, set a default. By default, secondaries have zero
         # votes unless they are also nodes capable of being elected primary.
@@ -74,7 +94,6 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
     def setup(self):  # pylint: disable=too-many-branches,too-many-statements
         """Set up the replica set."""
         self.replset_name = self.mongod_options.get("replSet", "rs")
-
         if not self.nodes:
             for i in range(self.num_nodes):
                 node = self._new_mongod(i, self.replset_name)
@@ -159,6 +178,24 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         self._configure_repl_set(client, {"replSetInitiate": repl_config})
         self._await_primary()
 
+        if self.mixed_bin_versions is not None:
+            if self.mixed_bin_versions[0] == "new":
+                fcv_response = client.admin.command(
+                    {"getParameter": 1, "featureCompatibilityVersion": 1})
+                fcv = fcv_response["featureCompatibilityVersion"]["version"]
+                if fcv != ReplicaSetFixture._LATEST_FCV:
+                    msg = (("Server returned FCV{} when we expected FCV{}.").format(
+                        fcv, ReplicaSetFixture._LATEST_FCV))
+                    raise errors.ServerFailure(msg)
+
+            # Initiating a replica set with a single node will use "latest" FCV. This will
+            # cause IncompatibleServerVersion errors if additional "last-stable" binary version
+            # nodes are subsequently added to the set, since such nodes cannot set their FCV to
+            # "latest". Therefore, we make sure the primary is "last-stable" FCV before adding in
+            # nodes of different binary versions to the replica set.
+            client.admin.command(
+                {"setFeatureCompatibilityVersion": ReplicaSetFixture._LAST_STABLE_FCV})
+
         if self.nodes[1:]:
             # Wait to connect to each of the secondaries before running the replSetReconfig
             # command.
@@ -429,12 +466,13 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         return [node for node in self.nodes if node.port != primary.port]
 
     def get_initial_sync_node(self):
-        """Return initila sync node from the replica set."""
+        """Return initial sync node from the replica set."""
         return self.initial_sync_node
 
     def _new_mongod(self, index, replset_name):
         """Return a standalone.MongoDFixture configured to be used as replica-set member."""
-
+        mongod_executable = None if self.mixed_bin_versions is None else self.mixed_bin_versions[
+            index]
         mongod_logger = self._get_logger_for_mongod(index)
         mongod_options = self.mongod_options.copy()
         mongod_options["replSet"] = replset_name
@@ -442,7 +480,7 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
         mongod_options["set_parameters"] = mongod_options.get("set_parameters", {}).copy()
 
         return standalone.MongoDFixture(
-            mongod_logger, self.job_num, mongod_executable=self.mongod_executable,
+            mongod_logger, self.job_num, mongod_executable=mongod_executable,
             mongod_options=mongod_options, preserve_dbpath=self.preserve_dbpath)
 
     def _get_logger_for_mongod(self, index):
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index b3b744095ef..d29ec363700 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -23,11 +23,10 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
     _SHARD_REPLSET_NAME_PREFIX = "shard-rs"
 
     def __init__(  # pylint: disable=too-many-arguments,too-many-locals
-            self, logger, job_num, mongos_executable=None, mongos_options=None,
-            mongod_executable=None, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False,
-            num_shards=1, num_rs_nodes_per_shard=None, num_mongos=1, enable_sharding=None,
-            enable_balancer=True, enable_autosplit=True, auth_options=None, configsvr_options=None,
-            shard_options=None):
+            self, logger, job_num, mongos_executable=None, mongos_options=None, mongod_options=None,
+            dbpath_prefix=None, preserve_dbpath=False, num_shards=1, num_rs_nodes_per_shard=None,
+            num_mongos=1, enable_sharding=None, enable_balancer=True, enable_autosplit=True,
+            auth_options=None, configsvr_options=None, shard_options=None):
         """Initialize ShardedClusterFixture with different options for the cluster processes."""
 
         interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
@@ -37,7 +36,6 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
 
         self.mongos_executable = mongos_executable
         self.mongos_options = utils.default_if_none(mongos_options, {})
-        self.mongod_executable = mongod_executable
         self.mongod_options = utils.default_if_none(mongod_options, {})
         self.mongod_options["set_parameters"] = mongod_options.get("set_parameters", {}).copy()
         self.mongod_options["set_parameters"]["migrationLockAcquisitionMaxWaitMS"] = \
@@ -216,7 +214,6 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         configsvr_options = self.configsvr_options.copy()
 
         auth_options = configsvr_options.pop("auth_options", self.auth_options)
-        mongod_executable = configsvr_options.pop("mongod_executable", self.mongod_executable)
         preserve_dbpath = configsvr_options.pop("preserve_dbpath", self.preserve_dbpath)
         num_nodes = configsvr_options.pop("num_nodes", 1)
 
@@ -231,10 +228,9 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["storageEngine"] = "wiredTiger"
 
         return replicaset.ReplicaSetFixture(
-            mongod_logger, self.job_num, mongod_executable=mongod_executable,
-            mongod_options=mongod_options, preserve_dbpath=preserve_dbpath, num_nodes=num_nodes,
-            auth_options=auth_options, replset_config_options=replset_config_options,
-            **configsvr_options)
+            mongod_logger, self.job_num, mongod_options=mongod_options,
+            preserve_dbpath=preserve_dbpath, num_nodes=num_nodes, auth_options=auth_options,
+            replset_config_options=replset_config_options, **configsvr_options)
 
     def _new_rs_shard(self, index, num_rs_nodes_per_shard):
         """Return a replicaset.ReplicaSetFixture configured as a shard in a sharded cluster."""
@@ -244,7 +240,6 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         shard_options = self.shard_options.copy()
 
         auth_options = shard_options.pop("auth_options", self.auth_options)
-        mongod_executable = shard_options.pop("mongod_executable", self.mongod_executable)
         preserve_dbpath = shard_options.pop("preserve_dbpath", self.preserve_dbpath)
 
         replset_config_options = shard_options.pop("replset_config_options", {})
@@ -257,10 +252,10 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["replSet"] = ShardedClusterFixture._SHARD_REPLSET_NAME_PREFIX + str(index)
 
         return replicaset.ReplicaSetFixture(
-            mongod_logger, self.job_num, mongod_executable=mongod_executable,
-            mongod_options=mongod_options, preserve_dbpath=preserve_dbpath,
-            num_nodes=num_rs_nodes_per_shard, auth_options=auth_options,
-            replset_config_options=replset_config_options, **shard_options)
+            mongod_logger, self.job_num, mongod_options=mongod_options,
+            preserve_dbpath=preserve_dbpath, num_nodes=num_rs_nodes_per_shard,
+            auth_options=auth_options, replset_config_options=replset_config_options,
+            **shard_options)
 
     def _new_standalone_shard(self, index):
         """Return a standalone.MongoDFixture configured as a shard in a sharded cluster."""
@@ -269,7 +264,6 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
 
         shard_options = self.shard_options.copy()
 
-        mongod_executable = shard_options.pop("mongod_executable", self.mongod_executable)
         preserve_dbpath = shard_options.pop("preserve_dbpath", self.preserve_dbpath)
 
         mongod_options = self.mongod_options.copy()
@@ -277,9 +271,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongod_options["shardsvr"] = ""
         mongod_options["dbpath"] = os.path.join(self._dbpath_prefix, "shard{}".format(index))
 
-        return standalone.MongoDFixture(
-            mongod_logger, self.job_num, mongod_executable=mongod_executable,
-            mongod_options=mongod_options, preserve_dbpath=preserve_dbpath, **shard_options)
+        return standalone.MongoDFixture(mongod_logger, self.job_num, mongod_options=mongod_options,
+                                        preserve_dbpath=preserve_dbpath, **shard_options)
 
     def _new_mongos(self, index, total):
         """