SERVER-42776 Support mixed version test suites with ShardedClusterFixture

6 files changed, 104 insertions, 23 deletions

diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py
index 95aa585f400..781b7a8fd7e 100755
--- a/buildscripts/evergreen_gen_multiversion_tests.py
+++ b/buildscripts/evergreen_gen_multiversion_tests.py
@@ -32,10 +32,12 @@ REQUIRED_CONFIG_KEYS = {
 
 DEFAULT_CONFIG_VALUES = generate_resmoke.DEFAULT_CONFIG_VALUES
 CONFIG_DIR = DEFAULT_CONFIG_VALUES["generated_config_dir"]
+DEFAULT_CONFIG_VALUES["is_sharded"] = False
 TEST_SUITE_DIR = DEFAULT_CONFIG_VALUES["test_suites_dir"]
 CONFIG_FILE = generate_resmoke.CONFIG_FILE
 CONFIG_FORMAT_FN = generate_resmoke.CONFIG_FORMAT_FN
 MIXED_VERSION_CONFIGS = ["new-old-new", "new-new-old", "old-new-new"]
+SHARDED_MIXED_VERSION_CONFIGS = ["new-old-old-new"]
 
 
 def prepare_directory_for_suite(directory):
@@ -44,12 +46,43 @@ def prepare_directory_for_suite(directory):
         os.makedirs(directory)
 
 
-def update_suite_config_for_multiversion(suite_config):
-    """Update the suite_config with arguments for mixed version replica set testing."""
+def update_suite_config_for_multiversion_replset(suite_config):
+    """Update suite_config with arguments for multiversion tests using ReplicaSetFixture."""
     suite_config["executor"]["fixture"]["num_nodes"] = 3
     suite_config["executor"]["fixture"]["linear_chain"] = True
 
 
+def update_suite_config_for_multiversion_sharded(suite_config):
+    """Update suite_config with arguments for multiversion tests using ShardedClusterFixture."""
+
+    fixture_config = suite_config["executor"]["fixture"]
+    default_shards = "default_shards"
+    default_num_nodes = "default_nodes"
+    base_num_shards = (default_shards
+                       if "num_shards" not in fixture_config else fixture_config["num_shards"])
+    base_num_rs_nodes_per_shard = (default_num_nodes
+                                   if "num_rs_nodes_per_shard" not in fixture_config else
+                                   fixture_config["num_rs_nodes_per_shard"])
+
+    if base_num_shards is not default_shards or base_num_rs_nodes_per_shard is not default_num_nodes:
+        num_shard_num_nodes_pair = "{}-{}".format(base_num_shards, base_num_rs_nodes_per_shard)
+        assert num_shard_num_nodes_pair in {"default_shards-2"}, \
+               "The multiversion suite runs sharded clusters with 2 shards and 2 nodes per shard. "\
+               " acceptable, please add '{}' to this assert.".format(num_shard_num_nodes_pair)
+
+    suite_config["executor"]["fixture"]["num_shards"] = 2
+    suite_config["executor"]["fixture"]["num_rs_nodes_per_shard"] = 2
+
+
+class MultiversionConfig(object):
+    """An object containing the configurations to generate and run the multiversion tests with."""
+
+    def __init__(self, update_yaml, version_configs):
+        """Create new MultiversionConfig object."""
+        self.update_yaml = update_yaml
+        self.version_configs = version_configs
+
+
 class EvergreenConfigGenerator(object):
     """Generate evergreen configurations for multiversion tests."""
 
@@ -114,10 +147,15 @@ class EvergreenConfigGenerator(object):
         start_date = end_date - datetime.timedelta(days=generate_resmoke.LOOKBACK_DURATION_DAYS)
         suites = gen_suites.calculate_suites(start_date, end_date)
         # Render the given suites into yml files that can be used by resmoke.py.
+        if self.options.is_sharded:
+            config = MultiversionConfig(update_suite_config_for_multiversion_sharded,
+                                        SHARDED_MIXED_VERSION_CONFIGS)
+        else:
+            config = MultiversionConfig(update_suite_config_for_multiversion_replset,
+                                        MIXED_VERSION_CONFIGS)
         config_file_dict = generate_resmoke.render_suite_files(
-            suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR,
-            update_suite_config_for_multiversion)
-        for version_config in MIXED_VERSION_CONFIGS:
+            suites, self.options.suite, gen_suites.test_list, TEST_SUITE_DIR, config.update_yaml)
+        for version_config in config.version_configs:
             for suite in suites:
                 # Generate the newly divided test suites
                 source_suite = os.path.join(CONFIG_DIR, suite.name + ".yml")
@@ -155,8 +193,10 @@ def main(expansion_file, evergreen_config=None):
     """
     Create a configuration for generate tasks to create sub suites for the specified resmoke suite.
 
-    Tests will be generated to use 3 nodes and linear_chain=True.
-    The different binary version configurations tested are stored in MIXED_VERSION_CONFIGS.
+    Tests using ReplicaSetFixture will be generated to use 3 nodes and linear_chain=True.
+    Tests using ShardedClusterFixture will be generated to use 2 shards with 2 nodes each.
+    The different binary version configurations tested are stored in MIXED_VERSION_CONFIGS
+    and SHARDED_MIXED_VERSION_CONFIGS.
 
     The `--expansion-file` should contain all the configuration needed to generate the tasks.
     \f
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 67d127711e4..ef57ab66af7 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -288,7 +288,8 @@ def _make_parser():  # pylint: disable=too-many-statements
         "--mixedBinVersions", type="string", dest="mixed_bin_versions",
         metavar="version1-version2-..-versionN", help="Runs the test with the provided replica set"
         " binary version configuration. Specify 'old-new' to configure a replica set with a"
-        " 'last-stable' version primary and 'latest' version secondary.")
+        " 'last-stable' version primary and 'latest' version secondary. For a sharded cluster"
+        " with two shards and two replica set nodes each, specify 'old-new-old-new'.")
 
     evergreen_options = optparse.OptionGroup(
         parser, title=_EVERGREEN_OPTIONS_TITLE,
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index b5838f61a4e..ad75ed17114 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -30,6 +30,10 @@ class Fixture(object, metaclass=registry.make_registry_metaclass(_FIXTURES)):
     # is defined for all subclasses of Fixture.
     REGISTERED_NAME = "Fixture"
 
+    _LAST_STABLE_FCV = "4.2"
+    _LATEST_FCV = "4.4"
+    _LAST_STABLE_BIN_VERSION = "4.2"
+
     def __init__(self, logger, job_num, dbpath_prefix=None):
         """Initialize the fixture with a logger instance."""
 
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 36619eeac26..3a7a9e26bf8 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -22,11 +22,6 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
     # Error response codes copied from mongo/base/error_codes.err.
     _NODE_NOT_FOUND = 74
 
-    _LAST_STABLE_FCV = "4.2"
-    _LATEST_FCV = "4.4"
-
-    _LAST_STABLE_BIN_VERSION = "4.2"
-
     def __init__(  # pylint: disable=too-many-arguments, too-many-locals
             self, logger, job_num, mongod_options=None, dbpath_prefix=None, preserve_dbpath=False,
             num_nodes=2, start_initial_sync_node=False, write_concern_majority_journal_default=None,
@@ -56,13 +51,21 @@ class ReplicaSetFixture(interface.ReplFixture):  # pylint: disable=too-many-inst
             latest_mongod = mongod_executable
             last_stable_mongod = mongod_executable + "-" \
                                 + ReplicaSetFixture._LAST_STABLE_BIN_VERSION
-            self.mixed_bin_versions = [
-                latest_mongod if x == "new" else last_stable_mongod for x in self.mixed_bin_versions
-            ]
+            is_config_svr = "configsvr" in self.replset_config_options and self.replset_config_options[
+                "configsvr"]
+            if not is_config_svr:
+                self.mixed_bin_versions = [
+                    latest_mongod if (x == "new") else last_stable_mongod
+                    for x in self.mixed_bin_versions
+                ]
+            else:
+                # Our documented recommended path for upgrading shards lets us assume that config
+                # server secondaries will always be upgraded before the primary.
+                self.mixed_bin_versions = [last_stable_mongod, latest_mongod]
             num_versions = len(self.mixed_bin_versions)
-            if num_versions != num_nodes:
-                msg = (("The number of binary versions: {} do not match the number of nodes: "\
-                        "{}.")).format(num_versions, num_nodes)
+            if num_versions != num_nodes and not is_config_svr:
+                msg = (("The number of binary versions specified: {} do not match the number of"\
+                        " nodes in the replica set: {}.")).format(num_versions, num_nodes)
                 raise errors.ServerFailure(msg)
 
         # If voting_secondaries has not been set, set a default. By default, secondaries have zero
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 67991f0cb22..29325870284 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -26,7 +26,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
             self, logger, job_num, mongos_executable=None, mongos_options=None, mongod_options=None,
             dbpath_prefix=None, preserve_dbpath=False, num_shards=1, num_rs_nodes_per_shard=None,
             num_mongos=1, enable_sharding=None, enable_balancer=True, enable_autosplit=True,
-            auth_options=None, configsvr_options=None, shard_options=None):
+            auth_options=None, configsvr_options=None, shard_options=None, mixed_bin_versions=None):
         """Initialize ShardedClusterFixture with different options for the cluster processes."""
 
         interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
@@ -50,6 +50,14 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         self.auth_options = auth_options
         self.configsvr_options = utils.default_if_none(configsvr_options, {})
         self.shard_options = utils.default_if_none(shard_options, {})
+        self.mixed_bin_versions = utils.default_if_none(mixed_bin_versions,
+                                                        config.MIXED_BIN_VERSIONS)
+        if self.mixed_bin_versions is not None and num_rs_nodes_per_shard is not None:
+            num_mongods = self.num_shards * self.num_rs_nodes_per_shard
+            if len(self.mixed_bin_versions) != num_mongods:
+                msg = (("The number of binary versions specified: {} do not match the number of"\
+                        " nodes in the sharded cluster: {}.")).format(len(self.mixed_bin_versions), num_mongods)
+                raise errors.ServerFailure(msg)
 
         self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
@@ -250,7 +258,8 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         return replicaset.ReplicaSetFixture(
             mongod_logger, self.job_num, mongod_options=mongod_options,
             preserve_dbpath=preserve_dbpath, num_nodes=num_nodes, auth_options=auth_options,
-            replset_config_options=replset_config_options, **configsvr_options)
+            mixed_bin_versions=None, replset_config_options=replset_config_options,
+            **configsvr_options)
 
     def _new_rs_shard(self, index, num_rs_nodes_per_shard):
         """Return a replicaset.ReplicaSetFixture configured as a shard in a sharded cluster."""
@@ -265,6 +274,12 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         replset_config_options = shard_options.pop("replset_config_options", {})
         replset_config_options["configsvr"] = False
 
+        mixed_bin_versions = self.mixed_bin_versions
+        if mixed_bin_versions is not None:
+            start_index = index * num_rs_nodes_per_shard
+            mixed_bin_versions = mixed_bin_versions[start_index:start_index +
+                                                    num_rs_nodes_per_shard]
+
         mongod_options = self.mongod_options.copy()
         mongod_options.update(shard_options.pop("mongod_options", {}))
         mongod_options["shardsvr"] = ""
@@ -275,7 +290,7 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
             mongod_logger, self.job_num, mongod_options=mongod_options,
             preserve_dbpath=preserve_dbpath, num_nodes=num_rs_nodes_per_shard,
             auth_options=auth_options, replset_config_options=replset_config_options,
-            **shard_options)
+            mixed_bin_versions=mixed_bin_versions, **shard_options)
 
     def _new_standalone_shard(self, index):
         """Return a standalone.MongoDFixture configured as a shard in a sharded cluster."""
@@ -313,7 +328,13 @@ class ShardedClusterFixture(interface.Fixture):  # pylint: disable=too-many-inst
         mongos_options = self.mongos_options.copy()
         mongos_options["configdb"] = self.configsvr.get_internal_connection_string()
 
-        return _MongoSFixture(mongos_logger, self.job_num, mongos_executable=self.mongos_executable,
+        mongos_executable = utils.default_if_none(config.MONGOS_EXECUTABLE,
+                                                  config.DEFAULT_MONGOS_EXECUTABLE)
+        last_stable_executable = mongos_executable + "-" \
+                                + ShardedClusterFixture._LAST_STABLE_BIN_VERSION
+        mongos_executable = self.mongos_executable if self.mixed_bin_versions is None else last_stable_executable
+
+        return _MongoSFixture(mongos_logger, self.job_num, mongos_executable=mongos_executable,
                               mongos_options=mongos_options)
 
     def _add_shard(self, client, shard):
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 43bf9ffc6f4..11633f49774 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -7567,6 +7567,18 @@ tasks:
       resmoke_args: --storageEngine=wiredTiger
       fallback_num_sub_suites: 15
 
+- <<: *multiversion_template
+  name: sharded_causally_consistent_jscore_multiversion_passthrough_gen
+  tags: ["multiversion_passthrough"]
+  commands:
+  - func: "generate multiversion tasks"
+    vars:
+      suite: replica_sets_jscore_passthrough
+      resmoke_args: --storageEngine=wiredTiger
+      task_path_suffix: /data/multiversion
+      fallback_num_sub_suites: 4
+      is_sharded: true
+
 - name: retryable_writes_jscore_passthrough_gen
   tags: ["retry"]
   depends_on: