SERVER-42780 Create mixed version sharding test suite

4 files changed, 84 insertions, 12 deletions

diff --git a/buildscripts/resmokeconfig/suites/sharding_multiversion.yml b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
new file mode 100644
index 00000000000..eff347f2ba2
--- /dev/null
+++ b/buildscripts/resmokeconfig/suites/sharding_multiversion.yml
@@ -0,0 +1,15 @@
+test_kind: js_test
+
+selector:
+  roots:
+  - jstests/sharding/*.js
+
+executor:
+  config:
+    shell_options:
+      nodb: ''
+      readMode: commands
+      global_vars:
+        TestData:
+          randomBinVersions: true
+          mongosBinVersion: 'last-stable'
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 19398ff4be9..2f2c96dca1d 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -7417,6 +7417,16 @@ tasks:
       resmoke_args: --storageEngine=wiredTiger
       fallback_num_sub_suites: 32
 
+- name: sharding_multiversion_gen
+  tags: ["random_multiversion_replica_sets"]
+  commands:
+  - func: "generate resmoke tasks"
+    vars:
+      use_large_distro: "true"
+      resmoke_args: --storageEngine=wiredTiger --dryRun=tests
+      fallback_num_sub_suites: 32
+      use_multiversion: /data/multiversion
+
 - name: sharding_rs_matching_disabled_gen
   tags: ["sharding", "common"]
   commands:
diff --git a/src/mongo/shell/replsettest.js b/src/mongo/shell/replsettest.js
index 023ae601a81..07c947d8e22 100644
--- a/src/mongo/shell/replsettest.js
+++ b/src/mongo/shell/replsettest.js
@@ -62,6 +62,10 @@
  *     settings {object}: Setting used in the replica set config document.
  *        Example:
  *              settings: { chainingAllowed: false, ... }
+ *
+ *     seedRandomNumberGenerator {boolean}: Indicates whether the random number generator should
+ *        be seeded when randomBinVersions is true. For ReplSetTests started by ShardingTest, the
+ *        seed is generated as part of ShardingTest.
  *   }
  *
  * Member variables:
@@ -579,9 +583,11 @@ var ReplSetTest = function(opts) {
 
         var nodes = [];
 
-        if (jsTest.options().useRandomBinVersionsWithinReplicaSet) {
+        if (jsTest.options().useRandomBinVersionsWithinReplicaSet &&
+            self.seedRandomNumberGenerator) {
             // Set the random seed to the value passed in by TestData. The seed is undefined
-            // by default.
+            // by default. For sharded clusters, the seed is already initialized as part of
+            // ShardingTest.
             Random.setRandomSeed(jsTest.options().seed);
         }
 
@@ -1035,18 +1041,36 @@ var ReplSetTest = function(opts) {
         // nodes are subsequently added to the set, since such nodes cannot set their FCV to
         // "latest". Therefore, we make sure the primary is "last-stable" FCV before adding in
         // nodes of different binary versions to the replica set.
-        let isMultiversion = false;
+        let lastStableBinVersionWasSpecifiedForSomeNode = false;
         Object.keys(this.nodeOptions).forEach(function(key, index) {
             let val = self.nodeOptions[key];
             if (typeof (val) === "object" && val.hasOwnProperty("binVersion") &&
                 MongoRunner.areBinVersionsTheSame(val.binVersion, lastStableFCV)) {
-                isMultiversion = true;
+                lastStableBinVersionWasSpecifiedForSomeNode = true;
             }
         });
-        if (isMultiversion || jsTest.options().useRandomBinVersionsWithinReplicaSet) {
-            assert.commandWorked(
-                self.getPrimary().adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
-            checkFCV(self.getPrimary().getDB("admin"), lastStableFCV);
+
+        // Set the FCV to 'last-stable' if we are running a mixed version replica set. If this is a
+        // config server, the FCV will be set as part of ShardingTest.
+        let setLastStableFCV = (lastStableBinVersionWasSpecifiedForSomeNode ||
+                                jsTest.options().useRandomBinVersionsWithinReplicaSet) &&
+            !self.isConfigServer;
+        if (setLastStableFCV && jsTest.options().replSetFeatureCompatibilityVersion) {
+            throw new Error(
+                "The FCV will be set to 'last-stable' automatically when starting up a replica " +
+                "set with mixed binary versions. Therefore, we expect an empty value for " +
+                "'replSetFeatureCompatibilityVersion'.");
+        }
+
+        if (setLastStableFCV) {
+            // Authenticate before running the command.
+            asCluster(self.nodes, function setFCV() {
+                let fcv = lastStableFCV;
+                print("Setting feature compatibility version for replica set to '" + fcv + "'");
+                assert.commandWorked(
+                    self.getPrimary().adminCommand({setFeatureCompatibilityVersion: fcv}));
+                checkFCV(self.getPrimary().getDB("admin"), lastStableFCV);
+            });
         }
 
         // Reconfigure the set to contain the correct number of nodes (if necessary).
@@ -2382,8 +2406,14 @@ var ReplSetTest = function(opts) {
         delete options.rsConfig;
 
         if (jsTest.options().useRandomBinVersionsWithinReplicaSet) {
-            const rand = Random.rand();
-            options.binVersion = rand < 0.5 ? "latest" : "last-stable";
+            if (self.isConfigServer) {
+                // Our documented upgrade/downgrade paths for a sharded cluster lets us assume that
+                // config server nodes will always be fully upgraded before the shard nodes.
+                options.binVersion = "latest";
+            } else {
+                const rand = Random.rand();
+                options.binVersion = rand < 0.5 ? "latest" : "last-stable";
+            }
             print("Randomly assigned binary version: " + options.binVersion + " to node: " + n);
         }
 
@@ -2730,6 +2760,9 @@ var ReplSetTest = function(opts) {
         self.protocolVersion = opts.protocolVersion;
         self.waitForKeys = opts.waitForKeys;
 
+        self.seedRandomNumberGenerator = opts.seedRandomNumberGenerator || true;
+        self.isConfigServer = opts.isConfigServer;
+
         _useBridge = opts.useBridge || false;
         _bridgeOptions = opts.bridgeOptions || {};
 
diff --git a/src/mongo/shell/shardingtest.js b/src/mongo/shell/shardingtest.js
index c52eb0807ef..a87e1bfc57b 100644
--- a/src/mongo/shell/shardingtest.js
+++ b/src/mongo/shell/shardingtest.js
@@ -1198,6 +1198,16 @@ var ShardingTest = function(params) {
     otherParams.migrationLockAcquisitionMaxWaitMS =
         otherParams.migrationLockAcquisitionMaxWaitMS || 30000;
 
+    let randomSeedAlreadySet = false;
+
+    if (jsTest.options().randomBinVersions) {
+        // We avoid setting the random seed unequivocally to avoid unexpected behavior in tests
+        // that already make use of Random.setRandomSeed(). This conditional can be removed if
+        // it becomes the standard to always be generating the seed through ShardingTest.
+        Random.setRandomSeed(jsTest.options().seed);
+        randomSeedAlreadySet = true;
+    }
+
     // Start the MongoD servers (shards)
     let startTime = new Date();  // Measure the execution time of startup and initiate.
     for (var i = 0; i < numShards; i++) {
@@ -1276,7 +1286,8 @@ var ShardingTest = function(params) {
                 keyFile: keyFile,
                 protocolVersion: protocolVersion,
                 waitForKeys: false,
-                settings: rsSettings
+                settings: rsSettings,
+                seedRandomNumberGenerator: !randomSeedAlreadySet,
             });
 
             print("ShardingTest starting replica set for shard: " + setName);
@@ -1410,6 +1421,8 @@ var ShardingTest = function(params) {
         keyFile: keyFile,
         waitForKeys: false,
         name: testName + "-configRS",
+        seedRandomNumberGenerator: !randomSeedAlreadySet,
+        isConfigServer: true,
     };
 
     // when using CSRS, always use wiredTiger as the storage engine
@@ -1483,7 +1496,8 @@ var ShardingTest = function(params) {
     const configRS = this.configRS;
     if (_hasNewFeatureCompatibilityVersion() && _isMixedVersionCluster()) {
         function setFeatureCompatibilityVersion() {
-            assert.commandWorked(csrsPrimary.adminCommand({setFeatureCompatibilityVersion: '4.2'}));
+            assert.commandWorked(
+                csrsPrimary.adminCommand({setFeatureCompatibilityVersion: lastStableFCV}));
 
             // Wait for the new featureCompatibilityVersion to propagate to all nodes in the CSRS
             // to ensure that older versions of mongos can successfully connect.