SERVER-67946 added downgrading->upgrading path sharding tests

1 files changed, 140 insertions, 13 deletions

diff --git a/jstests/multiVersion/genericSetFCVUsage/can_upgrade_FCV_after_failed_downgrade.js b/jstests/multiVersion/genericSetFCVUsage/can_upgrade_FCV_after_failed_downgrade.js
index 3dd726b3968..9d85302cf7b 100644
--- a/jstests/multiVersion/genericSetFCVUsage/can_upgrade_FCV_after_failed_downgrade.js
+++ b/jstests/multiVersion/genericSetFCVUsage/can_upgrade_FCV_after_failed_downgrade.js
@@ -1,24 +1,27 @@
 /**
  * Tests for the new fcv change path added:
- * kDowngradingFromLatestToLastLTS -> kUpgradingFromLastLTSToLatest -> kLatest
+ * kDowngradingFromLatestToLastLTS -> kUpgradingFromLastLTSToLatest -> kLatest.
+ *
+ * @tags: [featureFlagDowngradingToUpgrading]
  */
 
 (function() {
 "use strict";
 
 load("jstests/libs/fail_point_util.js");
+load("jstests/libs/feature_flag_util.js");
 
 const latest = "latest";
 
 function downgradingToUpgradingTest(conn, adminDB) {
-    // 1) startup: latest
+    // 1) Startup: latest version.
     let fcvDoc = adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
     jsTestLog("1) current FCV (should be latest):");
     printjson(fcvDoc);
     checkFCV(adminDB, latestFCV);
 
-    // 2) should be stuck in downgrading from latest to lastLTS
-    assert.commandWorked(  // failpoint: fail after transitional state
+    // 2) Should be stuck in downgrading from latest to lastLTS.
+    assert.commandWorked(  // failpoint: fail after transitional state.
         conn.adminCommand({configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
     assert.commandFailed(adminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
     fcvDoc = adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
@@ -26,7 +29,7 @@ function downgradingToUpgradingTest(conn, adminDB) {
     printjson(fcvDoc);
     checkFCV(adminDB, lastLTSFCV, lastLTSFCV);
 
-    // 3) fcv should be set to latest
+    // 3) FCV should be set to latest.
     assert.commandWorked(conn.adminCommand({configureFailPoint: 'failDowngrading', mode: "off"}));
     assert.commandWorked(adminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
     let newFcvDoc = adminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
@@ -34,7 +37,7 @@ function downgradingToUpgradingTest(conn, adminDB) {
     printjson(fcvDoc);
     checkFCV(adminDB, latestFCV);
 
-    // if config server, also check that timestamp from fcvDoc should be "earlier" (return -1)
+    // If config server, also check that timestamp from fcvDoc should be "earlier" (return -1).
     if (fcvDoc.changeTimestamp != null) {
         assert(timestampCmp(fcvDoc.changeTimestamp, newFcvDoc.changeTimestamp) == -1);
     }
@@ -46,6 +49,12 @@ function runStandaloneTest() {
         null, conn, "mongod was unable to start up with version=" + latest + " and no data files");
     const adminDB = conn.getDB("admin");
 
+    if (!FeatureFlagUtil.isEnabled(adminDB, "DowngradingToUpgrading")) {
+        jsTestLog("Skipping as featureFlagDowngradingToUpgrading is not enabled");
+        MongoRunner.stopMongod(conn);
+        return;
+    }
+
     downgradingToUpgradingTest(conn, adminDB);
 
     MongoRunner.stopMongod(conn);
@@ -58,12 +67,18 @@ function runReplicaSetTest() {
     const primaryAdminDB = rst.getPrimary().getDB("admin");
     const primary = rst.getPrimary();
 
+    if (!FeatureFlagUtil.isEnabled(primaryAdminDB, "DowngradingToUpgrading")) {
+        jsTestLog("Skipping as featureFlagDowngradingToUpgrading is not enabled");
+        rst.stopSet();
+        return;
+    }
+
     downgradingToUpgradingTest(primary, primaryAdminDB);
 
     rst.stopSet();
 }
 
-function runShardingTestTimestamp() {
+function testConfigServerFCVTimestampIsAlwaysNewer() {
     const st =
         new ShardingTest({shards: {rs0: {nodes: [{binVersion: latest}, {binVersion: latest}]}}});
     const mongosAdminDB = st.s.getDB("admin");
@@ -71,12 +86,18 @@ function runShardingTestTimestamp() {
     const shardPrimaryAdminDB = st.rs0.getPrimary().getDB("admin");
     checkFCV(shardPrimaryAdminDB, latestFCV);
 
+    if (!FeatureFlagUtil.isEnabled(mongosAdminDB, "DowngradingToUpgrading")) {
+        jsTestLog("Skipping as featureFlagDowngradingToUpgrading is not enabled");
+        st.stop();
+        return;
+    }
+
     downgradingToUpgradingTest(configPrimary, mongosAdminDB);
 
-    /* check that a new timestamp is always generated with each setFCV call */
+    // Check that a new timestamp is always generated with each setFCV call.
     let fcvDoc;
     let newFcvDoc;
-    // 1) calling downgrade twice (one with failpoint)
+    // 1) Calling downgrade twice (one with failpoint).
     assert.commandWorked(
         configPrimary.adminCommand({configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
     assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
@@ -86,10 +107,10 @@ function runShardingTestTimestamp() {
     assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
     newFcvDoc = mongosAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
     checkFCV(mongosAdminDB, lastLTSFCV);
-    // timestamp from fcvDoc should be "earlier" (return -1)
+    // Timestamp from fcvDoc should be less than the timestamp from newFcvDoc.
     assert(timestampCmp(fcvDoc.changeTimestamp, newFcvDoc.changeTimestamp) == -1);
 
-    // 2) calling upgrade twice (one with failpoint)
+    // 2) Calling upgrade twice (one with failpoint).
     assert.commandWorked(
         configPrimary.adminCommand({configureFailPoint: 'failUpgrading', mode: "alwaysOn"}));
     assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
@@ -99,13 +120,119 @@ function runShardingTestTimestamp() {
     assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
     newFcvDoc = mongosAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
     checkFCV(mongosAdminDB, latestFCV);
-    // timestamp from fcvDoc should be "earlier" (return -1)
+    // Timestamp from fcvDoc should be less than the timestamp from newFcvDoc.
     assert(timestampCmp(fcvDoc.changeTimestamp, newFcvDoc.changeTimestamp) == -1);
 
     st.stop();
 }
 
+// Helper for runShardingTest(): check that a sharded cluster can upgrade to latest fcv and
+// assert all servers/router in the cluster have the latest fcv.
+function setFCVToLatestSharding(
+    mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB) {
+    assert.commandWorked(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: latestFCV}));
+    checkFCV(mongosAdminDB, latestFCV);
+    checkFCV(configPrimaryAdminDB, latestFCV);
+    checkFCV(shard0PrimaryAdminDB, latestFCV);
+    checkFCV(shard1PrimaryAdminDB, latestFCV);
+}
+
+// Tests downgrading->upgrading on different failing scenarios for configServer or shards.
+function runShardingTest() {
+    const st = new ShardingTest({shards: 2, mongos: 1, config: 1});
+    const mongosAdminDB = st.s.getDB("admin");
+    const configPrimary = st.configRS.getPrimary();
+    const configPrimaryAdminDB = configPrimary.getDB("admin");
+    const shard0Primary = st.rs0.getPrimary();
+    const shard0PrimaryAdminDB = shard0Primary.getDB("admin");
+    const shard1Primary = st.rs1.getPrimary();
+    const shard1PrimaryAdminDB = shard1Primary.getDB("admin");
+
+    // Make sure all servers start as latest version.
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    // 1. Test downgrading to upgraded where both shards failed initial downgrade.
+    assert.commandWorked(
+        shard0Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
+    assert.commandWorked(
+        shard1Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
+    assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    checkFCV(configPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard0PrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard1PrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+
+    assert.commandWorked(
+        shard0Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "off"}));
+    assert.commandWorked(
+        shard1Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "off"}));
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    // 2. Test downgrading to upgraded with config in downgrading, both shards in upgraded.
+    assert.commandWorked(configPrimary.adminCommand(
+        {configureFailPoint: 'failBeforeSendingShardsToDowngrading', mode: "alwaysOn"}));
+    assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    checkFCV(configPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard0PrimaryAdminDB, latestFCV);
+    checkFCV(shard1PrimaryAdminDB, latestFCV);
+
+    assert.commandWorked(configPrimary.adminCommand(
+        {configureFailPoint: 'failBeforeSendingShardsToDowngrading', mode: "off"}));
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    // 3. Test downgrading to upgraded path with 1 shard in downgrading + 1 shard in upgraded
+    // (specifically: config and shard0 in downgrading, shard1 in upgraded/latest).
+    assert.commandWorked(
+        shard0Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
+    assert.commandWorked(shard1Primary.adminCommand(
+        {configureFailPoint: 'failBeforeTransitioning', mode: "alwaysOn"}));
+    assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    checkFCV(configPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard0PrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard1PrimaryAdminDB, latestFCV);
+
+    assert.commandWorked(
+        shard0Primary.adminCommand({configureFailPoint: 'failDowngrading', mode: "off"}));
+    assert.commandWorked(
+        shard1Primary.adminCommand({configureFailPoint: 'failBeforeTransitioning', mode: "off"}));
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    // 4. Test downgrading to upgraded where both shards succeed initial downgrade but config
+    // fails.
+    assert.commandWorked(configPrimary.adminCommand(
+        {configureFailPoint: 'failBeforeUpdatingFcvDoc', mode: "alwaysOn"}));
+    assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    checkFCV(configPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard0PrimaryAdminDB, lastLTSFCV);
+    checkFCV(shard1PrimaryAdminDB, lastLTSFCV);
+
+    assert.commandWorked(
+        configPrimary.adminCommand({configureFailPoint: 'failBeforeUpdatingFcvDoc', mode: "off"}));
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    // 5. Test downgrading to upgraded where only one shard failed initial downgrade
+    // (specifically: shard0 in downgraded, shard1 + config in downgrading).
+    assert.commandWorked(shard1PrimaryAdminDB.adminCommand(
+        {configureFailPoint: 'failDowngrading', mode: "alwaysOn"}));
+    assert.commandFailed(mongosAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+    checkFCV(configPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+    checkFCV(shard0PrimaryAdminDB, lastLTSFCV);
+    checkFCV(shard1PrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+
+    assert.commandWorked(
+        shard1PrimaryAdminDB.adminCommand({configureFailPoint: 'failDowngrading', mode: "off"}));
+    setFCVToLatestSharding(
+        mongosAdminDB, configPrimaryAdminDB, shard0PrimaryAdminDB, shard1PrimaryAdminDB);
+
+    st.stop();
+}
+
 runStandaloneTest();
 runReplicaSetTest();
-runShardingTestTimestamp();
+testConfigServerFCVTimestampIsAlwaysNewer();
+runShardingTest();
 })();