SERVER-68143 Add rollback tests for downgrading to upgraded path

3 files changed, 349 insertions, 30 deletions

diff --git a/jstests/multiVersion/genericSetFCVUsage/rollback_with_node_in_downgrading.js b/jstests/multiVersion/genericSetFCVUsage/rollback_with_node_in_downgrading.js
new file mode 100644
index 00000000000..87a4e1e2ae1
--- /dev/null
+++ b/jstests/multiVersion/genericSetFCVUsage/rollback_with_node_in_downgrading.js
@@ -0,0 +1,20 @@
+/**
+ * Multiversion rollback test. Checks that rollback succeeds between a
+ * latest version rollback node and a downgrading version sync source, and a
+ * downgrading version rollback node and a lastLTS version sync source.
+ */
+
+(function() {
+"use strict";
+load("jstests/multiVersion/libs/multiversion_rollback.js");
+
+let testName = "multiversion_rollback_latest_from_downgrading";
+jsTestLog("Testing multiversion rollback with a node in latest syncing from a node in downgrading");
+testMultiversionRollbackLatestFromDowngrading(testName, true /* upgradeImmediately */);
+testMultiversionRollbackLatestFromDowngrading(testName, false /* upgradeImmediately */);
+
+testName = "multiversion_rollback_downgrading_from_last_lts";
+jsTestLog(
+    "Testing multiversion rollback with a node in downgrading syncing from a node in lastLTS");
+testMultiversionRollbackDowngradingFromLastLTS(testName);
+})();
diff --git a/jstests/multiVersion/libs/multiversion_rollback.js b/jstests/multiVersion/libs/multiversion_rollback.js
index 9263052a956..37a7abadbfb 100644
--- a/jstests/multiVersion/libs/multiversion_rollback.js
+++ b/jstests/multiVersion/libs/multiversion_rollback.js
@@ -9,6 +9,41 @@
 
 load("jstests/replsets/libs/rollback_test.js");
 load("jstests/libs/collection_drop_recreate.js");
+load('jstests/libs/parallel_shell_helpers.js');
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/feature_flag_util.js");
+
+function printFCVDoc(nodeAdminDB, logMessage) {
+    const fcvDoc = nodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(logMessage + ` ${tojson(fcvDoc)}`);
+}
+
+function CommonOps(dbName, node) {
+    // Insert four documents on both nodes.
+    assert.commandWorked(node.getDB(dbName)["bothNodesKeep"].insert({a: 1}));
+    assert.commandWorked(node.getDB(dbName)["rollbackNodeDeletes"].insert({b: 1}));
+    assert.commandWorked(node.getDB(dbName)["rollbackNodeUpdates"].insert({c: 1}));
+    assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].insert({d: 1}));
+}
+
+function RollbackOps(dbName, node) {
+    // Perform operations only on the rollback node:
+    //   1. Delete a document.
+    //   2. Update a document only on this node.
+    //   3. Update a document on both nodes.
+    // All three documents will be refetched during rollback.
+    assert.commandWorked(node.getDB(dbName)["rollbackNodeDeletes"].remove({b: 1}));
+    assert.commandWorked(node.getDB(dbName)["rollbackNodeUpdates"].update({c: 1}, {c: 0}));
+    assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].update({d: 1}, {d: 0}));
+}
+
+function SyncSourceOps(dbName, node) {
+    // Perform operations only on the sync source:
+    //   1. Make a conflicting write on one of the documents the rollback node updates.
+    //   2. Insert a new document.
+    assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].update({d: 1}, {d: 2}));
+    assert.commandWorked(node.getDB(dbName)["syncSourceInserts"].insert({e: 1}));
+}
 
 /**
  * Executes and validates rollback between a pair of nodes with the given versions.
@@ -24,48 +59,21 @@ function testMultiversionRollback(testName, rollbackNodeVersion, syncSourceVersi
 
     let dbName = testName;
 
-    let CommonOps = (node) => {
-        // Insert four documents on both nodes.
-        assert.commandWorked(node.getDB(dbName)["bothNodesKeep"].insert({a: 1}));
-        assert.commandWorked(node.getDB(dbName)["rollbackNodeDeletes"].insert({b: 1}));
-        assert.commandWorked(node.getDB(dbName)["rollbackNodeUpdates"].insert({c: 1}));
-        assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].insert({d: 1}));
-    };
-
-    let RollbackOps = (node) => {
-        // Perform operations only on the rollback node:
-        //   1. Delete a document.
-        //   2. Update a document only on this node.
-        //   3. Update a document on both nodes.
-        // All three documents will be refetched during rollback.
-        assert.commandWorked(node.getDB(dbName)["rollbackNodeDeletes"].remove({b: 1}));
-        assert.commandWorked(node.getDB(dbName)["rollbackNodeUpdates"].update({c: 1}, {c: 0}));
-        assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].update({d: 1}, {d: 0}));
-    };
-
-    let SyncSourceOps = (node) => {
-        // Perform operations only on the sync source:
-        //   1. Make a conflicting write on one of the documents the rollback node updates.
-        //   2. Insert a new document.
-        assert.commandWorked(node.getDB(dbName)["bothNodesUpdate"].update({d: 1}, {d: 2}));
-        assert.commandWorked(node.getDB(dbName)["syncSourceInserts"].insert({e: 1}));
-    };
-
     // Set up replica set.
     let replSet = setupReplicaSet(testName, rollbackNodeVersion, syncSourceVersion);
 
     // Set up Rollback Test.
     let rollbackTest = new RollbackTest(testName, replSet);
 
-    CommonOps(rollbackTest.getPrimary());
+    CommonOps(dbName, rollbackTest.getPrimary());
 
     // Perform operations that will be rolled back.
     let rollbackNode = rollbackTest.transitionToRollbackOperations();
-    RollbackOps(rollbackNode);
+    RollbackOps(dbName, rollbackNode);
 
     // Perform different operations only on the sync source.
     let syncSource = rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
-    SyncSourceOps(syncSource);
+    SyncSourceOps(dbName, syncSource);
 
     // Wait for rollback to finish.
     rollbackTest.transitionToSyncSourceOperationsDuringRollback();
@@ -74,6 +82,194 @@ function testMultiversionRollback(testName, rollbackNodeVersion, syncSourceVersi
     rollbackTest.stop();
 }
 
+// Test rollback between latest rollback node and downgrading sync node.
+function testMultiversionRollbackLatestFromDowngrading(testName, upgradeImmediately) {
+    const dbName = testName;
+    const replSet = new ReplSetTest(
+        {name: testName, nodes: 3, useBridge: true, settings: {chainingAllowed: false}});
+
+    replSet.startSet();
+    replSet.initiateWithHighElectionTimeout();
+
+    const config = replSet.getReplSetConfigFromNode();
+    config.members[1].priority = 0;
+    reconfig(replSet, config, true);
+
+    const rollbackTest = new RollbackTest(testName, replSet);
+
+    const primary = rollbackTest.getPrimary();
+    const primaryAdminDB = primary.getDB("admin");
+    const secondary = rollbackTest.getSecondary();
+    const secondaryAdminDB = secondary.getDB("admin");
+
+    printFCVDoc(primaryAdminDB, "Primary's FCV at start: ");
+    checkFCV(primaryAdminDB, latestFCV);
+    printFCVDoc(secondaryAdminDB, "Secondary's FCV at start: ");
+    checkFCV(secondaryAdminDB, latestFCV);
+
+    CommonOps(dbName, rollbackTest.getPrimary());
+
+    printFCVDoc(primaryAdminDB, "Primary's FCV before RollbackOps: ");
+    checkFCV(primaryAdminDB, latestFCV);
+    printFCVDoc(secondaryAdminDB, "Secondary's FCV before RollbackOps: ");
+    checkFCV(secondaryAdminDB, latestFCV);
+
+    const rollbackNode = rollbackTest.transitionToRollbackOperations();
+    const rollbackAdminDB = rollbackNode.getDB("admin");
+
+    // Do some operations to be rolled back.
+    RollbackOps(dbName, rollbackNode);
+
+    const syncSource = rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    const syncSourceAdminDB = syncSource.getDB("admin");
+
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV before SyncSourceOps: ");
+    checkFCV(rollbackAdminDB, latestFCV);
+    printFCVDoc(syncSourceAdminDB, "Sync source's FCV before SyncSourceOps: ");
+    checkFCV(syncSourceAdminDB, latestFCV);
+
+    // Restart server replication on the tiebreaker node so that we can replicate the FCV change.
+    let tiebreaker = rollbackTest.getTieBreaker();
+    restartServerReplication(tiebreaker);
+
+    jsTestLog("Starting SyncSourceOps");
+    SyncSourceOps(dbName, syncSource);
+    jsTestLog("Setting sync source FCV to downgrading to lastLTS");
+
+    // Set the failpoint so that downgrading will fail.
+    assert.commandWorked(
+        syncSource.adminCommand({configureFailPoint: "failDowngrading", mode: "alwaysOn"}));
+
+    assert.commandFailed(syncSource.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+    // Sync source's FCV should be in downgrading to lastLTS while the rollback node's FCV should be
+    // in latest.
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV after SyncSourceOps: ");
+    checkFCV(rollbackAdminDB, latestFCV);
+    printFCVDoc(syncSourceAdminDB, "Sync source's FCV after SyncSourceOps: ");
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // Rollback node should now have synced from the sync source and its FCV should be downgrading
+    // to lastLTS.
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV after rolling back: ");
+    checkFCV(rollbackAdminDB, lastLTSFCV, lastLTSFCV);
+
+    const newPrimary = rollbackTest.getPrimary();
+    const newPrimaryAdminDB = newPrimary.getDB("admin");
+
+    printFCVDoc(newPrimaryAdminDB, "New primary's FCV after rolling back: ");
+    checkFCV(newPrimaryAdminDB, lastLTSFCV, lastLTSFCV);
+
+    if (upgradeImmediately &&
+        FeatureFlagUtil.isEnabled(newPrimaryAdminDB,
+                                  "DowngradingToUpgrading",
+                                  null /* user not specified */,
+                                  true /* ignores FCV */)) {
+        // We can upgrade immediately.
+        assert.commandWorked(newPrimary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+        printFCVDoc(newPrimaryAdminDB, "New primary's FCV after completing upgrade: ");
+        checkFCV(newPrimaryAdminDB, latestFCV);
+    } else {
+        // We can finish downgrading.
+        assert.commandWorked(
+            newPrimary.adminCommand({configureFailPoint: "failDowngrading", mode: "off"}));
+        assert.commandWorked(newPrimary.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+        printFCVDoc(newPrimaryAdminDB, "New primary's FCV after completing downgrade: ");
+        checkFCV(newPrimaryAdminDB, lastLTSFCV);
+    }
+
+    rollbackTest.stop();
+}
+
+// Test rollback between downgrading rollback node and lastLTS sync node.
+function testMultiversionRollbackDowngradingFromLastLTS(testName) {
+    const dbName = testName;
+    const replSet = new ReplSetTest(
+        {name: testName, nodes: 3, useBridge: true, settings: {chainingAllowed: false}});
+
+    replSet.startSet();
+    replSet.initiateWithHighElectionTimeout();
+
+    const config = replSet.getReplSetConfigFromNode();
+    config.members[1].priority = 0;
+    reconfig(replSet, config, true);
+
+    const rollbackTest = new RollbackTest(testName, replSet);
+
+    const primary = rollbackTest.getPrimary();
+    const primaryAdminDB = primary.getDB("admin");
+    const secondary = rollbackTest.getSecondary();
+    const secondaryAdminDB = secondary.getDB("admin");
+
+    printFCVDoc(primaryAdminDB, "Primary's FCV at start: ");
+    checkFCV(primaryAdminDB, latestFCV);
+    printFCVDoc(secondaryAdminDB, "Secondary's FCV at start: ");
+    checkFCV(secondaryAdminDB, latestFCV);
+
+    // Execute common operations, and make the set get into downgrading to lastLTS.
+    CommonOps(dbName, rollbackTest.getPrimary());
+
+    // Set the failpoint so that downgrading will fail.
+    assert.commandWorked(
+        primary.adminCommand({configureFailPoint: "failDowngrading", mode: "alwaysOn"}));
+    assert.commandFailed(primary.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+    printFCVDoc(primaryAdminDB, "Primary's FCV before RollbackOps: ");
+    checkFCV(primaryAdminDB, lastLTSFCV, lastLTSFCV);
+    printFCVDoc(secondaryAdminDB, "Secondary's FCV before RollbackOps: ");
+    checkFCV(secondaryAdminDB, lastLTSFCV, lastLTSFCV);
+
+    const rollbackNode = rollbackTest.transitionToRollbackOperations();
+    const rollbackAdminDB = rollbackNode.getDB("admin");
+
+    // Do some operations to be rolled back.
+    RollbackOps(dbName, rollbackNode);
+
+    const syncSource = rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    const syncSourceAdminDB = syncSource.getDB("admin");
+
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV before SyncSourceOps: ");
+    checkFCV(rollbackAdminDB, lastLTSFCV, lastLTSFCV);
+    printFCVDoc(syncSourceAdminDB, "Sync source's FCV before SyncSourceOps: ");
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    // Restart server replication on the tiebreaker node so that we can replicate the FCV change.
+    let tiebreaker = rollbackTest.getTieBreaker();
+    restartServerReplication(tiebreaker);
+
+    jsTestLog("Starting SyncSourceOps");
+    SyncSourceOps(dbName, syncSource);
+    jsTestLog("Setting sync source FCV to lastLTS");
+    assert.commandWorked(syncSource.adminCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+    // Sync source's FCV should be in lastLTS while the rollback node's FCV should be in downgrading
+    // to lastLTS.
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV after sync source ops: ");
+    checkFCV(rollbackAdminDB, lastLTSFCV, lastLTSFCV);
+    printFCVDoc(syncSourceAdminDB, "Sync source's FCV after sync source ops: ");
+    checkFCV(syncSourceAdminDB, lastLTSFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // Rollback node should now have synced from the sync source and its FCV should be lastLTS.
+    printFCVDoc(rollbackAdminDB, "Rollback node's FCV after rolling back: ");
+    checkFCV(rollbackAdminDB, lastLTSFCV);
+
+    const newPrimaryAdminDB = rollbackTest.getPrimary().getDB("admin");
+    printFCVDoc(newPrimaryAdminDB, "New primary's FCV after rolling back: ");
+    checkFCV(newPrimaryAdminDB, lastLTSFCV);
+
+    rollbackTest.stop();
+}
+
 /**
  * Sets up a multiversion replica set.
  *
diff --git a/jstests/replsets/rollback_set_fcv.js b/jstests/replsets/rollback_set_fcv.js
index 4a82ed33857..388f603dfb6 100644
--- a/jstests/replsets/rollback_set_fcv.js
+++ b/jstests/replsets/rollback_set_fcv.js
@@ -137,6 +137,106 @@ function rollbackFCVFromDowngradedOrUpgraded(fromFCV, toFCV, failPoint) {
     }
 }
 
+// Test rolling back from upgrading to downgrading.
+// Start off with downgrading from latest to lastLTS.
+// Go to upgrading from lastLTS to latest state.
+// Rollback and make sure the FCV doc is back in the downgrading from latest to lastLTS state.
+function rollbackFCVFromUpgradingToDowngrading() {
+    let fcvDoc;
+    const rollbackNode = rollbackTest.getPrimary();
+    const syncSource = rollbackTest.getSecondary();
+    const rollbackNodeAdminDB = rollbackNode.getDB('admin');
+    const syncSourceAdminDB = syncSource.getDB('admin');
+
+    // Ensure the cluster starts at the correct FCV.
+    assert.commandWorked(rollbackNode.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+    if (!FeatureFlagUtil.isEnabled(rollbackNodeAdminDB, "DowngradingToUpgrading")) {
+        jsTestLog(
+            "Skipping rollbackFCVFromUpgradingToDowngrading as featureFlagDowngradingToUpgrading is not enabled");
+        return;
+    }
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version at start: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, latestFCV);
+
+    // Set the failpoints so that both upgrading and downgrading would fail.
+    assert.commandWorked(
+        rollbackNode.adminCommand({configureFailPoint: "failDowngrading", mode: "alwaysOn"}));
+    assert.commandWorked(
+        rollbackNode.adminCommand({configureFailPoint: "failUpgrading", mode: "alwaysOn"}));
+
+    // Go to downgrading state (downgrading from latest to lastLTS).
+    assert.commandFailed(
+        rollbackNodeAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version after downgrading: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, lastLTSFCV);
+
+    // Wait until the config has propagated to the other nodes and the rollbackNode has learned of
+    // it, so that the config replication check in 'setFeatureCompatibilityVersion' is satisfied.
+    // This is only important since 'setFeatureCompatibilityVersion' is known to implicitly call
+    // internal reconfigs as part of upgrade/downgrade behavior.
+    rollbackTest.getTestFixture().waitForConfigReplication(rollbackNode);
+    // Wait for the majority commit point to be updated on the sync source, because checkFCV calls
+    // getParameter for the featureCompatibilityVersion, which will wait until the FCV change makes
+    // it into the node's majority committed snapshot.
+    rollbackTest.getTestFixture().awaitLastOpCommitted(undefined /* timeout */, [syncSource]);
+
+    // test rolling back from upgrading to downgrading
+    jsTestLog("Testing rolling back FCV from {version: " + lastLTSFCV + ", targetVersion: " +
+              latestFCV + "} to {version: " + lastLTSFCV + ", targetVersion: " + lastLTSFCV + "}");
+
+    rollbackTest.transitionToRollbackOperations();
+    let setFCVInParallel = startParallelShell(funWithArgs(setFCV, latestFCV), rollbackNode.port);
+    // Wait for the FCV update to be reflected on the rollbackNode. This should eventually be rolled
+    // back.
+    assert.soon(
+        function() {
+            let featureCompatibilityVersion = getFCVFromDocument(rollbackNode);
+            jsTestLog(`rollbackNode's version in parallel shell (should eventually be upgrading): ${
+                tojson(featureCompatibilityVersion)}`);
+            return !featureCompatibilityVersion.hasOwnProperty('previousVersion') &&
+                featureCompatibilityVersion.hasOwnProperty('targetVersion') &&
+                featureCompatibilityVersion.targetVersion == latestFCV;
+        },
+        "Failed waiting for the server to unset the previous version and set the target version to " +
+            latestFCV);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, latestFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+
+    setFCVInParallel();
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`Rollback node's version after setFCVInParallel: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, latestFCV);
+    // Secondaries should never have received the FCV update.
+    fcvDoc = syncSourceAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`syncSource's version (should still be downgrading): ${tojson(fcvDoc)}`);
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // The rollbackNode should have rolled back their FCV to be consistent with the rest of the
+    // replica set.
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version after rollback: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, lastLTSFCV);
+    fcvDoc = syncSourceAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`SyncSource's version after rollback: ${tojson(fcvDoc)}`);
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    const newPrimary = rollbackTest.getPrimary();
+    const newPrimaryAdminDB = newPrimary.getDB('admin');
+    // We should now be able to set the FCV from downgrading to upgrading to upgraded.
+    assert.commandWorked(newPrimary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+    checkFCV(newPrimaryAdminDB, latestFCV);
+}
+
 const testName = jsTest.name();
 
 const rollbackTest = new RollbackTest(testName);
@@ -153,5 +253,8 @@ rollbackFCVFromDowngradedOrUpgraded(lastLTSFCV, latestFCV, "hangWhileDowngrading
 // Tests the case where we roll back the FCV state from fully upgraded to upgrading.
 rollbackFCVFromDowngradedOrUpgraded(latestFCV, lastLTSFCV, "hangWhileUpgrading");
 
+// Tests the case where we roll back the FCV state from upgrading to downgrading.
+rollbackFCVFromUpgradingToDowngrading();
+
 rollbackTest.stop();
 }());