SERVER-68143 Add rollback tests for downgrading to upgraded path

1 files changed, 103 insertions, 0 deletions

diff --git a/jstests/replsets/rollback_set_fcv.js b/jstests/replsets/rollback_set_fcv.js
index 4a82ed33857..388f603dfb6 100644
--- a/jstests/replsets/rollback_set_fcv.js
+++ b/jstests/replsets/rollback_set_fcv.js
@@ -137,6 +137,106 @@ function rollbackFCVFromDowngradedOrUpgraded(fromFCV, toFCV, failPoint) {
     }
 }
 
+// Test rolling back from upgrading to downgrading.
+// Start off with downgrading from latest to lastLTS.
+// Go to upgrading from lastLTS to latest state.
+// Rollback and make sure the FCV doc is back in the downgrading from latest to lastLTS state.
+function rollbackFCVFromUpgradingToDowngrading() {
+    let fcvDoc;
+    const rollbackNode = rollbackTest.getPrimary();
+    const syncSource = rollbackTest.getSecondary();
+    const rollbackNodeAdminDB = rollbackNode.getDB('admin');
+    const syncSourceAdminDB = syncSource.getDB('admin');
+
+    // Ensure the cluster starts at the correct FCV.
+    assert.commandWorked(rollbackNode.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+
+    if (!FeatureFlagUtil.isEnabled(rollbackNodeAdminDB, "DowngradingToUpgrading")) {
+        jsTestLog(
+            "Skipping rollbackFCVFromUpgradingToDowngrading as featureFlagDowngradingToUpgrading is not enabled");
+        return;
+    }
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version at start: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, latestFCV);
+
+    // Set the failpoints so that both upgrading and downgrading would fail.
+    assert.commandWorked(
+        rollbackNode.adminCommand({configureFailPoint: "failDowngrading", mode: "alwaysOn"}));
+    assert.commandWorked(
+        rollbackNode.adminCommand({configureFailPoint: "failUpgrading", mode: "alwaysOn"}));
+
+    // Go to downgrading state (downgrading from latest to lastLTS).
+    assert.commandFailed(
+        rollbackNodeAdminDB.runCommand({setFeatureCompatibilityVersion: lastLTSFCV}));
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version after downgrading: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, lastLTSFCV);
+
+    // Wait until the config has propagated to the other nodes and the rollbackNode has learned of
+    // it, so that the config replication check in 'setFeatureCompatibilityVersion' is satisfied.
+    // This is only important since 'setFeatureCompatibilityVersion' is known to implicitly call
+    // internal reconfigs as part of upgrade/downgrade behavior.
+    rollbackTest.getTestFixture().waitForConfigReplication(rollbackNode);
+    // Wait for the majority commit point to be updated on the sync source, because checkFCV calls
+    // getParameter for the featureCompatibilityVersion, which will wait until the FCV change makes
+    // it into the node's majority committed snapshot.
+    rollbackTest.getTestFixture().awaitLastOpCommitted(undefined /* timeout */, [syncSource]);
+
+    // test rolling back from upgrading to downgrading
+    jsTestLog("Testing rolling back FCV from {version: " + lastLTSFCV + ", targetVersion: " +
+              latestFCV + "} to {version: " + lastLTSFCV + ", targetVersion: " + lastLTSFCV + "}");
+
+    rollbackTest.transitionToRollbackOperations();
+    let setFCVInParallel = startParallelShell(funWithArgs(setFCV, latestFCV), rollbackNode.port);
+    // Wait for the FCV update to be reflected on the rollbackNode. This should eventually be rolled
+    // back.
+    assert.soon(
+        function() {
+            let featureCompatibilityVersion = getFCVFromDocument(rollbackNode);
+            jsTestLog(`rollbackNode's version in parallel shell (should eventually be upgrading): ${
+                tojson(featureCompatibilityVersion)}`);
+            return !featureCompatibilityVersion.hasOwnProperty('previousVersion') &&
+                featureCompatibilityVersion.hasOwnProperty('targetVersion') &&
+                featureCompatibilityVersion.targetVersion == latestFCV;
+        },
+        "Failed waiting for the server to unset the previous version and set the target version to " +
+            latestFCV);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, latestFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+
+    setFCVInParallel();
+
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`Rollback node's version after setFCVInParallel: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, latestFCV);
+    // Secondaries should never have received the FCV update.
+    fcvDoc = syncSourceAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`syncSource's version (should still be downgrading): ${tojson(fcvDoc)}`);
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+    rollbackTest.transitionToSteadyStateOperations();
+
+    // The rollbackNode should have rolled back their FCV to be consistent with the rest of the
+    // replica set.
+    fcvDoc = rollbackNodeAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`rollbackNode's version after rollback: ${tojson(fcvDoc)}`);
+    checkFCV(rollbackNodeAdminDB, lastLTSFCV, lastLTSFCV);
+    fcvDoc = syncSourceAdminDB.system.version.findOne({_id: 'featureCompatibilityVersion'});
+    jsTestLog(`SyncSource's version after rollback: ${tojson(fcvDoc)}`);
+    checkFCV(syncSourceAdminDB, lastLTSFCV, lastLTSFCV);
+
+    const newPrimary = rollbackTest.getPrimary();
+    const newPrimaryAdminDB = newPrimary.getDB('admin');
+    // We should now be able to set the FCV from downgrading to upgrading to upgraded.
+    assert.commandWorked(newPrimary.adminCommand({setFeatureCompatibilityVersion: latestFCV}));
+    checkFCV(newPrimaryAdminDB, latestFCV);
+}
+
 const testName = jsTest.name();
 
 const rollbackTest = new RollbackTest(testName);
@@ -153,5 +253,8 @@ rollbackFCVFromDowngradedOrUpgraded(lastLTSFCV, latestFCV, "hangWhileDowngrading
 // Tests the case where we roll back the FCV state from fully upgraded to upgrading.
 rollbackFCVFromDowngradedOrUpgraded(latestFCV, lastLTSFCV, "hangWhileUpgrading");
 
+// Tests the case where we roll back the FCV state from upgrading to downgrading.
+rollbackFCVFromUpgradingToDowngrading();
+
 rollbackTest.stop();
 }());