SERVER-46758 In-memory FCV value should properly reflect the on-disk FCV after a rollback

(cherry picked from commit aa527109a28bec0b6fe2763fce8a447ead0c02dd)

5 files changed, 208 insertions, 29 deletions

diff --git a/jstests/replsets/rollback_set_fcv.js b/jstests/replsets/rollback_set_fcv.js
new file mode 100644
index 00000000000..7a787532e7b
--- /dev/null
+++ b/jstests/replsets/rollback_set_fcv.js
@@ -0,0 +1,128 @@
+/*
+ * Tests the following scenarios where the featureCompatibilityVersion document is rolled back and
+ * verify that the in-memory and on-disk FCV stay consistent.
+ * - the FCV document is rolled back from fully upgraded to upgrading
+ * - the FCV document is rolled back from upgrading to fully downgraded
+ * - the FCV document is rolled back from fully downgraded to downgrading
+ * - the FCV document is rolled back from downgrading to fully upgraded
+ *
+ *  @tags: [multiversion_incompatible]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/replsets/libs/rollback_test.js");
+load('jstests/libs/parallel_shell_helpers.js');
+load("jstests/libs/fail_point_util.js");
+
+function setFCV(fcv) {
+    assert.commandFailedWithCode(db.adminCommand({setFeatureCompatibilityVersion: fcv}),
+                                 ErrorCodes.InterruptedDueToReplStateChange);
+}
+
+// fromFCV refers to the FCV we will test rolling back from.
+// toFCV refers to the FCV we will test rolling back to.
+function rollbackFCVFromDowngradingOrUpgrading(fromFCV, toFCV) {
+    let primary = rollbackTest.getPrimary();
+    let secondary = rollbackTest.getSecondary();
+    let primaryAdminDB = primary.getDB('admin');
+    let secondaryAdminDB = secondary.getDB('admin');
+
+    // Ensure the cluster starts at the correct FCV.
+    assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: toFCV}));
+
+    jsTestLog("Testing rolling back FCV from {version: " + lastStableFCV +
+              ", targetVersion: " + fromFCV + "} to {version: " + toFCV + "}");
+
+    rollbackTest.transitionToRollbackOperations();
+    let setFCVInParallel = startParallelShell(funWithArgs(setFCV, fromFCV), primary.port);
+    // Wait for the FCV update to be reflected on the primary. This should eventually be rolled
+    // back.
+    assert.soon(function() {
+        let res = assert.commandWorked(
+            primary.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}));
+        return res.featureCompatibilityVersion.hasOwnProperty('targetVersion');
+    }, "Failed waiting for the server to set the targetVersion: " + fromFCV);
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    // Secondaries should never have received the FCV update.
+    checkFCV(secondaryAdminDB, toFCV);
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+    setFCVInParallel();
+    rollbackTest.transitionToSteadyStateOperations();
+    // The primary should have rolled back their FCV to be consistent with the rest of the replica
+    // set.
+    checkFCV(primaryAdminDB, toFCV);
+    checkFCV(secondaryAdminDB, toFCV);
+
+    let newPrimary = rollbackTest.getPrimary();
+    // As a rule, we forbid downgrading a node while a node is still in the upgrading state and
+    // vice versa. Ensure that the in-memory and on-disk FCV are consistent by checking that we are
+    // able to set the FCV back to the original version.
+    assert.commandWorked(newPrimary.adminCommand({setFeatureCompatibilityVersion: toFCV}));
+}
+
+// fromFCV refers to the FCV we will test rolling back from.
+// toFCV refers to the FCV we will test rolling back to.
+function rollbackFCVFromDowngradedOrUpgraded(fromFCV, toFCV, failPoint) {
+    let primary = rollbackTest.getPrimary();
+    let secondary = rollbackTest.getSecondary();
+    let primaryAdminDB = primary.getDB('admin');
+    let secondaryAdminDB = secondary.getDB('admin');
+
+    // Complete the upgrade/downgrade to ensure we are not in the upgrading/downgrading state.
+    assert.commandWorked(primary.adminCommand({setFeatureCompatibilityVersion: toFCV}));
+
+    jsTestLog("Testing rolling back FCV from {version: " + fromFCV +
+              "} to {version: " + lastStableFCV + ", targetVersion: " + fromFCV + "}");
+
+    // A failpoint to hang right before unsetting the targetVersion.
+    const hangBeforeUnsettingTargetVersion = configureFailPoint(primary, failPoint);
+    let setFCVInParallel = startParallelShell(funWithArgs(setFCV, fromFCV), primary.port);
+    hangBeforeUnsettingTargetVersion.wait();
+    rollbackTest.transitionToRollbackOperations();
+    // Turn off the failpoint so the primary will proceed to unset the targetVersion. This update
+    // should never make it to the secondary.
+    hangBeforeUnsettingTargetVersion.off();
+    assert.soon(function() {
+        let res = assert.commandWorked(
+            primary.adminCommand({getParameter: 1, featureCompatibilityVersion: 1}));
+        return !res.featureCompatibilityVersion.hasOwnProperty('targetVersion') &&
+            res.featureCompatibilityVersion.version === fromFCV;
+    }, "Failed waiting for server to unset the targetVersion or to set the FCV to " + fromFCV);
+    rollbackTest.transitionToSyncSourceOperationsBeforeRollback();
+    // The secondary should never have received the update to unset the targetVersion.
+    checkFCV(secondaryAdminDB, lastStableFCV, fromFCV);
+    rollbackTest.transitionToSyncSourceOperationsDuringRollback();
+    setFCVInParallel();
+    rollbackTest.transitionToSteadyStateOperations();
+    // The primary should have rolled back their FCV to contain the targetVersion.
+    checkFCV(primaryAdminDB, lastStableFCV, fromFCV);
+    checkFCV(secondaryAdminDB, lastStableFCV, fromFCV);
+
+    let newPrimary = rollbackTest.getPrimary();
+    // As a rule, we forbid downgrading a node while a node is still in the upgrading state and
+    // vice versa. Ensure that the in-memory and on-disk FCV are consistent by checking that this
+    // rule is upheld after rollback.
+    assert.commandFailedWithCode(newPrimary.adminCommand({setFeatureCompatibilityVersion: toFCV}),
+                                 ErrorCodes.IllegalOperation);
+}
+
+const testName = jsTest.name();
+
+const rollbackTest = new RollbackTest(testName);
+
+// Tests the case where we roll back the FCV state from downgrading to fully upgraded.
+rollbackFCVFromDowngradingOrUpgrading(lastStableFCV, latestFCV);
+
+// Tests the case where we roll back the FCV state from upgrading to fully downgraded.
+rollbackFCVFromDowngradingOrUpgrading(latestFCV, lastStableFCV);
+
+// Tests the case where we roll back the FCV state from fully downgraded to downgrading.
+rollbackFCVFromDowngradedOrUpgraded(lastStableFCV, latestFCV, "hangWhileDowngrading");
+
+// Tests the case where we roll back the FCV state from fully upgraded to upgrading.
+rollbackFCVFromDowngradedOrUpgraded(latestFCV, lastStableFCV, "hangWhileUpgrading");
+
+rollbackTest.stop();
+}());
\ No newline at end of file
diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp
index ec1d65deb3b..f0b4e99409c 100644
--- a/src/mongo/db/commands/feature_compatibility_version.cpp
+++ b/src/mongo/db/commands/feature_compatibility_version.cpp
@@ -161,35 +161,8 @@ void FeatureCompatibilityVersion::onInsertOrUpdate(OperationContext* opCtx, cons
               << FeatureCompatibilityVersionParser::toString(newVersion);
     }
 
-    opCtx->recoveryUnit()->onCommit([opCtx, newVersion](boost::optional<Timestamp>) {
-        serverGlobalParams.featureCompatibility.setVersion(newVersion);
-        updateMinWireVersion();
-
-        if (newVersion != ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40) {
-            // Close all incoming connections from internal clients with binary versions lower than
-            // ours.
-            opCtx->getServiceContext()->getServiceEntryPoint()->endAllSessions(
-                transport::Session::kLatestVersionInternalClientKeepOpen |
-                transport::Session::kExternalClientKeepOpen);
-            // Close all outgoing connections to servers with binary versions lower than ours.
-            executor::EgressTagCloserManager::get(opCtx->getServiceContext())
-                .dropConnections(transport::Session::kKeepOpen);
-        }
-
-        if (newVersion != ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42) {
-            if (MONGO_FAIL_POINT(hangBeforeAbortingRunningTransactionsOnFCVDowngrade)) {
-                log() << "featureCompatibilityVersion - "
-                         "hangBeforeAbortingRunningTransactionsOnFCVDowngrade fail point enabled. "
-                         "Blocking until fail point is disabled.";
-                MONGO_FAIL_POINT_PAUSE_WHILE_SET(
-                    hangBeforeAbortingRunningTransactionsOnFCVDowngrade);
-            }
-            // Abort all open transactions when downgrading the featureCompatibilityVersion.
-            SessionKiller::Matcher matcherAllSessions(
-                KillAllSessionsByPatternSet{makeKillAllSessionsByPattern(opCtx)});
-            killSessionsAbortUnpreparedTransactions(opCtx, matcherAllSessions);
-        }
-    });
+    opCtx->recoveryUnit()->onCommit(
+        [opCtx, newVersion](boost::optional<Timestamp>) { _setVersion(opCtx, newVersion); });
 }
 
 void FeatureCompatibilityVersion::updateMinWireVersion() {
@@ -212,6 +185,55 @@ void FeatureCompatibilityVersion::updateMinWireVersion() {
     }
 }
 
+void FeatureCompatibilityVersion::_setVersion(
+    OperationContext* opCtx, ServerGlobalParams::FeatureCompatibility::Version newVersion) {
+    serverGlobalParams.featureCompatibility.setVersion(newVersion);
+    updateMinWireVersion();
+
+    if (newVersion != ServerGlobalParams::FeatureCompatibility::Version::kFullyDowngradedTo40) {
+        // Close all incoming connections from internal clients with binary versions lower than
+        // ours.
+        opCtx->getServiceContext()->getServiceEntryPoint()->endAllSessions(
+            transport::Session::kLatestVersionInternalClientKeepOpen |
+            transport::Session::kExternalClientKeepOpen);
+        // Close all outgoing connections to servers with binary versions lower than ours.
+        executor::EgressTagCloserManager::get(opCtx->getServiceContext())
+            .dropConnections(transport::Session::kKeepOpen);
+    }
+
+    if (newVersion != ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42) {
+        if (MONGO_FAIL_POINT(hangBeforeAbortingRunningTransactionsOnFCVDowngrade)) {
+            log() << "featureCompatibilityVersion - "
+                     "hangBeforeAbortingRunningTransactionsOnFCVDowngrade fail point enabled. "
+                     "Blocking until fail point is disabled.";
+            MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangBeforeAbortingRunningTransactionsOnFCVDowngrade);
+        }
+        // Abort all open transactions when downgrading the featureCompatibilityVersion.
+        SessionKiller::Matcher matcherAllSessions(
+            KillAllSessionsByPatternSet{makeKillAllSessionsByPattern(opCtx)});
+        killSessionsAbortUnpreparedTransactions(opCtx, matcherAllSessions);
+    }
+}
+
+void FeatureCompatibilityVersion::onReplicationRollback(OperationContext* opCtx) {
+    const auto query = BSON("_id" << FeatureCompatibilityVersionParser::kParameterName);
+    const auto swFcv = repl::StorageInterface::get(opCtx)->findById(
+        opCtx, NamespaceString::kServerConfigurationNamespace, query["_id"]);
+    if (swFcv.isOK()) {
+        const auto featureCompatibilityVersion = swFcv.getValue();
+        auto swVersion = FeatureCompatibilityVersionParser::parse(featureCompatibilityVersion);
+        const auto memoryFcv = serverGlobalParams.featureCompatibility.getVersion();
+        if (swVersion.isOK() && (swVersion.getValue() != memoryFcv)) {
+            auto diskFcv = swVersion.getValue();
+            log() << "Setting featureCompatibilityVersion from '"
+                  << FeatureCompatibilityVersionParser::toString(diskFcv) << "' to '"
+                  << FeatureCompatibilityVersionParser::toString(memoryFcv)
+                  << "' as part of rollback.";
+            _setVersion(opCtx, diskFcv);
+        }
+    }
+}
+
 void FeatureCompatibilityVersion::_validateVersion(StringData version) {
     uassert(40284,
             str::stream() << "featureCompatibilityVersion must be '"
diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h
index 475243d6233..0e06d36488f 100644
--- a/src/mongo/db/commands/feature_compatibility_version.h
+++ b/src/mongo/db/commands/feature_compatibility_version.h
@@ -100,6 +100,11 @@ public:
      */
     static void updateMinWireVersion();
 
+    /**
+     * Ensures the in-memory and on-disk FCV states are consistent after a rollback.
+     */
+    static void onReplicationRollback(OperationContext* opCtx);
+
 private:
     /**
      * Validate version. Uasserts if invalid.
@@ -111,6 +116,13 @@ private:
      */
     typedef stdx::function<void(BSONObjBuilder)> UpdateBuilder;
     static void _runUpdateCommand(OperationContext* opCtx, UpdateBuilder callback);
+
+    /**
+     * Set the FCV to newVersion, making sure to close any outgoing connections with incompatible
+     * servers and closing open transactions if necessary. Increments the server TopologyVersion.
+     */
+    static void _setVersion(OperationContext* opCtx,
+                            ServerGlobalParams::FeatureCompatibility::Version newVersion);
 };
 
 
diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index 0127b1f401a..28b771984e9 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -70,6 +70,8 @@ MONGO_FAIL_POINT_DEFINE(featureCompatibilityDowngrade);
 MONGO_FAIL_POINT_DEFINE(featureCompatibilityUpgrade);
 MONGO_FAIL_POINT_DEFINE(pauseBeforeUpgradingSessions);
 MONGO_FAIL_POINT_DEFINE(pauseBeforeDowngradingSessions);
+MONGO_FAIL_POINT_DEFINE(hangWhileUpgrading);
+MONGO_FAIL_POINT_DEFINE(hangWhileDowngrading);
 
 /**
  * Returns a set of the logical session ids of each entry in config.transactions that matches the
@@ -366,6 +368,12 @@ public:
                                      << requestedVersion)))));
             }
 
+            if (MONGO_FAIL_POINT(hangWhileUpgrading)) {
+                log() << "featureCompatibilityVersion - "
+                         "hangWhileUpgrading fail point enabled. "
+                         "Blocking until fail point is disabled.";
+                MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangWhileUpgrading);
+            }
             FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion);
         } else if (requestedVersion == FeatureCompatibilityVersionParser::kVersion40) {
             uassert(ErrorCodes::IllegalOperation,
@@ -410,6 +418,12 @@ public:
                                      << requestedVersion)))));
             }
 
+            if (MONGO_FAIL_POINT(hangWhileDowngrading)) {
+                log() << "featureCompatibilityVersion - "
+                         "hangWhileDowngrading fail point enabled. "
+                         "Blocking until fail point is disabled.";
+                MONGO_FAIL_POINT_PAUSE_WHILE_SET(hangWhileDowngrading);
+            }
             FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, requestedVersion);
         }
 
diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp
index 84c627f69c1..d4715bb3b96 100644
--- a/src/mongo/db/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer_impl.cpp
@@ -1469,6 +1469,9 @@ void OpObserverImpl::onReplicationRollback(OperationContext* opCtx,
             shardRegistry->clearEntries();
         }
     }
+
+    // Make sure the in-memory FCV matches the on-disk FCV.
+    FeatureCompatibilityVersion::onReplicationRollback(opCtx);
 }
 
 }  // namespace mongo