SERVER-53643 Wait for FCV to be majority committed before reporting it.

(cherry picked from commit f34d72aed2861b91fdd2907058d1fbec7f66e328)

2 files changed, 54 insertions, 6 deletions

diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp
index 6a4fc551788..44a76df34d3 100644
--- a/src/mongo/db/commands/feature_compatibility_version.cpp
+++ b/src/mongo/db/commands/feature_compatibility_version.cpp
@@ -43,6 +43,7 @@
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/operation_context.h"
 #include "mongo/db/repl/optime.h"
+#include "mongo/db/repl/read_concern_level.h"
 #include "mongo/db/repl/storage_interface.h"
 #include "mongo/db/s/collection_sharding_state.h"
 #include "mongo/db/s/sharding_state.h"
@@ -63,6 +64,10 @@ namespace mongo {
 using repl::UnreplicatedWritesBlock;
 
 Lock::ResourceMutex FeatureCompatibilityVersion::fcvLock("featureCompatibilityVersionLock");
+// lastFCVUpdateTimestamp contains the latest oplog entry timestamp which updated the FCV.
+// It is reset on rollback.
+Timestamp lastFCVUpdateTimestamp;
+SimpleMutex lastFCVUpdateTimestampMutex;
 
 void FeatureCompatibilityVersion::setTargetUpgrade(OperationContext* opCtx) {
     // Sets both 'version' and 'targetVersion' fields.
@@ -162,7 +167,7 @@ void FeatureCompatibilityVersion::onInsertOrUpdate(OperationContext* opCtx, cons
     }
 
     opCtx->recoveryUnit()->onCommit(
-        [opCtx, newVersion](boost::optional<Timestamp>) { _setVersion(opCtx, newVersion); });
+        [opCtx, newVersion](boost::optional<Timestamp> ts) { _setVersion(opCtx, newVersion, ts); });
 }
 
 void FeatureCompatibilityVersion::updateMinWireVersion() {
@@ -186,7 +191,17 @@ void FeatureCompatibilityVersion::updateMinWireVersion() {
 }
 
 void FeatureCompatibilityVersion::_setVersion(
-    OperationContext* opCtx, ServerGlobalParams::FeatureCompatibility::Version newVersion) {
+    OperationContext* opCtx,
+    ServerGlobalParams::FeatureCompatibility::Version newVersion,
+    boost::optional<Timestamp> commitTs) {
+    // We set the last FCV update timestamp before setting the new FCV, to make sure we never
+    // read an FCV that is not stable.  We might still read a stale one.
+    {
+        stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex);
+        if (commitTs && *commitTs > lastFCVUpdateTimestamp) {
+            lastFCVUpdateTimestamp = *commitTs;
+        }
+    }
     serverGlobalParams.featureCompatibility.setVersion(newVersion);
     updateMinWireVersion();
 
@@ -230,7 +245,10 @@ void FeatureCompatibilityVersion::onReplicationRollback(OperationContext* opCtx)
                   << FeatureCompatibilityVersionParser::toString(memoryFcv) << "' to '"
                   << FeatureCompatibilityVersionParser::toString(diskFcv)
                   << "' as part of rollback.";
-            _setVersion(opCtx, diskFcv);
+            _setVersion(opCtx, diskFcv, boost::none);
+            // The rollback FCV is already in the stable snapshot.
+            stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex);
+            lastFCVUpdateTimestamp = Timestamp();
         }
     }
 }
@@ -302,7 +320,7 @@ public:
                 featureCompatibilityVersionBuilder.append(
                     FeatureCompatibilityVersionParser::kVersionField,
                     FeatureCompatibilityVersionParser::kVersion40);
-                return;
+                break;
             case ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo40:
                 featureCompatibilityVersionBuilder.append(
                     FeatureCompatibilityVersionParser::kVersionField,
@@ -323,11 +341,39 @@ public:
                 featureCompatibilityVersionBuilder.append(
                     FeatureCompatibilityVersionParser::kVersionField,
                     FeatureCompatibilityVersionParser::kVersion36);
-                return;
+                break;
             case ServerGlobalParams::FeatureCompatibility::Version::kUnsetDefault36Behavior:
                 // getVersion() does not return this value.
                 MONGO_UNREACHABLE;
         }
+        // If the FCV has been recently set to the fully upgraded FCV but is not part of the
+        // majority snapshot, then if we do a binary upgrade, we may see the old FCV at startup.
+        // It is not safe to do oplog application on the new binary at that point.  So we make sure
+        // that when we report the FCV, it is in the majority snapshot.
+        // (The same consideration applies at downgrade, where if a recently-set fully downgraded
+        // FCV is not part of the majority snapshot, the downgraded binary will see the upgrade FCV
+        // and fail.)
+        const auto replCoordinator = repl::ReplicationCoordinator::get(opCtx);
+        const bool isReplSet = replCoordinator &&
+            replCoordinator->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet;
+        auto neededMajorityTimestamp = [] {
+            stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex);
+            return lastFCVUpdateTimestamp;
+        }();
+        if (isReplSet && !neededMajorityTimestamp.isNull()) {
+            auto status = replCoordinator->waitUntilOpTimeForRead(
+                opCtx,
+                repl::ReadConcernArgs(
+                    repl::OpTime(neededMajorityTimestamp, repl::OpTime::kUninitializedTerm),
+                    repl::ReadConcernLevel::kMajorityReadConcern));
+            // If majority reads are not supported, we will take a full snapshot on clean shutdown
+            // and the new FCV will be included, so upgrade is possible.
+            if (status.code() != ErrorCodes::CommandNotSupported)
+                uassertStatusOK(
+                    status.withContext("Most recent 'featureCompatibilityVersion' was not in the "
+                                       "majority snapshot on this node"));
+        }
+        return;
     }
 
     virtual Status set(const BSONElement& newValueElement) {
diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h
index 2015be977a7..63bc490e1d6 100644
--- a/src/mongo/db/commands/feature_compatibility_version.h
+++ b/src/mongo/db/commands/feature_compatibility_version.h
@@ -127,9 +127,11 @@ private:
     /**
      * Set the FCV to newVersion, making sure to close any outgoing connections with incompatible
      * servers and closing open transactions if necessary.
+     * If the commitTimestamp is set, advances the lastFCVUpdateTimestamp to it.
      */
     static void _setVersion(OperationContext* opCtx,
-                            ServerGlobalParams::FeatureCompatibility::Version newVersion);
+                            ServerGlobalParams::FeatureCompatibility::Version newVersion,
+                            boost::optional<Timestamp> commitTimestamp);
 };