diff options
author | Matthew Russotto <matthew.russotto@mongodb.com> | 2021-06-30 17:12:30 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-06-30 21:26:04 +0000 |
commit | e0987d2383a8146cb49ef9a01bbe9195a48af27b (patch) | |
tree | 588865b804f9fc5be694a551c2dad9064b768f09 | |
parent | 07726214c77fc4f1bafcb9ab68cb44cf488c235c (diff) | |
download | mongo-e0987d2383a8146cb49ef9a01bbe9195a48af27b.tar.gz |
SERVER-53643 Wait for FCV to be majority committed before reporting it.
(cherry picked from commit f34d72aed2861b91fdd2907058d1fbec7f66e328)
-rw-r--r-- | jstests/multiVersion/major_version_upgrade.js | 9 | ||||
-rw-r--r-- | jstests/replsets/rollback_set_fcv.js | 20 | ||||
-rw-r--r-- | src/mongo/db/commands/feature_compatibility_version.cpp | 56 | ||||
-rw-r--r-- | src/mongo/db/commands/feature_compatibility_version.h | 4 |
4 files changed, 75 insertions, 14 deletions
diff --git a/jstests/multiVersion/major_version_upgrade.js b/jstests/multiVersion/major_version_upgrade.js index dd534513420..3be916fcf5f 100644 --- a/jstests/multiVersion/major_version_upgrade.js +++ b/jstests/multiVersion/major_version_upgrade.js @@ -13,6 +13,7 @@ 'use strict'; load('jstests/libs/get_index_helpers.js'); + load('jstests/libs/feature_compatibility_version.js'); load('jstests/multiVersion/libs/multi_rs.js'); load('jstests/multiVersion/libs/verify_versions.js'); @@ -302,6 +303,14 @@ assert.commandWorked(primaryAdminDB.runCommand( {setFeatureCompatibilityVersion: version.featureCompatibilityVersion})); rst.awaitReplication(); + // Make sure we reach the new featureCompatibilityVersion in the committed snapshot on + // on all nodes before continuing to upgrade. + // checkFCV does not work for version 3.4 (and below) + if (version.featureCompatibilityVersion != '3.4') { + for (let n of rst.nodes) { + checkFCV(n.getDB("admin"), version.featureCompatibilityVersion); + } + } } } diff --git a/jstests/replsets/rollback_set_fcv.js b/jstests/replsets/rollback_set_fcv.js index 3ec0e8c2ebd..2e91bf987c9 100644 --- a/jstests/replsets/rollback_set_fcv.js +++ b/jstests/replsets/rollback_set_fcv.js @@ -21,6 +21,12 @@ assert.commandFailed(db.adminCommand({setFeatureCompatibilityVersion: fcv})); } + // Using getParameter results in waiting for the current FCV to be majority committed. In this + // test, it never will, so we need to get the FCV directly. + function getFCVFromDocument(conn) { + return conn.getDB("admin").system.version.find().readConcern("local").toArray()[0]; + } + // fromFCV refers to the FCV we will test rolling back from. // toFCV refers to the FCV we will test rolling back to. function rollbackFCVFromDowngradingOrUpgrading(fromFCV, toFCV) { @@ -40,9 +46,8 @@ // Wait for the FCV update to be reflected on the primary. This should eventually be rolled // back. assert.soon(function() { - let res = assert.commandWorked( - primary.adminCommand({getParameter: 1, featureCompatibilityVersion: 1})); - return res.featureCompatibilityVersion.hasOwnProperty('targetVersion'); + let featureCompatibilityVersion = getFCVFromDocument(primary); + return featureCompatibilityVersion.hasOwnProperty('targetVersion'); }, "Failed waiting for the server to set the targetVersion: " + fromFCV); rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); // Secondaries should never have received the FCV update. @@ -94,10 +99,9 @@ assert.commandWorked(primary.adminCommand({configureFailPoint: failPoint, mode: "off"})); assert.soon(function() { - let res = assert.commandWorked( - primary.adminCommand({getParameter: 1, featureCompatibilityVersion: 1})); - return !res.featureCompatibilityVersion.hasOwnProperty('targetVersion') && - res.featureCompatibilityVersion.version === fromFCV; + let featureCompatibilityVersion = getFCVFromDocument(primary); + return !featureCompatibilityVersion.hasOwnProperty('targetVersion') && + featureCompatibilityVersion.version === fromFCV; }, "Failed waiting for server to unset the targetVersion or to set the FCV to " + fromFCV); rollbackTest.transitionToSyncSourceOperationsBeforeRollback(); // The secondary should never have received the update to unset the targetVersion. @@ -135,4 +139,4 @@ rollbackFCVFromDowngradedOrUpgraded(latestFCV, lastStableFCV, "hangWhileUpgrading"); rollbackTest.stop(); -}());
\ No newline at end of file +}()); diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp index 6a4fc551788..44a76df34d3 100644 --- a/src/mongo/db/commands/feature_compatibility_version.cpp +++ b/src/mongo/db/commands/feature_compatibility_version.cpp @@ -43,6 +43,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/operation_context.h" #include "mongo/db/repl/optime.h" +#include "mongo/db/repl/read_concern_level.h" #include "mongo/db/repl/storage_interface.h" #include "mongo/db/s/collection_sharding_state.h" #include "mongo/db/s/sharding_state.h" @@ -63,6 +64,10 @@ namespace mongo { using repl::UnreplicatedWritesBlock; Lock::ResourceMutex FeatureCompatibilityVersion::fcvLock("featureCompatibilityVersionLock"); +// lastFCVUpdateTimestamp contains the latest oplog entry timestamp which updated the FCV. +// It is reset on rollback. +Timestamp lastFCVUpdateTimestamp; +SimpleMutex lastFCVUpdateTimestampMutex; void FeatureCompatibilityVersion::setTargetUpgrade(OperationContext* opCtx) { // Sets both 'version' and 'targetVersion' fields. @@ -162,7 +167,7 @@ void FeatureCompatibilityVersion::onInsertOrUpdate(OperationContext* opCtx, cons } opCtx->recoveryUnit()->onCommit( - [opCtx, newVersion](boost::optional<Timestamp>) { _setVersion(opCtx, newVersion); }); + [opCtx, newVersion](boost::optional<Timestamp> ts) { _setVersion(opCtx, newVersion, ts); }); } void FeatureCompatibilityVersion::updateMinWireVersion() { @@ -186,7 +191,17 @@ void FeatureCompatibilityVersion::updateMinWireVersion() { } void FeatureCompatibilityVersion::_setVersion( - OperationContext* opCtx, ServerGlobalParams::FeatureCompatibility::Version newVersion) { + OperationContext* opCtx, + ServerGlobalParams::FeatureCompatibility::Version newVersion, + boost::optional<Timestamp> commitTs) { + // We set the last FCV update timestamp before setting the new FCV, to make sure we never + // read an FCV that is not stable. We might still read a stale one. + { + stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex); + if (commitTs && *commitTs > lastFCVUpdateTimestamp) { + lastFCVUpdateTimestamp = *commitTs; + } + } serverGlobalParams.featureCompatibility.setVersion(newVersion); updateMinWireVersion(); @@ -230,7 +245,10 @@ void FeatureCompatibilityVersion::onReplicationRollback(OperationContext* opCtx) << FeatureCompatibilityVersionParser::toString(memoryFcv) << "' to '" << FeatureCompatibilityVersionParser::toString(diskFcv) << "' as part of rollback."; - _setVersion(opCtx, diskFcv); + _setVersion(opCtx, diskFcv, boost::none); + // The rollback FCV is already in the stable snapshot. + stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex); + lastFCVUpdateTimestamp = Timestamp(); } } } @@ -302,7 +320,7 @@ public: featureCompatibilityVersionBuilder.append( FeatureCompatibilityVersionParser::kVersionField, FeatureCompatibilityVersionParser::kVersion40); - return; + break; case ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo40: featureCompatibilityVersionBuilder.append( FeatureCompatibilityVersionParser::kVersionField, @@ -323,11 +341,39 @@ public: featureCompatibilityVersionBuilder.append( FeatureCompatibilityVersionParser::kVersionField, FeatureCompatibilityVersionParser::kVersion36); - return; + break; case ServerGlobalParams::FeatureCompatibility::Version::kUnsetDefault36Behavior: // getVersion() does not return this value. MONGO_UNREACHABLE; } + // If the FCV has been recently set to the fully upgraded FCV but is not part of the + // majority snapshot, then if we do a binary upgrade, we may see the old FCV at startup. + // It is not safe to do oplog application on the new binary at that point. So we make sure + // that when we report the FCV, it is in the majority snapshot. + // (The same consideration applies at downgrade, where if a recently-set fully downgraded + // FCV is not part of the majority snapshot, the downgraded binary will see the upgrade FCV + // and fail.) + const auto replCoordinator = repl::ReplicationCoordinator::get(opCtx); + const bool isReplSet = replCoordinator && + replCoordinator->getReplicationMode() == repl::ReplicationCoordinator::modeReplSet; + auto neededMajorityTimestamp = [] { + stdx::lock_guard<SimpleMutex> lk(lastFCVUpdateTimestampMutex); + return lastFCVUpdateTimestamp; + }(); + if (isReplSet && !neededMajorityTimestamp.isNull()) { + auto status = replCoordinator->waitUntilOpTimeForRead( + opCtx, + repl::ReadConcernArgs( + repl::OpTime(neededMajorityTimestamp, repl::OpTime::kUninitializedTerm), + repl::ReadConcernLevel::kMajorityReadConcern)); + // If majority reads are not supported, we will take a full snapshot on clean shutdown + // and the new FCV will be included, so upgrade is possible. + if (status.code() != ErrorCodes::CommandNotSupported) + uassertStatusOK( + status.withContext("Most recent 'featureCompatibilityVersion' was not in the " + "majority snapshot on this node")); + } + return; } virtual Status set(const BSONElement& newValueElement) { diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h index 2015be977a7..63bc490e1d6 100644 --- a/src/mongo/db/commands/feature_compatibility_version.h +++ b/src/mongo/db/commands/feature_compatibility_version.h @@ -127,9 +127,11 @@ private: /** * Set the FCV to newVersion, making sure to close any outgoing connections with incompatible * servers and closing open transactions if necessary. + * If the commitTimestamp is set, advances the lastFCVUpdateTimestamp to it. */ static void _setVersion(OperationContext* opCtx, - ServerGlobalParams::FeatureCompatibility::Version newVersion); + ServerGlobalParams::FeatureCompatibility::Version newVersion, + boost::optional<Timestamp> commitTimestamp); }; |