SERVER-31209 Persist targetVersion in FCV document to indicate an upgrade/downgrade in progress.

10 files changed, 260 insertions, 124 deletions

diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp
index 5f6f7d61748..2891b7977bc 100644
--- a/src/mongo/db/catalog/database_impl.cpp
+++ b/src/mongo/db/catalog/database_impl.cpp
@@ -763,7 +763,7 @@ Collection* DatabaseImpl::createCollection(OperationContext* opCtx,
 
     CollectionOptions optionsWithUUID = options;
     if (enableCollectionUUIDs && !optionsWithUUID.uuid &&
-        serverGlobalParams.featureCompatibility.isSchemaVersion36.load() == true) {
+        serverGlobalParams.featureCompatibility.isSchemaVersion36()) {
         optionsWithUUID.uuid.emplace(CollectionUUID::gen());
     }
 
diff --git a/src/mongo/db/commands/feature_compatibility_version.cpp b/src/mongo/db/commands/feature_compatibility_version.cpp
index f6515bc0cd1..c35d1eb993f 100644
--- a/src/mongo/db/commands/feature_compatibility_version.cpp
+++ b/src/mongo/db/commands/feature_compatibility_version.cpp
@@ -55,51 +55,21 @@ constexpr StringData FeatureCompatibilityVersion::kCommandName;
 constexpr StringData FeatureCompatibilityVersion::kDatabase;
 constexpr StringData FeatureCompatibilityVersion::kParameterName;
 constexpr StringData FeatureCompatibilityVersion::kVersionField;
+constexpr StringData FeatureCompatibilityVersion::kTargetVersionField;
 
-namespace {
-BSONObj makeUpdateCommand(StringData newVersion, BSONObj writeConcern) {
-    BSONObjBuilder updateCmd;
+StatusWith<FeatureCompatibilityVersionInfo> FeatureCompatibilityVersion::parse(
+    const BSONObj& featureCompatibilityVersionInfo) {
+    FeatureCompatibilityVersionInfo versionInfo;
 
-    NamespaceString nss(FeatureCompatibilityVersion::kCollection);
-    updateCmd.append("update", nss.coll());
-    {
-        BSONArrayBuilder updates(updateCmd.subarrayStart("updates"));
-        {
-            BSONObjBuilder updateSpec(updates.subobjStart());
-            {
-                BSONObjBuilder queryFilter(updateSpec.subobjStart("q"));
-                queryFilter.append("_id", FeatureCompatibilityVersion::kParameterName);
-            }
-            {
-                BSONObjBuilder updateMods(updateSpec.subobjStart("u"));
-                updateMods.append(FeatureCompatibilityVersion::kVersionField, newVersion);
-            }
-            updateSpec.appendBool("upsert", true);
-        }
-    }
-
-    if (!writeConcern.isEmpty()) {
-        updateCmd.append(WriteConcernOptions::kWriteConcernField, writeConcern);
-    }
-
-    return updateCmd.obj();
-}
-}  // namespace
-
-StatusWith<ServerGlobalParams::FeatureCompatibility::Version> FeatureCompatibilityVersion::parse(
-    const BSONObj& featureCompatibilityVersionDoc) {
-    bool foundVersionField = false;
-    ServerGlobalParams::FeatureCompatibility::Version version;
-
-    for (auto&& elem : featureCompatibilityVersionDoc) {
+    for (auto&& elem : featureCompatibilityVersionInfo) {
         auto fieldName = elem.fieldNameStringData();
         if (fieldName == "_id") {
             continue;
-        } else if (fieldName == FeatureCompatibilityVersion::kVersionField) {
-            foundVersionField = true;
+        } else if (fieldName == FeatureCompatibilityVersion::kVersionField ||
+                   fieldName == FeatureCompatibilityVersion::kTargetVersionField) {
             if (elem.type() != BSONType::String) {
                 return Status(ErrorCodes::TypeMismatch,
-                              str::stream() << FeatureCompatibilityVersion::kVersionField
+                              str::stream() << fieldName
                                             << " must be of type String, but was of type "
                                             << typeName(elem.type())
                                             << ". Contents of "
@@ -107,20 +77,20 @@ StatusWith<ServerGlobalParams::FeatureCompatibility::Version> FeatureCompatibili
                                             << " document in "
                                             << FeatureCompatibilityVersion::kCollection
                                             << ": "
-                                            << featureCompatibilityVersionDoc
+                                            << featureCompatibilityVersionInfo
                                             << ". See "
                                             << feature_compatibility_version::kDochubLink
                                             << ".");
             }
+
+            ServerGlobalParams::FeatureCompatibility::Version version;
             if (elem.String() == FeatureCompatibilityVersionCommandParser::kVersion36) {
                 version = ServerGlobalParams::FeatureCompatibility::Version::k36;
             } else if (elem.String() == FeatureCompatibilityVersionCommandParser::kVersion34) {
                 version = ServerGlobalParams::FeatureCompatibility::Version::k34;
             } else {
                 return Status(ErrorCodes::BadValue,
-                              str::stream() << "Invalid value for "
-                                            << FeatureCompatibilityVersion::kVersionField
-                                            << ", found "
+                              str::stream() << "Invalid value for " << fieldName << ", found "
                                             << elem.String()
                                             << ", expected '"
                                             << FeatureCompatibilityVersionCommandParser::kVersion36
@@ -131,27 +101,32 @@ StatusWith<ServerGlobalParams::FeatureCompatibility::Version> FeatureCompatibili
                                             << " document in "
                                             << FeatureCompatibilityVersion::kCollection
                                             << ": "
-                                            << featureCompatibilityVersionDoc
+                                            << featureCompatibilityVersionInfo
                                             << ". See "
                                             << feature_compatibility_version::kDochubLink
                                             << ".");
             }
+
+            if (fieldName == FeatureCompatibilityVersion::kVersionField) {
+                versionInfo.version = version;
+            } else if (fieldName == FeatureCompatibilityVersion::kTargetVersionField) {
+                versionInfo.targetVersion = version;
+            }
         } else {
             return Status(ErrorCodes::BadValue,
-                          str::stream() << "Unrecognized field '" << elem.fieldName()
-                                        << "''. Contents of "
+                          str::stream() << "Unrecognized field '" << fieldName << "'. Contents of "
                                         << FeatureCompatibilityVersion::kParameterName
                                         << " document in "
                                         << FeatureCompatibilityVersion::kCollection
                                         << ": "
-                                        << featureCompatibilityVersionDoc
+                                        << featureCompatibilityVersionInfo
                                         << ". See "
                                         << feature_compatibility_version::kDochubLink
                                         << ".");
         }
     }
 
-    if (!foundVersionField) {
+    if (versionInfo.version == ServerGlobalParams::FeatureCompatibility::Version::kUnset) {
         return Status(ErrorCodes::BadValue,
                       str::stream() << "Missing required field '"
                                     << FeatureCompatibilityVersion::kVersionField
@@ -160,45 +135,45 @@ StatusWith<ServerGlobalParams::FeatureCompatibility::Version> FeatureCompatibili
                                     << " document in "
                                     << FeatureCompatibilityVersion::kCollection
                                     << ": "
-                                    << featureCompatibilityVersionDoc
+                                    << featureCompatibilityVersionInfo
                                     << ". See "
                                     << feature_compatibility_version::kDochubLink
                                     << ".");
     }
 
-    return version;
+    return versionInfo;
 }
 
 void FeatureCompatibilityVersion::set(OperationContext* opCtx, StringData version) {
-    uassert(40284,
-            str::stream() << "featureCompatibilityVersion must be '"
-                          << FeatureCompatibilityVersionCommandParser::kVersion36
-                          << "' or '"
-                          << FeatureCompatibilityVersionCommandParser::kVersion34
-                          << "'. See "
-                          << feature_compatibility_version::kDochubLink
-                          << ".",
-            version == FeatureCompatibilityVersionCommandParser::kVersion36 ||
-                version == FeatureCompatibilityVersionCommandParser::kVersion34);
+    // Upgrades to a version, which sets the 'version' field only.
+    _runUpdateCommand(opCtx, version, [version](auto updateMods) {
+        BSONObjBuilder setOp(updateMods.subobjStart("$set"));
+        setOp.append(FeatureCompatibilityVersion::kVersionField, version);
+    });
+}
 
-    DBDirectClient client(opCtx);
-    NamespaceString nss(FeatureCompatibilityVersion::kCollection);
+void FeatureCompatibilityVersion::setTargetUpgrade(OperationContext* opCtx, StringData version) {
+    // Only set 'targetVersion' field.
+    _runUpdateCommand(opCtx, version, [version](auto updateMods) {
+        BSONObjBuilder setOp(updateMods.subobjStart("$set"));
+        setOp.append(FeatureCompatibilityVersion::kTargetVersionField, version);
+    });
+}
 
-    // Update the featureCompatibilityVersion document stored in the "admin.system.version"
-    // collection.
-    BSONObj updateResult;
-    client.runCommand(nss.db().toString(),
-                      makeUpdateCommand(version, WriteConcernOptions::Majority),
-                      updateResult);
-    uassertStatusOK(getStatusFromCommandResult(updateResult));
-    uassertStatusOK(getWriteConcernStatusFromCommandResult(updateResult));
+void FeatureCompatibilityVersion::setTargetDowngrade(OperationContext* opCtx, StringData version) {
+    // Set both 'version' and 'targetVersion' fields.
+    _runUpdateCommand(opCtx, version, [version](auto updateMods) {
+        updateMods.append(FeatureCompatibilityVersion::kVersionField, version);
+        updateMods.append(FeatureCompatibilityVersion::kTargetVersionField, version);
+    });
+}
 
-    // Close all internal connections to versions lower than 3.6.
-    if (version == FeatureCompatibilityVersionCommandParser::kVersion36) {
-        opCtx->getServiceContext()->getServiceEntryPoint()->endAllSessions(
-            transport::Session::kLatestVersionInternalClientKeepOpen |
-            transport::Session::kExternalClientKeepOpen);
-    }
+void FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(OperationContext* opCtx,
+                                                                StringData version) {
+    // Updates 'version' field, while also unsetting the 'targetVersion' field.
+    _runUpdateCommand(opCtx, version, [version](auto updateMods) {
+        updateMods.append(FeatureCompatibilityVersion::kVersionField, version);
+    });
 }
 
 void FeatureCompatibilityVersion::setIfCleanStartup(OperationContext* opCtx,
@@ -225,9 +200,10 @@ void FeatureCompatibilityVersion::setIfCleanStartup(OperationContext* opCtx,
             // the "admin.system.version" collection.
             invariant(autoDB.justCreated());
 
-            // We update the value of the isSchemaVersion36 server parameter so the
-            // admin.system.version collection gets a UUID.
-            serverGlobalParams.featureCompatibility.isSchemaVersion36.store(true);
+            // We update the value of the version server parameter so that the admin.system.version
+            // collection gets a UUID.
+            serverGlobalParams.featureCompatibility.version.store(
+                ServerGlobalParams::FeatureCompatibility::Version::k36);
 
             uassertStatusOK(storageInterface->createCollection(opCtx, nss, {}));
         }
@@ -253,13 +229,27 @@ void FeatureCompatibilityVersion::onInsertOrUpdate(OperationContext* opCtx, cons
         idElement.String() != FeatureCompatibilityVersion::kParameterName) {
         return;
     }
-    auto newVersion = uassertStatusOK(FeatureCompatibilityVersion::parse(doc));
-    log() << "setting featureCompatibilityVersion to " << toString(newVersion);
-    opCtx->recoveryUnit()->onCommit([newVersion]() {
-        serverGlobalParams.featureCompatibility.version.store(newVersion);
-        serverGlobalParams.featureCompatibility.isSchemaVersion36.store(
-            serverGlobalParams.featureCompatibility.version.load() ==
-            ServerGlobalParams::FeatureCompatibility::Version::k36);
+    auto versionInfo = uassertStatusOK(FeatureCompatibilityVersion::parse(doc));
+
+    // Indicate if an upgrade or downgrade is in progress.
+    if (versionInfo.targetVersion != ServerGlobalParams::FeatureCompatibility::Version::kUnset) {
+        log() << "targeting featureCompatibilityVersion " << toString(versionInfo.targetVersion);
+    }
+
+    // To avoid extra log messages when the targetVersion is set/unset, only log when the version
+    // changes.
+    auto oldVersion = serverGlobalParams.featureCompatibility.version.load();
+    auto newVersion = versionInfo.version;
+    if (oldVersion != newVersion) {
+        log() << "setting featureCompatibilityVersion to " << toString(newVersion);
+    }
+
+    // On commit, update the server parameters, and close any incoming connections with a wire
+    // version that is below the minimum.
+    opCtx->recoveryUnit()->onCommit([opCtx, versionInfo]() {
+        serverGlobalParams.featureCompatibility.version.store(versionInfo.version);
+        serverGlobalParams.featureCompatibility.targetVersion.store(versionInfo.targetVersion);
+        _closeConnectionsBelowVersion(opCtx, versionInfo);
     });
 }
 
@@ -274,7 +264,8 @@ void FeatureCompatibilityVersion::onDelete(OperationContext* opCtx, const BSONOb
     opCtx->recoveryUnit()->onCommit([]() {
         serverGlobalParams.featureCompatibility.version.store(
             ServerGlobalParams::FeatureCompatibility::Version::k34);
-        serverGlobalParams.featureCompatibility.isSchemaVersion36.store(false);
+        serverGlobalParams.featureCompatibility.targetVersion.store(
+            ServerGlobalParams::FeatureCompatibility::Version::kUnset);
     });
 }
 
@@ -284,10 +275,70 @@ void FeatureCompatibilityVersion::onDropCollection(OperationContext* opCtx) {
     opCtx->recoveryUnit()->onCommit([]() {
         serverGlobalParams.featureCompatibility.version.store(
             ServerGlobalParams::FeatureCompatibility::Version::k34);
-        serverGlobalParams.featureCompatibility.isSchemaVersion36.store(false);
+        serverGlobalParams.featureCompatibility.targetVersion.store(
+            ServerGlobalParams::FeatureCompatibility::Version::kUnset);
     });
 }
 
+void FeatureCompatibilityVersion::_validateVersion(StringData version) {
+    uassert(40284,
+            str::stream() << "featureCompatibilityVersion must be '"
+                          << FeatureCompatibilityVersionCommandParser::kVersion36
+                          << "' or '"
+                          << FeatureCompatibilityVersionCommandParser::kVersion34
+                          << "'. See "
+                          << feature_compatibility_version::kDochubLink
+                          << ".",
+            version == FeatureCompatibilityVersionCommandParser::kVersion36 ||
+                version == FeatureCompatibilityVersionCommandParser::kVersion34);
+}
+
+void FeatureCompatibilityVersion::_closeConnectionsBelowVersion(
+    OperationContext* opCtx, FeatureCompatibilityVersionInfo versionInfo) {
+
+    // Close all internal connections to versions lower than 3.6.
+    if (versionInfo.version == ServerGlobalParams::FeatureCompatibility::Version::k36 ||
+        versionInfo.targetVersion == ServerGlobalParams::FeatureCompatibility::Version::k36) {
+        opCtx->getServiceContext()->getServiceEntryPoint()->endAllSessions(
+            transport::Session::kLatestVersionInternalClientKeepOpen |
+            transport::Session::kExternalClientKeepOpen);
+    }
+}
+
+void FeatureCompatibilityVersion::_runUpdateCommand(OperationContext* opCtx,
+                                                    StringData version,
+                                                    UpdateBuilder builder) {
+    _validateVersion(version);
+
+    DBDirectClient client(opCtx);
+    NamespaceString nss(FeatureCompatibilityVersion::kCollection);
+
+    BSONObjBuilder updateCmd;
+    updateCmd.append("update", nss.coll());
+    {
+        BSONArrayBuilder updates(updateCmd.subarrayStart("updates"));
+        {
+            BSONObjBuilder updateSpec(updates.subobjStart());
+            {
+                BSONObjBuilder queryFilter(updateSpec.subobjStart("q"));
+                queryFilter.append("_id", FeatureCompatibilityVersion::kParameterName);
+            }
+            {
+                BSONObjBuilder updateMods(updateSpec.subobjStart("u"));
+                builder(std::move(updateMods));
+            }
+            updateSpec.appendBool("upsert", true);
+        }
+    }
+    updateCmd.append(WriteConcernOptions::kWriteConcernField, WriteConcernOptions::Majority);
+
+    // Update the featureCompatibilityVersion document stored in the "admin.system.version"
+    // collection.
+    BSONObj updateResult;
+    client.runCommand(nss.db().toString(), updateCmd.obj(), updateResult);
+    uassertStatusOK(getStatusFromCommandResult(updateResult));
+    uassertStatusOK(getWriteConcernStatusFromCommandResult(updateResult));
+}
 /**
  * Read-only server parameter for featureCompatibilityVersion.
  */
diff --git a/src/mongo/db/commands/feature_compatibility_version.h b/src/mongo/db/commands/feature_compatibility_version.h
index d89b2e996d5..abb48b82101 100644
--- a/src/mongo/db/commands/feature_compatibility_version.h
+++ b/src/mongo/db/commands/feature_compatibility_version.h
@@ -41,6 +41,18 @@ class BSONObj;
 class OperationContext;
 
 /**
+ * Store state of featureCompatibilityVersion document.
+ **/
+struct FeatureCompatibilityVersionInfo {
+    ServerGlobalParams::FeatureCompatibility::Version version;
+    ServerGlobalParams::FeatureCompatibility::Version targetVersion;
+
+    FeatureCompatibilityVersionInfo()
+        : version(ServerGlobalParams::FeatureCompatibility::Version::kUnset),
+          targetVersion(ServerGlobalParams::FeatureCompatibility::Version::kUnset) {}
+};
+
+/**
  * Startup parameter to ignore featureCompatibilityVersion checks. This parameter cannot be set if
  * the node is started with --replSet, --master, or --slave. This should never be set by end users.
  */
@@ -53,12 +65,13 @@ public:
     static constexpr StringData kDatabase = "admin"_sd;
     static constexpr StringData kParameterName = "featureCompatibilityVersion"_sd;
     static constexpr StringData kVersionField = "version"_sd;
+    static constexpr StringData kTargetVersionField = "targetVersion"_sd;
 
     /**
      * Parses the featureCompatibilityVersion document from admin.system.version, and returns the
      * version.
      */
-    static StatusWith<ServerGlobalParams::FeatureCompatibility::Version> parse(
+    static StatusWith<FeatureCompatibilityVersionInfo> parse(
         const BSONObj& featureCompatibilityVersionDoc);
 
     static StringData toString(ServerGlobalParams::FeatureCompatibility::Version version) {
@@ -67,6 +80,8 @@ public:
                 return FeatureCompatibilityVersionCommandParser::kVersion36;
             case ServerGlobalParams::FeatureCompatibility::Version::k34:
                 return FeatureCompatibilityVersionCommandParser::kVersion34;
+            case ServerGlobalParams::FeatureCompatibility::Version::kUnset:
+                return FeatureCompatibilityVersionCommandParser::kVersionUnset;
             default:
                 MONGO_UNREACHABLE;
         }
@@ -80,6 +95,27 @@ public:
     static void set(OperationContext* opCtx, StringData version);
 
     /**
+     * Indicate intent to perform an upgrade. Should be set before schemas are modified.
+     * This sets the 'targetVersion' field only.
+     * Use unsetTargetUpgradeOrDowngrade to indicate that the schemas have completed the upgrading.
+     */
+    static void setTargetUpgrade(OperationContext* opCtx, StringData version);
+
+    /**
+     * Indicate intent to perform a downgrade. Should be set before schemas are modified.
+     * This atomically updates both the 'version' and 'targetVersion' fields.
+     * Use unsetTargetUpgradeOrDowngrade to indicate that the schemas have completed downgrading.
+     */
+    static void setTargetDowngrade(OperationContext* opCtx, StringData version);
+
+    /**
+     * Indicate the completion of an upgrade or downgrade. Should be set only when schemas are
+     * done being upgraded or modified.
+     * Unsets the 'targetVersion' field and updates the 'version' field.
+     */
+    static void unsetTargetUpgradeOrDowngrade(OperationContext* opCtx, StringData version);
+
+    /**
      * If there are no non-local databases and we are not running with --shardsvr, set
      * featureCompatibilityVersion to the latest value.
      */
@@ -104,6 +140,28 @@ public:
      * Resets the server parameter to its default value on commit.
      */
     static void onDropCollection(OperationContext* opCtx);
+
+private:
+    /**
+     * Validate version. Uasserts if invalid.
+     */
+    static void _validateVersion(StringData version);
+
+    /**
+     * Close incoming connections from interal clients who cannot speak our highest wire protocol
+     * version.
+     */
+    static void _closeConnectionsBelowVersion(OperationContext* opCtx,
+                                              FeatureCompatibilityVersionInfo versionInfo);
+
+    /**
+     * Build update command.
+     */
+    typedef stdx::function<void(BSONObjBuilder)> UpdateBuilder;
+    static void _runUpdateCommand(OperationContext* opCtx,
+                                  StringData version,
+                                  UpdateBuilder callback);
 };
 
+
 }  // namespace mongo
diff --git a/src/mongo/db/commands/feature_compatibility_version_command_parser.cpp b/src/mongo/db/commands/feature_compatibility_version_command_parser.cpp
index ae781c8137f..1859bc13f9e 100644
--- a/src/mongo/db/commands/feature_compatibility_version_command_parser.cpp
+++ b/src/mongo/db/commands/feature_compatibility_version_command_parser.cpp
@@ -43,6 +43,7 @@ constexpr StringData kVersion32 = "3.2"_sd;
 
 constexpr StringData FeatureCompatibilityVersionCommandParser::kVersion34;
 constexpr StringData FeatureCompatibilityVersionCommandParser::kVersion36;
+constexpr StringData FeatureCompatibilityVersionCommandParser::kVersionUnset;
 
 StatusWith<std::string> FeatureCompatibilityVersionCommandParser::extractVersionFromCommand(
     StringData commandName, const BSONObj& cmdObj) {
diff --git a/src/mongo/db/commands/feature_compatibility_version_command_parser.h b/src/mongo/db/commands/feature_compatibility_version_command_parser.h
index c4544fdbff1..498c9c8261d 100644
--- a/src/mongo/db/commands/feature_compatibility_version_command_parser.h
+++ b/src/mongo/db/commands/feature_compatibility_version_command_parser.h
@@ -49,6 +49,7 @@ public:
      */
     static constexpr StringData kVersion34 = "3.4"_sd;
     static constexpr StringData kVersion36 = "3.6"_sd;
+    static constexpr StringData kVersionUnset = "Unset"_sd;
 
     /**
      * Interprets the specified BSON as a command and extracts the desired compatibility version
diff --git a/src/mongo/db/commands/mr.cpp b/src/mongo/db/commands/mr.cpp
index 84ebb935f39..b9ecf5b4006 100644
--- a/src/mongo/db/commands/mr.cpp
+++ b/src/mongo/db/commands/mr.cpp
@@ -421,7 +421,7 @@ void State::prepTempCollection() {
             options.setNoIdIndex();
             options.temp = true;
             if (enableCollectionUUIDs &&
-                serverGlobalParams.featureCompatibility.isSchemaVersion36.load() == true) {
+                serverGlobalParams.featureCompatibility.isSchemaVersion36()) {
                 options.uuid.emplace(UUID::gen());
             }
             incColl = incCtx.db()->createCollection(_opCtx, _config.incLong.ns(), options);
@@ -505,8 +505,7 @@ void State::prepTempCollection() {
 
         CollectionOptions options = finalOptions;
         options.temp = true;
-        if (enableCollectionUUIDs &&
-            serverGlobalParams.featureCompatibility.isSchemaVersion36.load() == true) {
+        if (enableCollectionUUIDs && serverGlobalParams.featureCompatibility.isSchemaVersion36()) {
             // If a UUID for the final output collection was sent by mongos (i.e., the final output
             // collection is sharded), use the UUID mongos sent when creating the temp collection.
             // When the temp collection is renamed to the final output collection, the UUID will be
diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index 77bfd2bb8d9..92d8ce3e0e3 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -128,30 +128,33 @@ public:
                 opCtx, writeConcern);
         }
 
+        // Perform an upgrade
         if (version != existingVersion && isFCVUpgrade(version)) {
-            serverGlobalParams.featureCompatibility.isSchemaVersion36.store(true);
+            // Set targetVersion to indicate that an upgrade is in progress, then unset when done.
+            FeatureCompatibilityVersion::setTargetUpgrade(opCtx, version);
             updateUUIDSchemaVersion(opCtx, /*upgrade*/ true);
-            existingVersion = version;
-        }
-
-        // For reproducing failure after adding UUIDs and before upgrade FCV document.
-        if (MONGO_FAIL_POINT(featureCompatibilityUpgrade)) {
-            exitCleanly(EXIT_CLEAN);
-        }
 
-        FeatureCompatibilityVersion::set(opCtx, version);
-
-        // For reproducing failure after downgrading FCV document, and before removing UUIDs.
-        if (MONGO_FAIL_POINT(featureCompatibilityDowngrade)) {
-            serverGlobalParams.featureCompatibility.isSchemaVersion36.store(false);
-            exitCleanly(EXIT_CLEAN);
-        }
+            // For reproducing failure after adding UUIDs and before upgrade FCV document.
+            if (MONGO_FAIL_POINT(featureCompatibilityUpgrade)) {
+                exitCleanly(EXIT_CLEAN);
+            }
+            // Unsets 'targetVersion' and sets 'version' to new version.
+            FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, version);
+
+            // Always perform a downgrade when setting the FCV to 3.4, in case of a crash after
+            // writing the document and before completing the upgrade.
+        } else if (version == FeatureCompatibilityVersionCommandParser::kVersion34) {
+            // Set targetVersion to indicate that a downgrade is in progress, then unset when
+            // complete.
+            FeatureCompatibilityVersion::setTargetDowngrade(opCtx, version);
+
+            // For reproducing failure after downgrading FCV document, and before removing UUIDs.
+            if (MONGO_FAIL_POINT(featureCompatibilityDowngrade)) {
+                exitCleanly(EXIT_CLEAN);
+            }
 
-        // Always perform a downgrade when setting the FCV to 3.4, in case of a crash after writing
-        // the document and before completing the upgrade.
-        if (version == FeatureCompatibilityVersionCommandParser::kVersion34) {
-            serverGlobalParams.featureCompatibility.isSchemaVersion36.store(false);
             updateUUIDSchemaVersion(opCtx, /*upgrade*/ false);
+            FeatureCompatibilityVersion::unsetTargetUpgradeOrDowngrade(opCtx, version);
         }
 
         // Ensure we try reading the keys for signing clusterTime immediately on upgrade to 3.6.
diff --git a/src/mongo/db/db.cpp b/src/mongo/db/db.cpp
index 8f29f0605bc..ed3ba14e852 100644
--- a/src/mongo/db/db.cpp
+++ b/src/mongo/db/db.cpp
@@ -405,17 +405,33 @@ void repairDatabasesAndCheckVersion(OperationContext* opCtx) {
                                      versionColl,
                                      BSON("_id" << FeatureCompatibilityVersion::kParameterName),
                                      featureCompatibilityVersion)) {
-                    auto version = FeatureCompatibilityVersion::parse(featureCompatibilityVersion);
-                    if (!version.isOK()) {
-                        severe() << version.getStatus();
+                    auto swVersionInfo =
+                        FeatureCompatibilityVersion::parse(featureCompatibilityVersion);
+                    if (!swVersionInfo.isOK()) {
+                        severe() << swVersionInfo.getStatus();
                         fassertFailedNoTrace(40283);
                     }
-                    serverGlobalParams.featureCompatibility.version.store(version.getValue());
-
-                    // Update schemaVersion parameter.
-                    serverGlobalParams.featureCompatibility.isSchemaVersion36.store(
-                        serverGlobalParams.featureCompatibility.version.load() ==
-                        ServerGlobalParams::FeatureCompatibility::Version::k36);
+                    auto versionInfo = swVersionInfo.getValue();
+                    serverGlobalParams.featureCompatibility.version.store(versionInfo.version);
+                    serverGlobalParams.featureCompatibility.targetVersion.store(
+                        versionInfo.targetVersion);
+
+                    // On startup, if the targetVersion field exists, then an upgrade/downgrade
+                    // did not complete successfully.
+                    if (versionInfo.targetVersion !=
+                        ServerGlobalParams::FeatureCompatibility::Version::kUnset) {
+                        log() << "** WARNING: A featureCompatibilityVersion upgrade or downgrade "
+                                 "did not complete. "
+                              << startupWarningsLog;
+                        log() << "**          The current featureCompatibilityVersion is "
+                              << FeatureCompatibilityVersion::toString(versionInfo.version)
+                              << " and the targeted version is "
+                              << FeatureCompatibilityVersion::toString(versionInfo.targetVersion)
+                              << "." << startupWarningsLog;
+                        log() << "**          To fix this, use the setFeatureCompatibilityVersion "
+                              << "command to resume upgrade to 3.6 or downgrade to 3.4."
+                              << startupWarningsLog;
+                    }
                 }
             }
         }
diff --git a/src/mongo/db/server_options.h b/src/mongo/db/server_options.h
index a82d20acc79..23f8bddff4f 100644
--- a/src/mongo/db/server_options.h
+++ b/src/mongo/db/server_options.h
@@ -158,14 +158,22 @@ struct ServerGlobalParams {
              * 3.4 node can participate in a cluster whose feature compatibility version is 3.6.
              */
             k36,
+            /**
+             * This is only used for targetVersion to indicate that no upgrade is in progress.
+             */
+            kUnset
         };
 
         // Read-only parameter featureCompatibilityVersion.
         AtomicWord<Version> version{Version::k34};
 
-        // Read-only global isSchemaVersion36. This determines whether to give Collections UUIDs
-        // upon creation.
-        AtomicWord<bool> isSchemaVersion36{false};
+        // If set, an upgrade or downgrade is in progress to the set version.
+        AtomicWord<Version> targetVersion{Version::kUnset};
+
+        // This determines whether to give Collections UUIDs upon creation.
+        bool isSchemaVersion36() {
+            return (version.load() == Version::k36 || targetVersion.load() == Version::k36);
+        }
 
         // Feature validation differs depending on the role of a mongod in a replica set or
         // master/slave configuration. Masters/primaries can accept user-initiated writes and
diff --git a/src/mongo/dbtests/dbtests.cpp b/src/mongo/dbtests/dbtests.cpp
index c4f26023093..8721fc4dd48 100644
--- a/src/mongo/dbtests/dbtests.cpp
+++ b/src/mongo/dbtests/dbtests.cpp
@@ -123,7 +123,6 @@ int dbtestsMain(int argc, char** argv, char** envp) {
     mongo::runGlobalInitializersOrDie(argc, argv, envp);
     serverGlobalParams.featureCompatibility.version.store(
         ServerGlobalParams::FeatureCompatibility::Version::k36);
-    serverGlobalParams.featureCompatibility.isSchemaVersion36.store(true);
     repl::ReplSettings replSettings;
     replSettings.setOplogSizeBytes(10 * 1024 * 1024);
     ServiceContext* service = getGlobalServiceContext();