SERVER-62266: Serialize setClusterParameter and drop cluster parameter collection during FCV 6.0 downgrade

6 files changed, 86 insertions, 41 deletions

diff --git a/src/mongo/db/commands/set_cluster_parameter_command.cpp b/src/mongo/db/commands/set_cluster_parameter_command.cpp
index 31df926ce68..005f17d0c06 100644
--- a/src/mongo/db/commands/set_cluster_parameter_command.cpp
+++ b/src/mongo/db/commands/set_cluster_parameter_command.cpp
@@ -34,6 +34,7 @@
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/commands.h"
 #include "mongo/db/commands/cluster_server_parameter_cmds_gen.h"
+#include "mongo/db/commands/feature_compatibility_version.h"
 #include "mongo/db/commands/set_cluster_parameter_invocation.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/idl/cluster_server_parameter_gen.h"
@@ -68,15 +69,16 @@ public:
         using InvocationBase::InvocationBase;
 
         void typedRun(OperationContext* opCtx) {
+            uassert(ErrorCodes::ErrorCodes::NotImplemented,
+                    "setClusterParameter can only run on mongos in sharded clusters",
+                    (serverGlobalParams.clusterRole == ClusterRole::None));
+
+            FixedFCVRegion fcvRegion(opCtx);
             uassert(
                 ErrorCodes::IllegalOperation,
                 "Cannot set cluster parameter, gFeatureFlagClusterWideConfig is not enabled",
                 gFeatureFlagClusterWideConfig.isEnabled(serverGlobalParams.featureCompatibility));
 
-            uassert(ErrorCodes::ErrorCodes::NotImplemented,
-                    "setClusterParameter can only run on mongos in sharded clusters",
-                    (serverGlobalParams.clusterRole == ClusterRole::None));
-
             // TODO SERVER-65249: This will eventually be made specific to the parameter being set
             // so that some parameters will be able to use setClusterParameter even on standalones.
             uassert(ErrorCodes::IllegalOperation,
diff --git a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
index 2ecf3bce9f0..a56f54ebbcb 100644
--- a/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
+++ b/src/mongo/db/commands/set_feature_compatibility_version_command.cpp
@@ -82,6 +82,7 @@
 #include "mongo/db/session_txn_record_gen.h"
 #include "mongo/db/timeseries/timeseries_index_schema_conversion_functions.h"
 #include "mongo/db/vector_clock.h"
+#include "mongo/idl/cluster_server_parameter_gen.h"
 #include "mongo/logv2/log.h"
 #include "mongo/rpc/get_status_from_command_result.h"
 #include "mongo/s/pm2423_feature_flags_gen.h"
@@ -370,7 +371,7 @@ public:
                             ErrorCodes::CannotDowngrade,
                             "Cannot downgrade while user write blocking is being changed",
                             ConfigsvrCoordinatorService::getService(opCtx)
-                                ->isAnyCoordinatorOfGivenTypeRunning(
+                                ->areAllCoordinatorsOfTypeFinished(
                                     opCtx, ConfigsvrCoordinatorTypeEnum::kSetUserWriteBlockMode));
                     }
 
@@ -384,6 +385,30 @@ public:
                             !isBlockingUserWrites);
                 }
 
+                // TODO (SERVER-65572): Remove setClusterParameter serialization and collection
+                // drop after this is backported to 6.0.
+                if (!gFeatureFlagClusterWideConfig.isEnabledOnVersion(requestedVersion)) {
+                    if (serverGlobalParams.clusterRole == ClusterRole::ConfigServer) {
+                        uassert(ErrorCodes::CannotDowngrade,
+                                "Cannot downgrade while cluster server parameter is being set",
+                                ConfigsvrCoordinatorService::getService(opCtx)
+                                    ->areAllCoordinatorsOfTypeFinished(
+                                        opCtx, ConfigsvrCoordinatorTypeEnum::kSetClusterParameter));
+                    }
+
+                    DropReply dropReply;
+                    const auto dropStatus = dropCollection(
+                        opCtx,
+                        NamespaceString::kClusterParametersNamespace,
+                        &dropReply,
+                        DropCollectionSystemCollectionMode::kAllowSystemCollectionDrops);
+                    uassert(
+                        dropStatus.code(),
+                        str::stream() << "Failed to drop the cluster server parameters collection"
+                                      << causedBy(dropStatus.reason()),
+                        dropStatus.isOK() || dropStatus.code() == ErrorCodes::NamespaceNotFound);
+                }
+
                 FeatureCompatibilityVersion::updateFeatureCompatibilityVersionDocument(
                     opCtx,
                     actualVersion,
diff --git a/src/mongo/db/s/config/configsvr_coordinator_service.cpp b/src/mongo/db/s/config/configsvr_coordinator_service.cpp
index b3ba449ac0f..d7ff2d006e0 100644
--- a/src/mongo/db/s/config/configsvr_coordinator_service.cpp
+++ b/src/mongo/db/s/config/configsvr_coordinator_service.cpp
@@ -93,10 +93,11 @@ ConfigsvrCoordinatorService::constructInstance(BSONObj initialState) {
     }
 }
 
-bool ConfigsvrCoordinatorService::isAnyCoordinatorOfGivenTypeRunning(
+bool ConfigsvrCoordinatorService::areAllCoordinatorsOfTypeFinished(
     OperationContext* opCtx, ConfigsvrCoordinatorTypeEnum coordinatorType) {
 
-    const auto instances = getAllInstances(opCtx);
+    // First, check if all in-memory ConfigsvrCoordinators are finished.
+    const auto& instances = getAllInstances(opCtx);
     for (const auto& instance : instances) {
         auto typedInstance = checked_pointer_cast<ConfigsvrCoordinator>(instance);
         if (typedInstance->coordinatorType() == coordinatorType) {
@@ -106,7 +107,14 @@ bool ConfigsvrCoordinatorService::isAnyCoordinatorOfGivenTypeRunning(
         }
     }
 
-    return true;
+    // If the POS has just been rebuilt on a newly-elected primary, there is a chance that the
+    // the coordinator instance does not exist yet. Query the state document namespace for any
+    // documents that will be built into instances.
+    DBDirectClient client(opCtx);
+    FindCommandRequest findStateDocs{NamespaceString::kConfigsvrCoordinatorsNamespace};
+    findStateDocs.setFilter(BSON("_id" << BSON("coordinatorType" << coordinatorType)));
+
+    return !client.find(std::move(findStateDocs))->more();
 }
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/config/configsvr_coordinator_service.h b/src/mongo/db/s/config/configsvr_coordinator_service.h
index 7b8f30a3ed3..dfc2ccddb38 100644
--- a/src/mongo/db/s/config/configsvr_coordinator_service.h
+++ b/src/mongo/db/s/config/configsvr_coordinator_service.h
@@ -68,8 +68,8 @@ public:
 
     std::shared_ptr<Instance> constructInstance(BSONObj initialState) override;
 
-    bool isAnyCoordinatorOfGivenTypeRunning(OperationContext* opCtx,
-                                            ConfigsvrCoordinatorTypeEnum coordinatorType);
+    bool areAllCoordinatorsOfTypeFinished(OperationContext* opCtx,
+                                          ConfigsvrCoordinatorTypeEnum coordinatorType);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp b/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp
index 6a0109194f6..da76faa6593 100644
--- a/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp
+++ b/src/mongo/db/s/config/configsvr_set_cluster_parameter_command.cpp
@@ -33,6 +33,7 @@
 
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/commands.h"
+#include "mongo/db/commands/feature_compatibility_version.h"
 #include "mongo/db/commands/set_cluster_parameter_invocation.h"
 #include "mongo/db/s/config/configsvr_coordinator_service.h"
 #include "mongo/db/s/config/set_cluster_parameter_coordinator.h"
@@ -58,46 +59,51 @@ public:
                     str::stream() << Request::kCommandName << " can only be run on config servers",
                     serverGlobalParams.clusterRole == ClusterRole::ConfigServer);
 
-            uassert(
-                ErrorCodes::IllegalOperation,
-                "featureFlagClusterWideConfig not enabled",
-                gFeatureFlagClusterWideConfig.isEnabled(serverGlobalParams.featureCompatibility));
+            const auto coordinatorCompletionFuture = [&]() -> SharedSemiFuture<void> {
+                FixedFCVRegion fcvRegion(opCtx);
+                uassert(ErrorCodes::IllegalOperation,
+                        "featureFlagClusterWideConfig not enabled",
+                        gFeatureFlagClusterWideConfig.isEnabled(
+                            serverGlobalParams.featureCompatibility));
 
-            // Validate parameter before creating coordinator.
-            {
-                BSONObj cmdParamObj = request().getCommandParameter();
-                BSONElement commandElement = cmdParamObj.firstElement();
-                StringData parameterName = commandElement.fieldName();
-                std::unique_ptr<ServerParameterService> sps =
-                    std::make_unique<ClusterParameterService>();
-                const ServerParameter* serverParameter = sps->getIfExists(parameterName);
+                // Validate parameter before creating coordinator.
+                {
+                    BSONObj cmdParamObj = request().getCommandParameter();
+                    BSONElement commandElement = cmdParamObj.firstElement();
+                    StringData parameterName = commandElement.fieldName();
+                    std::unique_ptr<ServerParameterService> sps =
+                        std::make_unique<ClusterParameterService>();
+                    const ServerParameter* serverParameter = sps->getIfExists(parameterName);
 
-                uassert(ErrorCodes::IllegalOperation,
-                        str::stream() << "Unknown Cluster Parameter " << parameterName,
-                        serverParameter != nullptr);
+                    uassert(ErrorCodes::IllegalOperation,
+                            str::stream() << "Unknown Cluster Parameter " << parameterName,
+                            serverParameter != nullptr);
 
-                uassert(ErrorCodes::IllegalOperation,
-                        "Cluster parameter value must be an object",
-                        BSONType::Object == commandElement.type());
+                    uassert(ErrorCodes::IllegalOperation,
+                            "Cluster parameter value must be an object",
+                            BSONType::Object == commandElement.type());
+
+                    BSONObjBuilder clusterParamBuilder;
+                    clusterParamBuilder << "_id" << parameterName;
+                    clusterParamBuilder.appendElements(commandElement.Obj());
 
-                BSONObjBuilder clusterParamBuilder;
-                clusterParamBuilder << "_id" << parameterName;
-                clusterParamBuilder.appendElements(commandElement.Obj());
+                    BSONObj clusterParam = clusterParamBuilder.obj();
 
-                BSONObj clusterParam = clusterParamBuilder.obj();
+                    uassertStatusOK(serverParameter->validate(clusterParam));
+                }
 
-                uassertStatusOK(serverParameter->validate(clusterParam));
-            }
+                SetClusterParameterCoordinatorDocument coordinatorDoc;
+                coordinatorDoc.setConfigsvrCoordinatorMetadata(
+                    {ConfigsvrCoordinatorTypeEnum::kSetClusterParameter});
+                coordinatorDoc.setParameter(request().getCommandParameter());
 
-            SetClusterParameterCoordinatorDocument coordinatorDoc;
-            coordinatorDoc.setConfigsvrCoordinatorMetadata(
-                {ConfigsvrCoordinatorTypeEnum::kSetClusterParameter});
-            coordinatorDoc.setParameter(request().getCommandParameter());
+                const auto service = ConfigsvrCoordinatorService::getService(opCtx);
+                const auto instance = service->getOrCreateService(opCtx, coordinatorDoc.toBSON());
 
-            const auto service = ConfigsvrCoordinatorService::getService(opCtx);
-            const auto instance = service->getOrCreateService(opCtx, coordinatorDoc.toBSON());
+                return instance->getCompletionFuture();
+            }();
 
-            instance->getCompletionFuture().get(opCtx);
+            coordinatorCompletionFuture.get(opCtx);
         }
 
     private:
diff --git a/src/mongo/db/s/shardsvr_set_cluster_parameter_command.cpp b/src/mongo/db/s/shardsvr_set_cluster_parameter_command.cpp
index c84ad1dd27e..08ddd4d5a01 100644
--- a/src/mongo/db/s/shardsvr_set_cluster_parameter_command.cpp
+++ b/src/mongo/db/s/shardsvr_set_cluster_parameter_command.cpp
@@ -45,6 +45,8 @@
 namespace mongo {
 namespace {
 
+MONGO_FAIL_POINT_DEFINE(hangInShardsvrSetClusterParameter);
+
 const WriteConcernOptions kLocalWriteConcern{
     1, WriteConcernOptions::SyncMode::UNSET, WriteConcernOptions::kNoTimeout};
 
@@ -64,6 +66,8 @@ public:
             CommandHelpers::uassertCommandRunWithMajority(Request::kCommandName,
                                                           opCtx->getWriteConcern());
 
+            hangInShardsvrSetClusterParameter.pauseWhileSet();
+
             SetClusterParameter setClusterParameterRequest(request().getCommandParameter());
             setClusterParameterRequest.setDbName(NamespaceString::kAdminDb);
             std::unique_ptr<ServerParameterService> parameterService =