SERVER-64245 Make SetUserWriteBlockModeCoordinator set the write blocking state on the shards

10 files changed, 268 insertions, 8 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index 9e51136bb28..c3152e8c8bf 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -2388,6 +2388,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/change_stream_options_manager',
         '$BUILD_DIR/mongo/db/commands/set_user_write_block_mode_command',
         '$BUILD_DIR/mongo/db/pipeline/change_stream_expired_pre_image_remover',
+        '$BUILD_DIR/mongo/db/s/shardsvr_set_user_write_block_mode_command',
         '$BUILD_DIR/mongo/idl/cluster_server_parameter',
         '$BUILD_DIR/mongo/idl/cluster_server_parameter_op_observer',
         '$BUILD_DIR/mongo/s/grid',
diff --git a/src/mongo/db/commands/set_user_write_block_mode_command.cpp b/src/mongo/db/commands/set_user_write_block_mode_command.cpp
index 3ecf129935a..89ce7edc6c3 100644
--- a/src/mongo/db/commands/set_user_write_block_mode_command.cpp
+++ b/src/mongo/db/commands/set_user_write_block_mode_command.cpp
@@ -61,6 +61,11 @@ public:
         using InvocationBase::InvocationBase;
 
         void typedRun(OperationContext* opCtx) {
+            uassert(ErrorCodes::IllegalOperation,
+                    str::stream() << Request::kCommandName
+                                  << " cannot be run on shardsvrs nor configsvrs",
+                    serverGlobalParams.clusterRole == ClusterRole::None);
+
             {
                 if (request().getGlobal()) {
                     UserWritesRecoverableCriticalSectionService::get(opCtx)
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index 952f4de9b9a..8a897f2f13f 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -176,13 +176,11 @@ env.Library(
         'user_writes_critical_section_document.idl',
         'user_writes_recoverable_critical_section_service.cpp',
     ],
-    LIBDEPS=[
-        'sharding_api_d',
-    ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/db/dbdirectclient',
         '$BUILD_DIR/mongo/db/repl/replica_set_aware_service',
         '$BUILD_DIR/mongo/db/rw_concern_d',
+        'sharding_api_d',
     ]
 )
 
@@ -460,6 +458,19 @@ env.Library(
 )
 
 env.Library(
+    target='shardsvr_set_user_write_block_mode_command',
+    source=[
+        'shardsvr_set_user_write_block_mode_command.cpp',
+    ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/auth/auth',
+        '$BUILD_DIR/mongo/db/commands',
+        '$BUILD_DIR/mongo/s/common_s',
+        'user_writes_recoverable_critical_section',
+    ]
+)
+
+env.Library(
     target='sharding_mongod_test_fixture',
     source=[
         'sharding_mongod_test_fixture.cpp',
diff --git a/src/mongo/db/s/config/set_user_write_block_mode_coordinator.cpp b/src/mongo/db/s/config/set_user_write_block_mode_coordinator.cpp
index fe3acbc5547..0271479e912 100644
--- a/src/mongo/db/s/config/set_user_write_block_mode_coordinator.cpp
+++ b/src/mongo/db/s/config/set_user_write_block_mode_coordinator.cpp
@@ -35,11 +35,53 @@
 
 #include "mongo/base/checked_cast.h"
 #include "mongo/db/persistent_task_store.h"
+#include "mongo/db/s/config/sharding_catalog_manager.h"
+#include "mongo/db/s/sharding_util.h"
 #include "mongo/logv2/log.h"
+#include "mongo/s/client/shard_registry.h"
+#include "mongo/s/grid.h"
 #include "mongo/s/request_types/sharded_ddl_commands_gen.h"
 
 namespace mongo {
 
+namespace {
+
+ShardsvrSetUserWriteBlockMode makeShardsvrSetUserWriteBlockModeCommand(
+    bool block, ShardsvrSetUserWriteBlockModePhaseEnum phase) {
+    ShardsvrSetUserWriteBlockMode shardsvrSetUserWriteBlockModeCmd;
+    shardsvrSetUserWriteBlockModeCmd.setDbName(NamespaceString::kAdminDb);
+    SetUserWriteBlockModeRequest setUserWriteBlockModeRequest(block /* global */);
+    shardsvrSetUserWriteBlockModeCmd.setSetUserWriteBlockModeRequest(
+        std::move(setUserWriteBlockModeRequest));
+    shardsvrSetUserWriteBlockModeCmd.setPhase(phase);
+
+    return shardsvrSetUserWriteBlockModeCmd;
+}
+
+void sendSetUserWriteBlockModeCmdToAllShards(OperationContext* opCtx,
+                                             std::shared_ptr<executor::TaskExecutor> executor,
+                                             bool block,
+                                             ShardsvrSetUserWriteBlockModePhaseEnum phase) {
+    // Ensure the topology is stable so we don't miss propagating the write blocking
+    // state to any concurrently added shard.
+    Lock::SharedLock stableTopologyRegion =
+        ShardingCatalogManager::get(opCtx)->enterStableTopologyRegion(opCtx);
+
+    const auto allShards = Grid::get(opCtx)->shardRegistry()->getAllShardIds(opCtx);
+
+    const auto shardsvrSetUserWriteBlockModeCmd =
+        makeShardsvrSetUserWriteBlockModeCommand(block, phase);
+
+    sharding_util::sendCommandToShards(
+        opCtx,
+        shardsvrSetUserWriteBlockModeCmd.getDbName(),
+        CommandHelpers::appendMajorityWriteConcern(shardsvrSetUserWriteBlockModeCmd.toBSON({})),
+        allShards,
+        executor);
+}
+
+}  // namespace
+
 bool SetUserWriteBlockModeCoordinator::hasSameOptions(const BSONObj& otherDocBSON) const {
     const auto otherDoc = StateDoc::parse(
         IDLParserErrorContext("SetUserWriteBlockModeCoordinatorDocument"), otherDocBSON);
@@ -90,8 +132,29 @@ ExecutorFuture<void> SetUserWriteBlockModeCoordinator::_runImpl(
     std::shared_ptr<executor::ScopedTaskExecutor> executor,
     const CancellationToken& token) noexcept {
     return ExecutorFuture<void>(**executor)
-        .then(_executePhase(Phase::kSetUserWriteBlockMode, [this, anchor = shared_from_this()] {
-            // TODO Implement
+        .then(_executePhase(Phase::kPrepare,
+                            [this, anchor = shared_from_this()] {
+                                auto opCtxHolder = cc().makeOperationContext();
+                                auto* opCtx = opCtxHolder.get();
+                                auto executor =
+                                    Grid::get(opCtx)->getExecutorPool()->getFixedExecutor();
+
+                                sendSetUserWriteBlockModeCmdToAllShards(
+                                    opCtx,
+                                    executor,
+                                    _doc.getBlock(),
+                                    ShardsvrSetUserWriteBlockModePhaseEnum::kPrepare);
+                            }))
+        .then(_executePhase(Phase::kComplete, [this, anchor = shared_from_this()] {
+            auto opCtxHolder = cc().makeOperationContext();
+            auto* opCtx = opCtxHolder.get();
+            auto executor = Grid::get(opCtx)->getExecutorPool()->getFixedExecutor();
+
+            sendSetUserWriteBlockModeCmdToAllShards(
+                opCtx,
+                executor,
+                _doc.getBlock(),
+                ShardsvrSetUserWriteBlockModePhaseEnum::kComplete);
         }));
 }
 
diff --git a/src/mongo/db/s/config/set_user_write_block_mode_coordinator_document.idl b/src/mongo/db/s/config/set_user_write_block_mode_coordinator_document.idl
index cf9f2a2dcd5..654624d03f1 100644
--- a/src/mongo/db/s/config/set_user_write_block_mode_coordinator_document.idl
+++ b/src/mongo/db/s/config/set_user_write_block_mode_coordinator_document.idl
@@ -37,11 +37,18 @@ imports:
 
 enums:
     SetUserWriteBlockModeCoordinatorPhase:
-        description: "Current refine collection shard key coordinator's operation state."
+        description: "Current SetUserWriteBlockMode coordinator phase"
         type: string
         values:
             kUnset: "unset"
-            kSetUserWriteBlockMode: "setUserWriteBlockMode"
+            # When enabling user write blocking, kPrepare will instruct the shards to start blocking
+            # new sharded DDL operations. When disabling user write blocking, kPrepare will instruct
+            # the shards to start accepting user writes again.
+            kPrepare: "prepare"
+            # When enabling user write blocking, kComplete will instruct the shards to start
+            # blocking user writes. When disabling user write blocking, kComplete will instruct the
+            # shards to start accepting sharded DDL operations again.
+            kComplete: "complete"
 
 structs:
     SetUserWriteBlockModeCoordinatorDocument:
diff --git a/src/mongo/db/s/config/sharding_catalog_manager.h b/src/mongo/db/s/config/sharding_catalog_manager.h
index 9653a0ce6d3..15cc5bbd6a6 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager.h
+++ b/src/mongo/db/s/config/sharding_catalog_manager.h
@@ -467,6 +467,12 @@ public:
      */
     RemoveShardProgress removeShard(OperationContext* opCtx, const ShardId& shardId);
 
+    /**
+     * Returns a scoped lock object, which holds the _kShardMembershipLock in shared mode. While
+     * this lock is held no topology changes can occur.
+     */
+    Lock::SharedLock enterStableTopologyRegion(OperationContext* opCtx);
+
     //
     // Cluster Upgrade Operations
     //
diff --git a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp
index cd97c57f44c..83e9361b6a3 100644
--- a/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp
+++ b/src/mongo/db/s/config/sharding_catalog_manager_shard_operations.cpp
@@ -998,6 +998,10 @@ RemoveShardProgress ShardingCatalogManager::removeShard(OperationContext* opCtx,
             boost::optional<RemoveShardProgress::DrainingShardUsage>(boost::none)};
 }
 
+Lock::SharedLock ShardingCatalogManager::enterStableTopologyRegion(OperationContext* opCtx) {
+    return Lock::SharedLock(opCtx->lockState(), _kShardMembershipLock);
+}
+
 void ShardingCatalogManager::appendConnectionStats(executor::ConnectionPoolStats* stats) {
     _executorForAddShard->appendConnectionStats(stats);
 }
diff --git a/src/mongo/db/s/shardsvr_set_user_write_block_mode_command.cpp b/src/mongo/db/s/shardsvr_set_user_write_block_mode_command.cpp
new file mode 100644
index 00000000000..36e2c22b041
--- /dev/null
+++ b/src/mongo/db/s/shardsvr_set_user_write_block_mode_command.cpp
@@ -0,0 +1,135 @@
+/**
+ *    Copyright (C) 2022-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kSharding
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/s/user_writes_recoverable_critical_section_service.h"
+#include "mongo/logv2/log.h"
+#include "mongo/s/request_types/sharded_ddl_commands_gen.h"
+
+namespace mongo {
+namespace {
+
+class ShardsvrSetUserWriteBlockCommand final
+    : public TypedCommand<ShardsvrSetUserWriteBlockCommand> {
+public:
+    using Request = ShardsvrSetUserWriteBlockMode;
+
+    class Invocation final : public InvocationBase {
+    public:
+        using InvocationBase::InvocationBase;
+
+        void typedRun(OperationContext* opCtx) {
+            uassert(ErrorCodes::IllegalOperation,
+                    str::stream() << Request::kCommandName << " can only be run on shard servers",
+                    serverGlobalParams.clusterRole == ClusterRole::ShardServer);
+            CommandHelpers::uassertCommandRunWithMajority(Request::kCommandName,
+                                                          opCtx->getWriteConcern());
+
+            const auto startBlocking = request().getGlobal();
+
+            if (startBlocking) {
+                switch (request().getPhase()) {
+                    case ShardsvrSetUserWriteBlockModePhaseEnum::kPrepare:
+                        UserWritesRecoverableCriticalSectionService::get(opCtx)
+                            ->acquireRecoverableCriticalSectionBlockNewShardedDDL(
+                                opCtx,
+                                UserWritesRecoverableCriticalSectionService::
+                                    kGlobalUserWritesNamespace);
+                        break;
+                    case ShardsvrSetUserWriteBlockModePhaseEnum::kComplete:
+                        UserWritesRecoverableCriticalSectionService::get(opCtx)
+                            ->promoteRecoverableCriticalSectionToBlockUserWrites(
+                                opCtx,
+                                UserWritesRecoverableCriticalSectionService::
+                                    kGlobalUserWritesNamespace);
+                        break;
+                    default:
+                        MONGO_UNREACHABLE;
+                }
+            } else {
+                switch (request().getPhase()) {
+                    case ShardsvrSetUserWriteBlockModePhaseEnum::kPrepare:
+                        UserWritesRecoverableCriticalSectionService::get(opCtx)
+                            ->demoteRecoverableCriticalSectionToNoLongerBlockUserWrites(
+                                opCtx,
+                                UserWritesRecoverableCriticalSectionService::
+                                    kGlobalUserWritesNamespace);
+                        break;
+                    case ShardsvrSetUserWriteBlockModePhaseEnum::kComplete:
+                        UserWritesRecoverableCriticalSectionService::get(opCtx)
+                            ->releaseRecoverableCriticalSection(
+                                opCtx,
+                                UserWritesRecoverableCriticalSectionService::
+                                    kGlobalUserWritesNamespace);
+                        break;
+                    default:
+                        MONGO_UNREACHABLE;
+                }
+            }
+        }
+
+    private:
+        NamespaceString ns() const override {
+            return NamespaceString();
+        }
+
+        bool supportsWriteConcern() const override {
+            return true;
+        }
+
+        void doCheckAuthorization(OperationContext* opCtx) const override {
+            uassert(ErrorCodes::Unauthorized,
+                    "Unauthorized",
+                    AuthorizationSession::get(opCtx->getClient())
+                        ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
+                                                           ActionType::internal));
+        }
+    };
+
+    std::string help() const override {
+        return "Internal command, which is exported by the shard servers. Do not call "
+               "directly. Enables/disables user write blocking on shardsvrs.";
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kNever;
+    }
+} shardsvrSetUserWriteBlockModeCmd;
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/s/commands/cluster_set_user_write_block_mode_command.cpp b/src/mongo/s/commands/cluster_set_user_write_block_mode_command.cpp
index cceb1449ddb..e95ae7c5f16 100644
--- a/src/mongo/s/commands/cluster_set_user_write_block_mode_command.cpp
+++ b/src/mongo/s/commands/cluster_set_user_write_block_mode_command.cpp
@@ -33,7 +33,6 @@
 
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/commands.h"
-#include "mongo/db/commands/set_user_write_block_mode_gen.h"
 #include "mongo/logv2/log.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/request_types/sharded_ddl_commands_gen.h"
diff --git a/src/mongo/s/request_types/sharded_ddl_commands.idl b/src/mongo/s/request_types/sharded_ddl_commands.idl
index 0d9cc898bd1..9bd7602a7dc 100644
--- a/src/mongo/s/request_types/sharded_ddl_commands.idl
+++ b/src/mongo/s/request_types/sharded_ddl_commands.idl
@@ -51,6 +51,20 @@ types:
         serializer: "mongo::CollectionType::toBSON"
         deserializer: "mongo::CollectionType"
 
+enums:
+    ShardsvrSetUserWriteBlockModePhase:
+        description: "Phase for the _shardsvrSetUserWriteBlockMode command."
+        type: string
+        values:
+            # When enabling user write blocking, kPrepare means the shard will start blocking
+            # sharded DDL operations. When disabling user write blocking, kPrepare means the shard
+            # will to start accepting user writes again.
+            kPrepare: "prepare"
+            # When enabling user write blocking, kComplete means the shard will start blocking
+            # user writes. When disabling user write blocking, kComplete means the shard
+            # will to start accepting sharded DDL operations again.
+            kComplete: "complete"
+
 structs:
 
     ConfigsvrCreateDatabaseResponse:
@@ -399,6 +413,21 @@ commands:
         chained_structs:
             SetUserWriteBlockModeRequest: SetUserWriteBlockModeRequest
 
+    _shardsvrSetUserWriteBlockMode:
+        command_name: _shardsvrSetUserWriteBlockMode
+        cpp_name: ShardsvrSetUserWriteBlockMode
+        description: "internal _shardsvrSetUserWriteBlockMode command"
+        namespace: ignored
+        api_version: ""
+        strict: false
+        chained_structs:
+            SetUserWriteBlockModeRequest: SetUserWriteBlockModeRequest
+        fields:
+            phase:
+                type: ShardsvrSetUserWriteBlockModePhase
+                description: "Determines the phase of the blocking/unblocking procedure to be
+                              executed."
+
     _configsvrSetClusterParameter:
         command_name: _configsvrSetClusterParameter
         cpp_name: ConfigsvrSetClusterParameter