SERVER-52760 Add shardsvr_reshard_collection command and a DDL coordinator to serialize behind other DDL operations

10 files changed, 308 insertions, 16 deletions

diff --git a/jstests/core/views/views_all_commands.js b/jstests/core/views/views_all_commands.js
index 93d1843d40d..93dc1803ffc 100644
--- a/jstests/core/views/views_all_commands.js
+++ b/jstests/core/views/views_all_commands.js
@@ -126,6 +126,7 @@ let viewsCommandTests = {
     _shardsvrRenameCollection: {skip: isAnInternalCommand},
     _shardsvrRenameCollectionParticipant: {skip: isAnInternalCommand},
     _shardsvrRenameCollectionUnblockParticipant: {skip: isAnInternalCommand},
+    _shardsvrReshardCollection: {skip: isAnInternalCommand},
     _shardsvrShardCollection: {skip: isAnInternalCommand},
     _transferMods: {skip: isAnInternalCommand},
     _vectorClockPersist: {skip: isAnInternalCommand},
diff --git a/jstests/replsets/db_reads_while_recovering_all_commands.js b/jstests/replsets/db_reads_while_recovering_all_commands.js
index 77b461c2829..78a557b4649 100644
--- a/jstests/replsets/db_reads_while_recovering_all_commands.js
+++ b/jstests/replsets/db_reads_while_recovering_all_commands.js
@@ -75,6 +75,7 @@ const allCommands = {
     _shardsvrDropDatabase: {skip: isPrimaryOnly},
     _shardsvrDropDatabaseParticipant: {skip: isPrimaryOnly},
     _shardsvrShardCollection: {skip: isPrimaryOnly},
+    _shardsvrReshardCollection: {skip: isPrimaryOnly},
     _shardsvrRefineCollectionShardKey: {skip: isPrimaryOnly},
     _transferMods: {skip: isPrimaryOnly},
     _vectorClockPersist: {skip: isPrimaryOnly},
diff --git a/jstests/sharding/libs/mongos_api_params_util.js b/jstests/sharding/libs/mongos_api_params_util.js
index 3741ebd82a7..e9742394569 100644
--- a/jstests/sharding/libs/mongos_api_params_util.js
+++ b/jstests/sharding/libs/mongos_api_params_util.js
@@ -946,7 +946,7 @@ let MongosAPIParametersUtil = (function() {
             run: {
                 inAPIVersion1: false,
                 permittedInTxn: false,
-                configServerCommandName: "_configsvrReshardCollection",
+                shardCommandName: "_shardsvrReshardCollection",
                 requiresShardedCollection: true,
                 // reshardCollection internally does atClusterTime reads.
                 requiresCommittedReads: true,
diff --git a/jstests/sharding/read_write_concern_defaults_application.js b/jstests/sharding/read_write_concern_defaults_application.js
index 5c1705ea960..78881616721 100644
--- a/jstests/sharding/read_write_concern_defaults_application.js
+++ b/jstests/sharding/read_write_concern_defaults_application.js
@@ -125,6 +125,7 @@ let testCases = {
     _shardsvrRenameCollection: {skip: "internal command"},
     _shardsvrRenameCollectionParticipant: {skip: "internal command"},
     _shardsvrRenameCollectionUnblockParticipant: {skip: "internal command"},
+    _shardsvrReshardCollection: {skip: "internal command"},
     _shardsvrShardCollection: {skip: "internal command"},
     _transferMods: {skip: "internal command"},
     _vectorClockPersist: {skip: "internal command"},
diff --git a/src/mongo/db/s/SConscript b/src/mongo/db/s/SConscript
index f1552d8deee..2c77e6e8155 100644
--- a/src/mongo/db/s/SConscript
+++ b/src/mongo/db/s/SConscript
@@ -345,6 +345,7 @@ env.Library(
         'move_primary_command.cpp',
         'rename_collection_coordinator.cpp',
         'refine_collection_shard_key_coordinator.cpp',
+        'reshard_collection_coordinator.cpp',
         'set_shard_version_command.cpp',
         'sharding_ddl_coordinator.cpp',
         'sharding_server_status.cpp',
@@ -358,6 +359,7 @@ env.Library(
         'shardsvr_refine_collection_shard_key_command.cpp',
         'shardsvr_rename_collection_command.cpp',
         'shardsvr_rename_collection_participant_command.cpp',
+        'shardsvr_reshard_collection_command.cpp',
         'shardsvr_shard_collection_command.cpp',
         'split_chunk_command.cpp',
         'split_vector_command.cpp',
diff --git a/src/mongo/db/s/reshard_collection_coordinator.cpp b/src/mongo/db/s/reshard_collection_coordinator.cpp
new file mode 100644
index 00000000000..452f17fdaf6
--- /dev/null
+++ b/src/mongo/db/s/reshard_collection_coordinator.cpp
@@ -0,0 +1,87 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/s/reshard_collection_coordinator.h"
+#include "mongo/logv2/log.h"
+#include "mongo/s/grid.h"
+#include "mongo/s/request_types/reshard_collection_gen.h"
+
+
+namespace mongo {
+
+ReshardCollectionCoordinator::ReshardCollectionCoordinator(
+    OperationContext* opCtx, const ShardsvrReshardCollection& reshardCollectionParams)
+    : ShardingDDLCoordinator_NORESILIENT(opCtx, reshardCollectionParams.getCommandParameter()),
+      _serviceContext(opCtx->getServiceContext()),
+      _request(reshardCollectionParams),
+      _nss(_request.getCommandParameter()) {}
+
+SemiFuture<void> ReshardCollectionCoordinator::runImpl(
+    std::shared_ptr<executor::TaskExecutor> executor) {
+    return ExecutorFuture<void>(executor, Status::OK())
+        .then([this, anchor = shared_from_this()]() {
+            ThreadClient tc("ReshardCollectionCoordinator", _serviceContext);
+            auto opCtxHolder = tc->makeOperationContext();
+            auto* opCtx = opCtxHolder.get();
+            _forwardableOpMetadata.setOn(opCtx);
+
+            ConfigsvrReshardCollection configsvrReshardCollection(_nss, _request.getKey());
+            configsvrReshardCollection.setDbName(_request.getDbName());
+            configsvrReshardCollection.setUnique(_request.getUnique());
+            configsvrReshardCollection.setCollation(_request.getCollation());
+            configsvrReshardCollection.set_presetReshardedChunks(
+                _request.get_presetReshardedChunks());
+            configsvrReshardCollection.setZones(_request.getZones());
+            configsvrReshardCollection.setNumInitialChunks(_request.getNumInitialChunks());
+
+            auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+            const auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts(
+                opCtx,
+                ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+                "admin",
+                CommandHelpers::appendMajorityWriteConcern(configsvrReshardCollection.toBSON({}),
+                                                           opCtx->getWriteConcern()),
+                Shard::RetryPolicy::kIdempotent));
+            uassertStatusOK(Shard::CommandResponse::getEffectiveStatus(std::move(cmdResponse)));
+        })
+        .onError([this, anchor = shared_from_this()](const Status& status) {
+            LOGV2_ERROR(5276001,
+                        "Error running reshard collection",
+                        "namespace"_attr = _nss,
+                        "error"_attr = redact(status));
+            return status;
+        })
+        .semi();
+}
+
+}  // namespace mongo
diff --git a/src/mongo/db/s/reshard_collection_coordinator.h b/src/mongo/db/s/reshard_collection_coordinator.h
new file mode 100644
index 00000000000..81b47ba5b34
--- /dev/null
+++ b/src/mongo/db/s/reshard_collection_coordinator.h
@@ -0,0 +1,51 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include "mongo/db/s/sharding_ddl_coordinator.h"
+#include "mongo/s/request_types/sharded_ddl_commands_gen.h"
+#include "mongo/util/future.h"
+
+namespace mongo {
+class ReshardCollectionCoordinator final
+    : public ShardingDDLCoordinator_NORESILIENT,
+      public std::enable_shared_from_this<ReshardCollectionCoordinator> {
+public:
+    ReshardCollectionCoordinator(OperationContext* opCtx,
+                                 const ShardsvrReshardCollection& reshardCollectionParams);
+
+private:
+    SemiFuture<void> runImpl(std::shared_ptr<executor::TaskExecutor> executor) override;
+
+    ServiceContext* _serviceContext;
+    const ShardsvrReshardCollection _request;
+    const NamespaceString& _nss;
+};
+}  // namespace mongo
\ No newline at end of file
diff --git a/src/mongo/db/s/shardsvr_reshard_collection_command.cpp b/src/mongo/db/s/shardsvr_reshard_collection_command.cpp
new file mode 100644
index 00000000000..fa6dbad0907
--- /dev/null
+++ b/src/mongo/db/s/shardsvr_reshard_collection_command.cpp
@@ -0,0 +1,105 @@
+/**
+ *    Copyright (C) 2021-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kCommand
+
+#include "mongo/platform/basic.h"
+
+#include "mongo/db/auth/authorization_session.h"
+#include "mongo/db/commands.h"
+#include "mongo/db/s/reshard_collection_coordinator.h"
+#include "mongo/db/s/sharding_state.h"
+#include "mongo/s/request_types/sharded_ddl_commands_gen.h"
+
+namespace mongo {
+namespace {
+
+class ShardsvrReshardCollectionCommand final
+    : public TypedCommand<ShardsvrReshardCollectionCommand> {
+public:
+    using Request = ShardsvrReshardCollection;
+
+    std::string help() const override {
+        return "Internal command. Do not call directly. Reshards a collection.";
+    }
+
+    bool adminOnly() const override {
+        return true;
+    }
+
+    AllowedOnSecondary secondaryAllowed(ServiceContext*) const override {
+        return AllowedOnSecondary::kNever;
+    }
+
+    class Invocation final : public InvocationBase {
+    public:
+        using InvocationBase::InvocationBase;
+
+        void typedRun(OperationContext* opCtx) {
+            uassertStatusOK(ShardingState::get(opCtx)->canAcceptShardedCommands());
+
+            uassert(ErrorCodes::InvalidOptions,
+                    str::stream() << Request::kCommandName
+                                  << " must be called with majority writeConcern, got "
+                                  << opCtx->getWriteConcern().wMode,
+                    opCtx->getWriteConcern().wMode == WriteConcernOptions::kMajority);
+            // (Generic FCV reference): To run this command and ensure the consistency of the
+            // metadata we need to make sure we are on a stable state.
+            uassert(
+                ErrorCodes::CommandNotSupported,
+                "Resharding is not supported for this version, please update the FCV to latest.",
+                !serverGlobalParams.featureCompatibility.isUpgradingOrDowngrading());
+
+            auto reshardCollectionCoordinator =
+                std::make_shared<ReshardCollectionCoordinator>(opCtx, request());
+            reshardCollectionCoordinator->run(opCtx).get(opCtx);
+        }
+
+    private:
+        NamespaceString ns() const override {
+            return request().getCommandParameter();
+        }
+
+        bool supportsWriteConcern() const override {
+            return true;
+        }
+
+        void doCheckAuthorization(OperationContext* opCtx) const override {
+            uassert(ErrorCodes::Unauthorized,
+                    "Unauthorized",
+                    AuthorizationSession::get(opCtx->getClient())
+                        ->isAuthorizedForActionsOnResource(ResourcePattern::forClusterResource(),
+                                                           ActionType::internal));
+        }
+    };
+
+} shardsvrReshardCollectionCommand;
+
+}  // namespace
+}  // namespace mongo
\ No newline at end of file
diff --git a/src/mongo/s/commands/cluster_reshard_collection_cmd.cpp b/src/mongo/s/commands/cluster_reshard_collection_cmd.cpp
index b3809bd4172..1e7535d0899 100644
--- a/src/mongo/s/commands/cluster_reshard_collection_cmd.cpp
+++ b/src/mongo/s/commands/cluster_reshard_collection_cmd.cpp
@@ -34,8 +34,12 @@
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/commands.h"
 #include "mongo/logv2/log.h"
+#include "mongo/rpc/get_status_from_command_result.h"
+#include "mongo/s/catalog_cache.h"
+#include "mongo/s/cluster_commands_helpers.h"
 #include "mongo/s/grid.h"
 #include "mongo/s/request_types/reshard_collection_gen.h"
+#include "mongo/s/request_types/sharded_ddl_commands_gen.h"
 
 namespace mongo {
 namespace {
@@ -49,26 +53,30 @@ public:
         using InvocationBase::InvocationBase;
 
         void typedRun(OperationContext* opCtx) {
-            const NamespaceString& nss = ns();
-            ConfigsvrReshardCollection configsvrReshardCollection(nss, request().getKey());
-            configsvrReshardCollection.setDbName(request().getDbName());
-            configsvrReshardCollection.setUnique(request().getUnique());
-            configsvrReshardCollection.setCollation(request().getCollation());
-            configsvrReshardCollection.set_presetReshardedChunks(
+            const auto& nss = ns();
+            ShardsvrReshardCollection shardsvrReshardCollection(nss, request().getKey());
+            shardsvrReshardCollection.setDbName(request().getDbName());
+            shardsvrReshardCollection.setUnique(request().getUnique());
+            shardsvrReshardCollection.setCollation(request().getCollation());
+            shardsvrReshardCollection.set_presetReshardedChunks(
                 request().get_presetReshardedChunks());
-            configsvrReshardCollection.setZones(request().getZones());
-            configsvrReshardCollection.setNumInitialChunks(request().getNumInitialChunks());
+            shardsvrReshardCollection.setZones(request().getZones());
+            shardsvrReshardCollection.setNumInitialChunks(request().getNumInitialChunks());
 
-            auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
-            auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts(
+            auto catalogCache = Grid::get(opCtx)->catalogCache();
+            const auto dbInfo = uassertStatusOK(catalogCache->getDatabase(opCtx, nss.db()));
+
+            auto cmdResponse = executeCommandAgainstDatabasePrimary(
                 opCtx,
-                ReadPreferenceSetting(ReadPreference::PrimaryOnly),
                 "admin",
-                CommandHelpers::appendMajorityWriteConcern(configsvrReshardCollection.toBSON({}),
+                dbInfo,
+                CommandHelpers::appendMajorityWriteConcern(shardsvrReshardCollection.toBSON({}),
                                                            opCtx->getWriteConcern()),
-                Shard::RetryPolicy::kIdempotent));
-            uassertStatusOK(cmdResponse.commandStatus);
-            uassertStatusOK(cmdResponse.writeConcernStatus);
+                ReadPreferenceSetting(ReadPreference::PrimaryOnly),
+                Shard::RetryPolicy::kIdempotent);
+
+            const auto remoteResponse = uassertStatusOK(cmdResponse.swResponse);
+            uassertStatusOK(getStatusFromCommandResult(remoteResponse.data));
         }
 
     private:
diff --git a/src/mongo/s/request_types/sharded_ddl_commands.idl b/src/mongo/s/request_types/sharded_ddl_commands.idl
index 005c713107d..faab01d847d 100644
--- a/src/mongo/s/request_types/sharded_ddl_commands.idl
+++ b/src/mongo/s/request_types/sharded_ddl_commands.idl
@@ -206,6 +206,42 @@ commands:
                 type: namespacestring
                 description: "The new namespace for the collection being renamed."
 
+    _shardsvrReshardCollection:
+        command_name: _shardsvrReshardCollection
+        cpp_name: ShardsvrReshardCollection
+        description: "The internal reshardCollection command on the shard."
+        strict: false
+        namespace: type
+        api_version: ""
+        type: namespacestring
+        fields:
+            key:
+                type: object
+                description: "The index specification document to use as the new shard key."
+                optional: false
+            unique:
+                type: bool
+                description: "Whether the shard key index should enforce a unique constraint."
+                optional: true
+            numInitialChunks:
+                type: safeInt64
+                description: "The number of chunks to create initially."
+                optional: true
+            collation:
+                type: object
+                description: "The collation to use for the shard key index."
+                optional: true
+            zones:
+                type: array<object>
+                description: "The zones associated with the new shard key."
+                optional: true
+            _presetReshardedChunks:
+                type: array<object>
+                description: >-
+                    "Mapping of chunk ranges to be used as the initial split output. This is only for 
+                    testing purposes."
+                optional: true
+
     _shardsvrRefineCollectionShardKey:
         description: "Parser for the _shardsvrRefineCollectionShardKey command"
         command_name: _shardsvrRefineCollectionShardKey