SERVER-62332 Make RefineCollectionShardKeyCoordinator disallow migrations while it's executing

13 files changed, 235 insertions, 87 deletions

diff --git a/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js b/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js
index 7816c30dd50..a1cb9fe517e 100644
--- a/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js
+++ b/jstests/sharding/range_deleter_interacts_correctly_with_refine_shard_key.js
@@ -8,6 +8,7 @@
 load("jstests/libs/fail_point_util.js");
 load('jstests/libs/parallel_shell_helpers.js');
 load('jstests/replsets/rslib.js');
+load('jstests/libs/feature_flag_util.js');
 
 TestData.skipCheckingUUIDsConsistentAcrossCluster = true;
 
@@ -187,6 +188,16 @@ function test(st, description, testBody) {
         "Migration recovery recovers correct decision for migration committed before shard key " +
             "refine",
         () => {
+            if (FeatureFlagUtil.isEnabled(st.configRS.getPrimary().getDB('admin'),
+                                          "RecoverableRefineCollectionShardKeyCoordinator")) {
+                // Skip because when RecoverableRefineCollectionShardKeyCoordinator is enabled,
+                // migrations will be stopped (and thus any migration pending recovery will be
+                // recovered) before committing the refineCollectionShardKey metadata changes, which
+                // makes this test case not possible anymore.
+                jsTestLog("Skip");
+                return;
+            }
+
             // This test must move the first chunk for refine shard key to work while migration
             // recovery is blocked. Insert some documents into the first chunk.
             for (let i = -100; i < -1; i++) {
diff --git a/jstests/sharding/refine_collection_shard_key_atomic.js b/jstests/sharding/refine_collection_shard_key_atomic.js
index 2b099cf9386..75781576c50 100644
--- a/jstests/sharding/refine_collection_shard_key_atomic.js
+++ b/jstests/sharding/refine_collection_shard_key_atomic.js
@@ -87,13 +87,30 @@ let awaitShellToRefineCollectionShardKey = startParallelShell(() => {
 }, mongos.port);
 hangBeforeCommitFailPoint.wait();
 
-// Verify that 'config.collections' has not been updated since we haven't committed the transaction.
+// Verify that 'config.collections' has not been updated since we haven't committed the transaction,
+// except for the 'allowMigrations' property which is updated by the
+// RefineCollectionShardKeyCoordinator before the commit phase.
 let newCollArr = mongos.getCollection(kConfigCollections).find({_id: kNsName}).toArray();
+newCollArr.forEach(element => {
+    delete element['allowMigrations'];
+});
 assert.sameMembers(oldCollArr, newCollArr);
 
-// Verify that 'config.chunks' has not been updated since we haven't committed the transaction.
+// Verify that 'config.chunks' has not been updated since we haven't committed the transaction,
+// except for the chunk version which has been bumped by the setAllowMigrations command prior to the
+// refineCollectionShardKey commit.
 let newChunkArr =
     findChunksUtil.findChunksByNs(mongos.getDB('config'), kNsName).sort({min: 1}).toArray();
+
+newChunkArr.forEach(element => {
+    delete element['lastmod'];
+});
+
+let oldChunkArrWithoutLastmod = oldChunkArr;
+oldChunkArrWithoutLastmod.forEach(element => {
+    delete element['lastmod'];
+});
+
 assert.sameMembers(oldChunkArr, newChunkArr);
 
 // Verify that 'config.tags' has not been updated since we haven't committed the transaction.
diff --git a/jstests/sharding/refine_collection_shard_key_basic.js b/jstests/sharding/refine_collection_shard_key_basic.js
index 79328fc0280..46d6e61f0cd 100644
--- a/jstests/sharding/refine_collection_shard_key_basic.js
+++ b/jstests/sharding/refine_collection_shard_key_basic.js
@@ -361,7 +361,7 @@ assert.commandWorked(mongos.getCollection(kNsName).createIndex(
 
 assert.commandFailedWithCode(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
-    ErrorCodes.OperationFailed);
+    ErrorCodes.InvalidOptions);
 
 // Should fail because only a multikey index exists for new shard key {_id: 1, aKey: 1}.
 dropAndReshardColl({_id: 1});
@@ -370,7 +370,7 @@ assert.commandWorked(mongos.getCollection(kNsName).insert({aKey: [1, 2, 3, 4, 5]
 
 assert.commandFailedWithCode(
     mongos.adminCommand({refineCollectionShardKey: kNsName, key: {_id: 1, aKey: 1}}),
-    ErrorCodes.OperationFailed);
+    ErrorCodes.InvalidOptions);
 
 // Should fail because current shard key {a: 1} is unique, new shard key is {a: 1, b: 1}, and an
 // index only exists on {a: 1, b: 1, c: 1}.
@@ -711,38 +711,6 @@ if (!isStepdownSuite) {
     assert(minKeyShardDB.system.profile.drop());
 })();
 
-(() => {
-    //
-    // Verify refineCollectionShardKey can return a StaleConfig error without crashing the config
-    // server.
-    //
-
-    // Create a sharded collection with one chunk on shard0.
-    const dbName = "testReturnStaleConfig";
-    const ns = dbName + ".fooReturnStaleConfig";
-    assert.commandWorked(st.s.adminCommand({enableSharding: dbName}));
-    st.ensurePrimaryShard(dbName, st.shard0.shardName);
-    assert.commandWorked(st.s.adminCommand({shardCollection: ns, key: {x: 1}}));
-
-    // Move the last chunk away from shard0 without refreshing shard1 so it will be stale when its
-    // indexes are read during the next refine. Disable refreshes on shard1 so it will repeatedly
-    // return StaleConfig until refineCollectionShardKey runs out of retries.
-    ShardVersioningUtil.moveChunkNotRefreshRecipient(st.s, ns, st.shard0, st.shard1, {x: 1});
-
-    let disableRefreshesFailPoint =
-        configureFailPoint(st.rs1.getPrimary(), "skipShardFilteringMetadataRefresh");
-
-    assert.commandWorked(st.rs1.getPrimary().getCollection(ns).createIndex({x: 1, y: 1}));
-    assert.commandFailedWithCode(
-        st.s.adminCommand({refineCollectionShardKey: ns, key: {x: 1, y: 1}}),
-        ErrorCodes.StaleConfig);
-
-    disableRefreshesFailPoint.off();
-
-    // The refresh should succeed now.
-    assert.commandWorked(st.s.adminCommand({refineCollectionShardKey: ns, key: {x: 1, y: 1}}));
-})();
-
 // Assumes the given arrays are sorted by the max field.
 function compareMinAndMaxFields(shardedArr, refinedArr) {
     assert(shardedArr.length && refinedArr.length, tojson(shardedArr) + ", " + tojson(refinedArr));
diff --git a/jstests/sharding/refine_collection_shard_key_drop_chunks.js b/jstests/sharding/refine_collection_shard_key_drop_chunks.js
index eb29d9594c9..548f6921988 100644
--- a/jstests/sharding/refine_collection_shard_key_drop_chunks.js
+++ b/jstests/sharding/refine_collection_shard_key_drop_chunks.js
@@ -6,7 +6,6 @@
 (function() {
 'use strict';
 
-load('jstests/libs/fail_point_util.js');
 load('jstests/sharding/libs/catalog_cache_loader_helpers.js');
 
 const st = new ShardingTest({shards: 1});
@@ -53,27 +52,17 @@ assert.eq({a: 5, b: 5}, chunkArr[1].max);
 assert.eq({a: 5, b: 5}, chunkArr[2]._id);
 assert.eq({a: MaxKey, b: MaxKey}, chunkArr[2].max);
 
-// Enable failpoint 'hangPersistCollectionAndChangedChunksAfterDropChunks' and flush the routing
-// table cache.
-let hangAfterDropChunksFailPoint =
-    configureFailPoint(shard, 'hangPersistCollectionAndChangedChunksAfterDropChunks');
-
 assert.commandWorked(mongos.adminCommand({refineCollectionShardKey: kNsName, key: newKeyDoc}));
 
-// Verify that all chunks belonging to 'db.foo' have been deleted.
-hangAfterDropChunksFailPoint.wait();
-chunkArr = shard.getCollection(configCacheChunks).find({}).sort({min: 1}).toArray();
-assert.eq(0, chunkArr.length);
-
-// Disable failpoint 'hangPersistCollectionAndChangedChunksAfterDropChunks' and continue
-// flushing the routing table cache.
-hangAfterDropChunksFailPoint.off();
-
 // Verify that 'config.cache.chunks.db.foo' is as expected after refineCollectionShardKey. NOTE: We
 // use assert.soon here because refineCollectionShardKey doesn't block for each shard to refresh.
 assert.soon(() => {
+    let collectionCacheArr =
+        shard.getCollection('config.cache.collections').find({_id: kNsName}).toArray();
+
     chunkArr = shard.getCollection(configCacheChunks).find({}).sort({min: 1}).toArray();
-    return (3 === chunkArr.length);
+    return collectionCacheArr.length === 1 && collectionCacheArr[0].epoch != collEntry.epoch &&
+        3 === chunkArr.length;
 });
 assert.eq({a: MinKey, b: MinKey, c: MinKey, d: MinKey}, chunkArr[0]._id);
 assert.eq({a: 0, b: 0, c: MinKey, d: MinKey}, chunkArr[0].max);
diff --git a/src/mongo/db/s/refine_collection_shard_key_coordinator.cpp b/src/mongo/db/s/refine_collection_shard_key_coordinator.cpp
index 8f741085cd1..e4f7b101cdd 100644
--- a/src/mongo/db/s/refine_collection_shard_key_coordinator.cpp
+++ b/src/mongo/db/s/refine_collection_shard_key_coordinator.cpp
@@ -33,6 +33,7 @@
 
 #include "mongo/db/commands.h"
 #include "mongo/db/s/dist_lock_manager.h"
+#include "mongo/db/s/sharding_ddl_util.h"
 #include "mongo/logv2/log.h"
 #include "mongo/s/catalog_cache.h"
 #include "mongo/s/grid.h"
@@ -42,11 +43,18 @@ namespace mongo {
 
 RefineCollectionShardKeyCoordinator::RefineCollectionShardKeyCoordinator(
     ShardingDDLCoordinatorService* service, const BSONObj& initialState)
+    : RefineCollectionShardKeyCoordinator(
+          service, initialState, true /* persistCoordinatorDocument */) {}
+
+RefineCollectionShardKeyCoordinator::RefineCollectionShardKeyCoordinator(
+    ShardingDDLCoordinatorService* service,
+    const BSONObj& initialState,
+    bool persistCoordinatorDocument)
     : ShardingDDLCoordinator(service, initialState),
       _doc(RefineCollectionShardKeyCoordinatorDocument::parse(
           IDLParserErrorContext("RefineCollectionShardKeyCoordinatorDocument"), initialState)),
-      _newShardKey(_doc.getNewShardKey()) {}
-
+      _newShardKey(_doc.getNewShardKey()),
+      _persistCoordinatorDocument(persistCoordinatorDocument) {}
 
 void RefineCollectionShardKeyCoordinator::checkIfOptionsConflict(const BSONObj& doc) const {
     // If we have two refine collections on the same namespace, then the arguments must be the same.
@@ -80,25 +88,53 @@ boost::optional<BSONObj> RefineCollectionShardKeyCoordinator::reportForCurrentOp
     return bob.obj();
 }
 
+void RefineCollectionShardKeyCoordinator::_enterPhase(Phase newPhase) {
+    if (!_persistCoordinatorDocument) {
+        return;
+    }
+
+    StateDoc newDoc(_doc);
+    newDoc.setPhase(newPhase);
+
+    LOGV2_DEBUG(
+        6233200,
+        2,
+        "Refine collection shard key coordinator phase transition",
+        "namespace"_attr = nss(),
+        "newPhase"_attr = RefineCollectionShardKeyCoordinatorPhase_serializer(newDoc.getPhase()),
+        "oldPhase"_attr = RefineCollectionShardKeyCoordinatorPhase_serializer(_doc.getPhase()));
+
+    if (_doc.getPhase() == Phase::kUnset) {
+        _doc = _insertStateDocument(std::move(newDoc));
+        return;
+    }
+    _doc = _updateStateDocument(cc().makeOperationContext().get(), std::move(newDoc));
+}
+
 ExecutorFuture<void> RefineCollectionShardKeyCoordinator::_runImpl(
     std::shared_ptr<executor::ScopedTaskExecutor> executor,
     const CancellationToken& token) noexcept {
     return ExecutorFuture<void>(**executor)
-        .then([this, anchor = shared_from_this()] {
-            auto opCtxHolder = cc().makeOperationContext();
-            auto* opCtx = opCtxHolder.get();
-            getForwardableOpMetadata().setOn(opCtx);
+        .then(_executePhase(
+            Phase::kRefineCollectionShardKey,
+            [this, anchor = shared_from_this()] {
+                auto opCtxHolder = cc().makeOperationContext();
+                auto* opCtx = opCtxHolder.get();
+                getForwardableOpMetadata().setOn(opCtx);
 
-            const auto cm = uassertStatusOK(
-                Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(
-                    opCtx, nss()));
-            ConfigsvrRefineCollectionShardKey configsvrRefineCollShardKey(
-                nss(), _newShardKey.toBSON(), cm.getVersion().epoch());
-            configsvrRefineCollShardKey.setDbName(nss().db().toString());
-            auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
+                const auto cm = uassertStatusOK(
+                    Grid::get(opCtx)->catalogCache()->getShardedCollectionRoutingInfoWithRefresh(
+                        opCtx, nss()));
+                ConfigsvrRefineCollectionShardKey configsvrRefineCollShardKey(
+                    nss(), _newShardKey.toBSON(), cm.getVersion().epoch());
+                configsvrRefineCollShardKey.setDbName(nss().db().toString());
+                auto configShard = Grid::get(opCtx)->shardRegistry()->getConfigShard();
 
-            try {
-                auto cmdResponse = uassertStatusOK(configShard->runCommandWithFixedRetryAttempts(
+                if (_persistCoordinatorDocument) {
+                    sharding_ddl_util::stopMigrations(opCtx, nss(), boost::none);
+                }
+
+                const auto cmdResponse = uassertStatusOK(configShard->runCommand(
                     opCtx,
                     ReadPreferenceSetting(ReadPreference::PrimaryOnly),
                     NamespaceString::kAdminDb.toString(),
@@ -106,20 +142,39 @@ ExecutorFuture<void> RefineCollectionShardKeyCoordinator::_runImpl(
                         configsvrRefineCollShardKey.toBSON({}), opCtx->getWriteConcern()),
                     Shard::RetryPolicy::kIdempotent));
 
-                uassertStatusOK(cmdResponse.commandStatus);
-                uassertStatusOK(cmdResponse.writeConcernStatus);
-            } catch (const DBException&) {
-                _completeOnError = true;
-                throw;
-            }
-        })
+                try {
+                    uassertStatusOK(
+                        Shard::CommandResponse::getEffectiveStatus(std::move(cmdResponse)));
+                } catch (const DBException&) {
+                    if (!_persistCoordinatorDocument) {
+                        _completeOnError = true;
+                    }
+                    throw;
+                }
+            }))
         .onError([this, anchor = shared_from_this()](const Status& status) {
             LOGV2_ERROR(5277700,
                         "Error running refine collection shard key",
                         "namespace"_attr = nss(),
                         "error"_attr = redact(status));
+
+            return status;
+        })
+        .onCompletion([this, anchor = shared_from_this()](const Status& status) {
+            auto opCtxHolder = cc().makeOperationContext();
+            auto* opCtx = opCtxHolder.get();
+            getForwardableOpMetadata().setOn(opCtx);
+            if (_persistCoordinatorDocument) {
+                sharding_ddl_util::resumeMigrations(opCtx, nss(), boost::none);
+            }
+
             return status;
         });
 }
 
+RefineCollectionShardKeyCoordinator_NORESILIENT::RefineCollectionShardKeyCoordinator_NORESILIENT(
+    ShardingDDLCoordinatorService* service, const BSONObj& initialState)
+    : RefineCollectionShardKeyCoordinator(
+          service, initialState, false /* persistCoordinatorDocument */) {}
+
 }  // namespace mongo
diff --git a/src/mongo/db/s/refine_collection_shard_key_coordinator.h b/src/mongo/db/s/refine_collection_shard_key_coordinator.h
index 1add4bee9a0..58155b5eed5 100644
--- a/src/mongo/db/s/refine_collection_shard_key_coordinator.h
+++ b/src/mongo/db/s/refine_collection_shard_key_coordinator.h
@@ -35,8 +35,11 @@
 
 namespace mongo {
 
-class RefineCollectionShardKeyCoordinator final : public ShardingDDLCoordinator {
+class RefineCollectionShardKeyCoordinator : public ShardingDDLCoordinator {
 public:
+    using StateDoc = RefineCollectionShardKeyCoordinatorDocument;
+    using Phase = RefineCollectionShardKeyCoordinatorPhaseEnum;
+
     RefineCollectionShardKeyCoordinator(ShardingDDLCoordinatorService* service,
                                         const BSONObj& initialState);
 
@@ -46,6 +49,11 @@ public:
         MongoProcessInterface::CurrentOpConnectionsMode connMode,
         MongoProcessInterface::CurrentOpSessionsMode sessionMode) noexcept override;
 
+protected:
+    RefineCollectionShardKeyCoordinator(ShardingDDLCoordinatorService* service,
+                                        const BSONObj& initialState,
+                                        bool persistCoordinatorDocument);
+
 private:
     ShardingDDLCoordinatorMetadata const& metadata() const override {
         return _doc.getShardingDDLCoordinatorMetadata();
@@ -54,8 +62,35 @@ private:
     ExecutorFuture<void> _runImpl(std::shared_ptr<executor::ScopedTaskExecutor> executor,
                                   const CancellationToken& token) noexcept override;
 
+    template <typename Func>
+    auto _executePhase(const Phase& newPhase, Func&& func) {
+        return [=] {
+            const auto& currPhase = _doc.getPhase();
+
+            if (currPhase > newPhase) {
+                // Do not execute this phase if we already reached a subsequent one.
+                return;
+            }
+            if (currPhase < newPhase) {
+                // Persist the new phase if this is the first time we are executing it.
+                _enterPhase(newPhase);
+            }
+            return func();
+        };
+    }
+
+    void _enterPhase(Phase newPhase);
+
     RefineCollectionShardKeyCoordinatorDocument _doc;
     const KeyPattern _newShardKey;
+    const bool _persistCoordinatorDocument;  // TODO: SERVER-62850 remove this then 6.0 branches out
+};
+
+// TODO: SERVER-62850 remove this then 6.0 branches out
+class RefineCollectionShardKeyCoordinator_NORESILIENT : public RefineCollectionShardKeyCoordinator {
+public:
+    RefineCollectionShardKeyCoordinator_NORESILIENT(ShardingDDLCoordinatorService* service,
+                                                    const BSONObj& initialState);
 };
 
 }  // namespace mongo
diff --git a/src/mongo/db/s/refine_collection_shard_key_coordinator_document.idl b/src/mongo/db/s/refine_collection_shard_key_coordinator_document.idl
index 4fa8173b6b2..b7bc9364dff 100644
--- a/src/mongo/db/s/refine_collection_shard_key_coordinator_document.idl
+++ b/src/mongo/db/s/refine_collection_shard_key_coordinator_document.idl
@@ -37,6 +37,14 @@ imports:
     - "mongo/db/s/sharding_ddl_coordinator.idl"
     - "mongo/s/request_types/sharded_ddl_commands.idl"
 
+enums:
+    RefineCollectionShardKeyCoordinatorPhase:
+        description: "Current refine collection shard key coordinator's operation state."
+        type: string
+        values:
+            kUnset: "unset"
+            kRefineCollectionShardKey: "RefineCollectionShardKey"
+
 structs:
     RefineCollectionShardKeyCoordinatorDocument:
         description: "Object with neccessary fields to refine a collection's shard key"
@@ -45,3 +53,8 @@ structs:
         chained_structs:
             ShardingDDLCoordinatorMetadata: ShardingDDLCoordinatorMetadata
             RefineCollectionShardKeyRequest: RefineCollectionShardKeyRequest
+        fields:
+            phase:
+                type: RefineCollectionShardKeyCoordinatorPhase
+                description: "Coordinator phase."
+                default: kUnset
diff --git a/src/mongo/db/s/shard_key_util.cpp b/src/mongo/db/s/shard_key_util.cpp
index 6f12df7d24d..e5719861559 100644
--- a/src/mongo/db/s/shard_key_util.cpp
+++ b/src/mongo/db/s/shard_key_util.cpp
@@ -278,8 +278,8 @@ void ValidationBehaviorsRefineShardKey::verifyUsefulNonMultiKeyIndex(
         Shard::RetryPolicy::kIdempotent));
     if (checkShardingIndexRes.commandStatus == ErrorCodes::UnknownError) {
         // CheckShardingIndex returns UnknownError if a compatible shard key index cannot be found,
-        // but we return OperationFailed to correspond with the shardCollection behavior.
-        uasserted(ErrorCodes::OperationFailed, checkShardingIndexRes.response["errmsg"].str());
+        // but we return InvalidOptions to correspond with the shardCollection behavior.
+        uasserted(ErrorCodes::InvalidOptions, checkShardingIndexRes.response["errmsg"].str());
     }
     // Rethrow any other error to allow retries on retryable errors.
     uassertStatusOK(checkShardingIndexRes.commandStatus);
diff --git a/src/mongo/db/s/sharding_ddl_coordinator.idl b/src/mongo/db/s/sharding_ddl_coordinator.idl
index 1a3e5610312..3b2f6b7b45d 100644
--- a/src/mongo/db/s/sharding_ddl_coordinator.idl
+++ b/src/mongo/db/s/sharding_ddl_coordinator.idl
@@ -49,6 +49,7 @@ enums:
             kRenameCollection: "renameCollection"
             kCreateCollection: "createCollection"
             kRefineCollectionShardKey: "refineCollectionShardKey"
+            kRefineCollectionShardKeyNoResilient: "refineCollectionShardKeyNoResilient"
             kSetAllowMigrations: "setAllowMigrations"
             kCollMod: "collMod"
             kReshardCollection: "reshardCollection"
diff --git a/src/mongo/db/s/sharding_ddl_coordinator_service.cpp b/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
index 7b7722eeb7a..9ef734b960f 100644
--- a/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
+++ b/src/mongo/db/s/sharding_ddl_coordinator_service.cpp
@@ -79,6 +79,10 @@ std::shared_ptr<ShardingDDLCoordinator> constructShardingDDLCoordinatorInstance(
             return std::make_shared<RefineCollectionShardKeyCoordinator>(service,
                                                                          std::move(initialState));
             break;
+        case DDLCoordinatorTypeEnum::kRefineCollectionShardKeyNoResilient:
+            return std::make_shared<RefineCollectionShardKeyCoordinator_NORESILIENT>(
+                service, std::move(initialState));
+            break;
         case DDLCoordinatorTypeEnum::kSetAllowMigrations:
             return std::make_shared<SetAllowMigrationsCoordinator>(service,
                                                                    std::move(initialState));
diff --git a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
index fdd0c12131c..49e3006d250 100644
--- a/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
+++ b/src/mongo/db/s/shardsvr_refine_collection_shard_key_command.cpp
@@ -31,7 +31,9 @@
 
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/commands.h"
+#include "mongo/db/commands/feature_compatibility_version.h"
 #include "mongo/db/s/refine_collection_shard_key_coordinator.h"
+#include "mongo/s/refine_collection_shard_key_coordinator_feature_flags_gen.h"
 #include "mongo/s/request_types/refine_collection_shard_key_gen.h"
 #include "mongo/s/request_types/sharded_ddl_commands_gen.h"
 
@@ -62,16 +64,26 @@ public:
         using InvocationBase::InvocationBase;
 
         void typedRun(OperationContext* opCtx) {
-            auto coordinatorDoc = RefineCollectionShardKeyCoordinatorDocument();
-            coordinatorDoc.setShardingDDLCoordinatorMetadata(
-                {{ns(), DDLCoordinatorTypeEnum::kRefineCollectionShardKey}});
-            coordinatorDoc.setRefineCollectionShardKeyRequest(
-                request().getRefineCollectionShardKeyRequest());
-
-            auto service = ShardingDDLCoordinatorService::getService(opCtx);
-            auto refineCoordinator = checked_pointer_cast<RefineCollectionShardKeyCoordinator>(
-                service->getOrCreateInstance(opCtx, coordinatorDoc.toBSON()));
-            refineCoordinator->getCompletionFuture().get(opCtx);
+            const auto coordinatorCompletionFuture = [&]() -> SharedSemiFuture<void> {
+                FixedFCVRegion fixedFcvRegion(opCtx);
+                const auto coordinatorType =
+                    feature_flags::gFeatureFlagRecoverableRefineCollectionShardKeyCoordinator
+                        .isEnabled(serverGlobalParams.featureCompatibility)
+                    ? DDLCoordinatorTypeEnum::kRefineCollectionShardKey
+                    : DDLCoordinatorTypeEnum::kRefineCollectionShardKeyNoResilient;
+
+                auto coordinatorDoc = RefineCollectionShardKeyCoordinatorDocument();
+                coordinatorDoc.setShardingDDLCoordinatorMetadata({{ns(), coordinatorType}});
+                coordinatorDoc.setRefineCollectionShardKeyRequest(
+                    request().getRefineCollectionShardKeyRequest());
+
+                auto service = ShardingDDLCoordinatorService::getService(opCtx);
+                auto refineCoordinator = checked_pointer_cast<RefineCollectionShardKeyCoordinator>(
+                    service->getOrCreateInstance(opCtx, coordinatorDoc.toBSON()));
+                return refineCoordinator->getCompletionFuture();
+            }();
+
+            coordinatorCompletionFuture.get(opCtx);
         }
 
         bool supportsWriteConcern() const override {
diff --git a/src/mongo/s/SConscript b/src/mongo/s/SConscript
index 52cb3941f0e..7da804def96 100644
--- a/src/mongo/s/SConscript
+++ b/src/mongo/s/SConscript
@@ -179,6 +179,7 @@ env.Library(
         'long_collection_names.idl',
         'mongod_and_mongos_server_parameters.idl',
         'pm2423_feature_flags.idl',
+        'refine_collection_shard_key_coordinator_feature_flags.idl',
         'request_types/abort_reshard_collection.idl',
         'request_types/add_shard_request_type.cpp',
         'request_types/add_shard_to_zone_request_type.cpp',
diff --git a/src/mongo/s/refine_collection_shard_key_coordinator_feature_flags.idl b/src/mongo/s/refine_collection_shard_key_coordinator_feature_flags.idl
new file mode 100644
index 00000000000..982ab41e9d5
--- /dev/null
+++ b/src/mongo/s/refine_collection_shard_key_coordinator_feature_flags.idl
@@ -0,0 +1,42 @@
+# Copyright (C) 2022-present MongoDB, Inc.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the Server Side Public License, version 1,
+# as published by MongoDB, Inc.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# Server Side Public License for more details.
+#
+# You should have received a copy of the Server Side Public License
+# along with this program. If not, see
+# <http://www.mongodb.com/licensing/server-side-public-license>.
+#
+# As a special exception, the copyright holders give permission to link the
+# code of portions of this program with the OpenSSL library under certain
+# conditions as described in each individual source file and distribute
+# linked combinations including the program with the OpenSSL library. You
+# must comply with the Server Side Public License in all respects for
+# all of the code used other than as permitted herein. If you modify file(s)
+# with this exception, you may extend this exception to your version of the
+# file(s), but you are not obligated to do so. If you do not wish to do so,
+# delete this exception statement from your version. If you delete this
+# exception statement from all source files in the program, then also delete
+# it in the license file.
+#
+
+# Feature flag related to RefineCollectionShardKeyCoordinator
+
+global:
+    cpp_namespace: "mongo::feature_flags"
+
+imports:
+    - "mongo/idl/basic_types.idl"
+
+feature_flags:
+    featureFlagRecoverableRefineCollectionShardKeyCoordinator:
+        description: Feature flag for enabling the recoverable RefineCollectionShardKeyCoordinator which
+            disallows migrations during its execution.
+        cpp_varname: gFeatureFlagRecoverableRefineCollectionShardKeyCoordinator
+        default: false