SERVER-58669 Allow collMod, createIndexes, and dropIndexes once resharding coordinator has made a decision

(cherry picked from commit a1e57d2985b672c898431b7787e53c547bbd2753)

SERVER-58917 Wait until donors/recipients are aware that coordinator has persisted decision before expecting successful collMod, createIndexes, and dropIndexes in resharding_prohibited_commands.js

(cherry picked from commit 8179692b362d8237f201719946eb46bac6a5e961)

3 files changed, 123 insertions, 8 deletions

diff --git a/jstests/sharding/resharding_prohibited_commands.js b/jstests/sharding/resharding_prohibited_commands.js
index 9167db7befb..d512b859ed3 100644
--- a/jstests/sharding/resharding_prohibited_commands.js
+++ b/jstests/sharding/resharding_prohibited_commands.js
@@ -13,26 +13,96 @@
 
 load("jstests/libs/discover_topology.js");
 load("jstests/sharding/libs/resharding_test_fixture.js");
+load("jstests/libs/fail_point_util.js");
+load('jstests/libs/parallel_shell_helpers.js');
 
 const reshardingTest = new ReshardingTest({numDonors: 2});
 reshardingTest.setup();
 
 const donorShardNames = reshardingTest.donorShardNames;
-const sourceCollection = reshardingTest.createShardedCollection({
+let sourceCollection = reshardingTest.createShardedCollection({
     ns: "reshardingDb.coll",
     shardKeyPattern: {oldKey: 1},
-    chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}],
+    chunks: [
+        {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]},
+        {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]}
+    ],
 });
 
 const recipientShardNames = reshardingTest.recipientShardNames;
+const mongos = sourceCollection.getMongo();
+let ns = sourceCollection.getFullName();
+
+const topology = DiscoverTopology.findConnectedNodes(mongos);
+const configPrimary = new Mongo(topology.configsvr.primary);
+const donor0 = new Mongo(topology.shards[donorShardNames[0]].primary);
+const donor1 = new Mongo(topology.shards[donorShardNames[1]].primary);
+
+let pauseCoordinatorBeforeRemovingStateDoc;
+let pauseBeforeRemoveDonor0Doc;
+let pauseBeforeRemoveDonor1Doc;
+pauseCoordinatorBeforeRemovingStateDoc =
+    configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeRemovingStateDoc");
+
+pauseBeforeRemoveDonor0Doc = configureFailPoint(donor0, "removeDonorDocFailpoint");
+pauseBeforeRemoveDonor1Doc = configureFailPoint(donor1, "removeDonorDocFailpoint");
+
+let awaitAbort;
+reshardingTest.withReshardingInBackground(
+    {
+        newShardKeyPattern: {newKey: 1},
+        newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}],
+    },
+    (tempNs) => {
+        const topology = DiscoverTopology.findConnectedNodes(mongos);
+        assert.soon(() => {
+            let res =
+                donor0.getCollection("config.localReshardingOperations.donor").find().toArray();
+            return res.length == 1;
+        }, "timed out waiting for resharding initialization on donor shard");
+
+        awaitAbort = startParallelShell(
+            funWithArgs(function(ns) {
+                assert.commandWorked(db.adminCommand({abortReshardCollection: ns}));
+            }, ns), mongos.port);
+    },
+    {
+        expectedErrorCode: ErrorCodes.ReshardCollectionAborted,
+        postDecisionPersistedFn: () => {
+            // Once the coordinator has persisted an abort decision, collMod, createIndexes, and
+            // dropIndexes should be able to succeed. Wait until both donors have heard that the
+            // coordinator has made the decision.
+            assert.soon(() => {
+                const res0 = donor0.getCollection("config.cache.collections").findOne({_id: ns});
+                const res1 = donor1.getCollection("config.cache.collections").findOne({_id: ns});
+                return res0.reshardingFields.state === "aborting" &&
+                    res1.reshardingFields.state === "aborting";
+            }, () => `timed out waiting for the coordinator to persist decision`);
+
+            const db = mongos.getDB("reshardingDb");
+            assert.commandWorked(db.runCommand({collMod: 'coll'}));
+            assert.commandWorked(db.runCommand(
+                {createIndexes: 'coll', indexes: [{name: "idx1", key: {newKey: 1, oldKey: 1}}]}));
+            assert.commandWorked(
+                db.runCommand({dropIndexes: 'coll', index: {newKey: 1, oldKey: 1}}));
+
+            pauseBeforeRemoveDonor0Doc.off();
+            pauseBeforeRemoveDonor1Doc.off();
+            pauseCoordinatorBeforeRemovingStateDoc.off();
+            awaitAbort();
+        }
+    });
+
+pauseCoordinatorBeforeRemovingStateDoc =
+    configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeRemovingStateDoc");
+
+let pauseBeforeRemoveRecipientDoc;
 reshardingTest.withReshardingInBackground(
     {
         newShardKeyPattern: {newKey: 1},
         newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}],
     },
     (tempNs) => {
-        const mongos = sourceCollection.getMongo();
-        const ns = sourceCollection.getFullName();
         const db = sourceCollection.getDB();
 
         let res;
@@ -77,7 +147,44 @@ reshardingTest.withReshardingInBackground(
             ErrorCodes.ReshardCollectionInProgress);
 
         mongos.getCollection(newNs).drop();
+
+        const recipient = new Mongo(topology.shards[recipientShardNames[0]].primary);
+        pauseBeforeRemoveRecipientDoc =
+            configureFailPoint(recipient, "removeRecipientDocFailpoint");
+    },
+    {
+        postDecisionPersistedFn: () => {
+            // Once the coordinator has persisted a commit decision, collMod, createIndexes, and
+            // dropIndexes should be able to succeed. Wait until the recipient is aware that the
+            // coordinator has persisted the decision.
+            const recipient = new Mongo(topology.shards[recipientShardNames[0]].primary);
+            assert.soon(() => {
+                const res = recipient.getCollection("config.cache.collections").findOne({_id: ns});
+                return res.reshardingFields.state === "committing";
+            }, () => `timed out waiting for the coordinator to persist decision: ${tojson(res)}`);
+
+            const db = mongos.getDB("reshardingDb");
+            assert.commandWorked(db.runCommand({collMod: 'coll'}));
+
+            // Create two indexes - one (idx1) that we will drop right away to ensure that
+            // dropIndexes succeeds, and the other (idx2) we will check still exists after the
+            // collection has been renamed.
+            assert.commandWorked(db.runCommand(
+                {createIndexes: 'coll', indexes: [{name: "idx1", key: {newKey: 1, oldKey: 1}}]}));
+            assert.commandWorked(db.runCommand(
+                {createIndexes: 'coll', indexes: [{name: "idx2", key: {newKey: 1, x: 1}}]}));
+
+            assert.commandWorked(
+                db.runCommand({dropIndexes: 'coll', index: {newKey: 1, oldKey: 1}}));
+
+            pauseBeforeRemoveRecipientDoc.off();
+            pauseCoordinatorBeforeRemovingStateDoc.off();
+        }
     });
 
+// Assert that 'idx2' still exists after we've renamed the collection and resharding is finished.
+let indexes = mongos.getDB("reshardingDb").runCommand({listIndexes: 'coll'});
+assert(indexes.cursor.firstBatch.some((index) => index.name === 'idx2'));
+
 reshardingTest.teardown();
 })();
diff --git a/src/mongo/db/s/collection_metadata.cpp b/src/mongo/db/s/collection_metadata.cpp
index c96a4848842..cacd174fbbc 100644
--- a/src/mongo/db/s/collection_metadata.cpp
+++ b/src/mongo/db/s/collection_metadata.cpp
@@ -95,11 +95,15 @@ boost::optional<ShardKeyPattern> CollectionMetadata::getReshardingKeyIfShouldFor
 }
 
 void CollectionMetadata::throwIfReshardingInProgress(NamespaceString const& nss) const {
-    if (isSharded() && getReshardingFields()) {
-        LOGV2(5277122, "reshardCollection in progress", "namespace"_attr = nss.toString());
+    if (isSharded()) {
+        const auto& reshardingFields = getReshardingFields();
+        // Throw if the coordinator is not in states "aborting", "committing", or "done".
+        if (reshardingFields && reshardingFields->getState() < CoordinatorStateEnum::kAborting) {
+            LOGV2(5277122, "reshardCollection in progress", "namespace"_attr = nss.toString());
 
-        uasserted(ErrorCodes::ReshardCollectionInProgress,
-                  "reshardCollection is in progress for namespace " + nss.toString());
+            uasserted(ErrorCodes::ReshardCollectionInProgress,
+                      "reshardCollection is in progress for namespace " + nss.toString());
+        }
     }
 }
 
diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
index 30fc19f9afb..eb419000180 100644
--- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
+++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp
@@ -74,6 +74,7 @@ MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorAfterPreparingToDonate);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCloning);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeBlockingWrites);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeDecisionPersisted);
+MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeRemovingStateDoc);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCompletion);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeStartingErrorFlow);
 MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforePersistingStateTransition);
@@ -1703,6 +1704,9 @@ ReshardingCoordinatorService::ReshardingCoordinator::_awaitAllParticipantShardsD
             auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc());
             auto& coordinatorDoc = coordinatorDocsChangedOnDisk[1];
 
+            reshardingPauseCoordinatorBeforeRemovingStateDoc.pauseWhileSetAndNotCanceled(
+                opCtx.get(), _ctHolder->getStepdownToken());
+
             boost::optional<Status> abortReason;
             if (coordinatorDoc.getAbortReason()) {
                 abortReason = getStatusFromAbortReason(coordinatorDoc);