diff options
author | jannaerin <golden.janna@gmail.com> | 2021-07-20 19:12:46 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-08-17 16:15:33 +0000 |
commit | 7758ae1a0a21c662f028a687ff6b25506a52c33c (patch) | |
tree | 474239e2bf416edcefe1f03a7251ea4f3c4ac6ec | |
parent | 128c79d29dbf11696e0b2a7efe91ec05eca8e4b3 (diff) | |
download | mongo-7758ae1a0a21c662f028a687ff6b25506a52c33c.tar.gz |
SERVER-58669 Allow collMod, createIndexes, and dropIndexes once resharding coordinator has made a decision
(cherry picked from commit a1e57d2985b672c898431b7787e53c547bbd2753)
SERVER-58917 Wait until donors/recipients are aware that coordinator has persisted decision before expecting successful collMod, createIndexes, and dropIndexes in resharding_prohibited_commands.js
(cherry picked from commit 8179692b362d8237f201719946eb46bac6a5e961)
-rw-r--r-- | jstests/sharding/resharding_prohibited_commands.js | 115 | ||||
-rw-r--r-- | src/mongo/db/s/collection_metadata.cpp | 12 | ||||
-rw-r--r-- | src/mongo/db/s/resharding/resharding_coordinator_service.cpp | 4 |
3 files changed, 123 insertions, 8 deletions
diff --git a/jstests/sharding/resharding_prohibited_commands.js b/jstests/sharding/resharding_prohibited_commands.js index 9167db7befb..d512b859ed3 100644 --- a/jstests/sharding/resharding_prohibited_commands.js +++ b/jstests/sharding/resharding_prohibited_commands.js @@ -13,26 +13,96 @@ load("jstests/libs/discover_topology.js"); load("jstests/sharding/libs/resharding_test_fixture.js"); +load("jstests/libs/fail_point_util.js"); +load('jstests/libs/parallel_shell_helpers.js'); const reshardingTest = new ReshardingTest({numDonors: 2}); reshardingTest.setup(); const donorShardNames = reshardingTest.donorShardNames; -const sourceCollection = reshardingTest.createShardedCollection({ +let sourceCollection = reshardingTest.createShardedCollection({ ns: "reshardingDb.coll", shardKeyPattern: {oldKey: 1}, - chunks: [{min: {oldKey: MinKey}, max: {oldKey: MaxKey}, shard: donorShardNames[0]}], + chunks: [ + {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, + {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]} + ], }); const recipientShardNames = reshardingTest.recipientShardNames; +const mongos = sourceCollection.getMongo(); +let ns = sourceCollection.getFullName(); + +const topology = DiscoverTopology.findConnectedNodes(mongos); +const configPrimary = new Mongo(topology.configsvr.primary); +const donor0 = new Mongo(topology.shards[donorShardNames[0]].primary); +const donor1 = new Mongo(topology.shards[donorShardNames[1]].primary); + +let pauseCoordinatorBeforeRemovingStateDoc; +let pauseBeforeRemoveDonor0Doc; +let pauseBeforeRemoveDonor1Doc; +pauseCoordinatorBeforeRemovingStateDoc = + configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeRemovingStateDoc"); + +pauseBeforeRemoveDonor0Doc = configureFailPoint(donor0, "removeDonorDocFailpoint"); +pauseBeforeRemoveDonor1Doc = configureFailPoint(donor1, "removeDonorDocFailpoint"); + +let awaitAbort; +reshardingTest.withReshardingInBackground( + { + newShardKeyPattern: {newKey: 1}, + newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}], + }, + (tempNs) => { + const topology = DiscoverTopology.findConnectedNodes(mongos); + assert.soon(() => { + let res = + donor0.getCollection("config.localReshardingOperations.donor").find().toArray(); + return res.length == 1; + }, "timed out waiting for resharding initialization on donor shard"); + + awaitAbort = startParallelShell( + funWithArgs(function(ns) { + assert.commandWorked(db.adminCommand({abortReshardCollection: ns})); + }, ns), mongos.port); + }, + { + expectedErrorCode: ErrorCodes.ReshardCollectionAborted, + postDecisionPersistedFn: () => { + // Once the coordinator has persisted an abort decision, collMod, createIndexes, and + // dropIndexes should be able to succeed. Wait until both donors have heard that the + // coordinator has made the decision. + assert.soon(() => { + const res0 = donor0.getCollection("config.cache.collections").findOne({_id: ns}); + const res1 = donor1.getCollection("config.cache.collections").findOne({_id: ns}); + return res0.reshardingFields.state === "aborting" && + res1.reshardingFields.state === "aborting"; + }, () => `timed out waiting for the coordinator to persist decision`); + + const db = mongos.getDB("reshardingDb"); + assert.commandWorked(db.runCommand({collMod: 'coll'})); + assert.commandWorked(db.runCommand( + {createIndexes: 'coll', indexes: [{name: "idx1", key: {newKey: 1, oldKey: 1}}]})); + assert.commandWorked( + db.runCommand({dropIndexes: 'coll', index: {newKey: 1, oldKey: 1}})); + + pauseBeforeRemoveDonor0Doc.off(); + pauseBeforeRemoveDonor1Doc.off(); + pauseCoordinatorBeforeRemovingStateDoc.off(); + awaitAbort(); + } + }); + +pauseCoordinatorBeforeRemovingStateDoc = + configureFailPoint(configPrimary, "reshardingPauseCoordinatorBeforeRemovingStateDoc"); + +let pauseBeforeRemoveRecipientDoc; reshardingTest.withReshardingInBackground( { newShardKeyPattern: {newKey: 1}, newChunks: [{min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}], }, (tempNs) => { - const mongos = sourceCollection.getMongo(); - const ns = sourceCollection.getFullName(); const db = sourceCollection.getDB(); let res; @@ -77,7 +147,44 @@ reshardingTest.withReshardingInBackground( ErrorCodes.ReshardCollectionInProgress); mongos.getCollection(newNs).drop(); + + const recipient = new Mongo(topology.shards[recipientShardNames[0]].primary); + pauseBeforeRemoveRecipientDoc = + configureFailPoint(recipient, "removeRecipientDocFailpoint"); + }, + { + postDecisionPersistedFn: () => { + // Once the coordinator has persisted a commit decision, collMod, createIndexes, and + // dropIndexes should be able to succeed. Wait until the recipient is aware that the + // coordinator has persisted the decision. + const recipient = new Mongo(topology.shards[recipientShardNames[0]].primary); + assert.soon(() => { + const res = recipient.getCollection("config.cache.collections").findOne({_id: ns}); + return res.reshardingFields.state === "committing"; + }, () => `timed out waiting for the coordinator to persist decision: ${tojson(res)}`); + + const db = mongos.getDB("reshardingDb"); + assert.commandWorked(db.runCommand({collMod: 'coll'})); + + // Create two indexes - one (idx1) that we will drop right away to ensure that + // dropIndexes succeeds, and the other (idx2) we will check still exists after the + // collection has been renamed. + assert.commandWorked(db.runCommand( + {createIndexes: 'coll', indexes: [{name: "idx1", key: {newKey: 1, oldKey: 1}}]})); + assert.commandWorked(db.runCommand( + {createIndexes: 'coll', indexes: [{name: "idx2", key: {newKey: 1, x: 1}}]})); + + assert.commandWorked( + db.runCommand({dropIndexes: 'coll', index: {newKey: 1, oldKey: 1}})); + + pauseBeforeRemoveRecipientDoc.off(); + pauseCoordinatorBeforeRemovingStateDoc.off(); + } }); +// Assert that 'idx2' still exists after we've renamed the collection and resharding is finished. +let indexes = mongos.getDB("reshardingDb").runCommand({listIndexes: 'coll'}); +assert(indexes.cursor.firstBatch.some((index) => index.name === 'idx2')); + reshardingTest.teardown(); })(); diff --git a/src/mongo/db/s/collection_metadata.cpp b/src/mongo/db/s/collection_metadata.cpp index c96a4848842..cacd174fbbc 100644 --- a/src/mongo/db/s/collection_metadata.cpp +++ b/src/mongo/db/s/collection_metadata.cpp @@ -95,11 +95,15 @@ boost::optional<ShardKeyPattern> CollectionMetadata::getReshardingKeyIfShouldFor } void CollectionMetadata::throwIfReshardingInProgress(NamespaceString const& nss) const { - if (isSharded() && getReshardingFields()) { - LOGV2(5277122, "reshardCollection in progress", "namespace"_attr = nss.toString()); + if (isSharded()) { + const auto& reshardingFields = getReshardingFields(); + // Throw if the coordinator is not in states "aborting", "committing", or "done". + if (reshardingFields && reshardingFields->getState() < CoordinatorStateEnum::kAborting) { + LOGV2(5277122, "reshardCollection in progress", "namespace"_attr = nss.toString()); - uasserted(ErrorCodes::ReshardCollectionInProgress, - "reshardCollection is in progress for namespace " + nss.toString()); + uasserted(ErrorCodes::ReshardCollectionInProgress, + "reshardCollection is in progress for namespace " + nss.toString()); + } } } diff --git a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp index 30fc19f9afb..eb419000180 100644 --- a/src/mongo/db/s/resharding/resharding_coordinator_service.cpp +++ b/src/mongo/db/s/resharding/resharding_coordinator_service.cpp @@ -74,6 +74,7 @@ MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorAfterPreparingToDonate); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCloning); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeBlockingWrites); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeDecisionPersisted); +MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeRemovingStateDoc); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeCompletion); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforeStartingErrorFlow); MONGO_FAIL_POINT_DEFINE(reshardingPauseCoordinatorBeforePersistingStateTransition); @@ -1703,6 +1704,9 @@ ReshardingCoordinatorService::ReshardingCoordinator::_awaitAllParticipantShardsD auto opCtx = _cancelableOpCtxFactory->makeOperationContext(&cc()); auto& coordinatorDoc = coordinatorDocsChangedOnDisk[1]; + reshardingPauseCoordinatorBeforeRemovingStateDoc.pauseWhileSetAndNotCanceled( + opCtx.get(), _ctHolder->getStepdownToken()); + boost::optional<Status> abortReason; if (coordinatorDoc.getAbortReason()) { abortReason = getStatusFromAbortReason(coordinatorDoc); |