diff options
4 files changed, 268 insertions, 11 deletions
diff --git a/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js b/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js new file mode 100644 index 00000000000..37a8bc95a41 --- /dev/null +++ b/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js @@ -0,0 +1,133 @@ +/** + * Tests that a tenant migration recipient instance shows up as active in serverStatus metrics until + * it has directly deleted its state doc (even if the state doc has actually already been deleted by + * the TTL monitor). + * + * @tags: [ + * incompatible_with_macos, + * incompatible_with_windows_tls, + * # Uses pauseTenantMigrationRecipientBeforeDeletingStateDoc failpoint, which was added in 6.2 + * requires_fcv_62, + * requires_majority_read_concern, + * requires_persistence, + * serverless, + * ] + */ + +(function() { +"use strict"; + +load("jstests/libs/fail_point_util.js"); +load("jstests/libs/uuid_util.js"); +load("jstests/replsets/libs/tenant_migration_test.js"); + +(() => { + jsTest.log("Test case where the TTL monitor deletes the state doc first"); + + const tmt = new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); + + const recipientPrimary = tmt.getRecipientPrimary(); + const fp = + configureFailPoint(recipientPrimary, "pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + + jsTest.log("Confirm serverStatus does not show any active or completed tenant migrations."); + let recipientStats = tmt.getTenantMigrationStats(recipientPrimary); + jsTest.log("recipientStats: " + tojson(recipientStats)); + assert.eq(0, recipientStats.currentMigrationsReceiving); + + jsTest.log("Start a tenant migration."); + const tenantId = "testTenantId"; + const migrationId = extractUUIDFromObject(UUID()); + const migrationOpts = { + migrationIdString: migrationId, + tenantId: tenantId, + recipientConnString: tmt.getRecipientConnString(), + }; + + TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts)); + + jsTest.log("Wait for the migration to reach the failpoint before deleting the state doc."); + fp.wait(); + + jsTest.log("Confirm the state doc has been deleted by the TTL monitor"); + assert.soon(() => { + return 0 == + recipientPrimary.getDB("config").getCollection("tenantMigrationRecipients").count(); + }); + + jsTest.log("Confirm the instance still shows up as active in serverStatus."); + recipientStats = tmt.getTenantMigrationStats(recipientPrimary); + jsTest.log("recipientStats: " + tojson(recipientStats)); + assert.eq(1, recipientStats.currentMigrationsReceiving); + + // TODO (SERVER-61717): Confirm the instance still shows up in the POS map. Currently, the + // instance is removed from the map as soon as its' state doc is deleted by the TTL monitor. + + jsTest.log("Turn off the failpoint."); + fp.off(); + + jsTest.log("Confirm the instance eventually stops showing up as active in serverStatus"); + assert.soon(() => { + recipientStats = tmt.getTenantMigrationStats(recipientPrimary); + return 0 == recipientStats.currentMigrationsReceiving; + }); + + // TODO (SERVER-61717): Confirm the instance eventually stops showing up in the POS map. + + tmt.stop(); +})(); + +(() => { + jsTest.log("Test case where the instance deletes the state doc first"); + + const tmt = new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); + + const recipientPrimary = tmt.getRecipientPrimary(); + + jsTest.log("Confirm the TTL index exists"); + const listIndexesRes1 = assert.commandWorked( + recipientPrimary.getDB("config").runCommand({listIndexes: "tenantMigrationRecipients"})); + assert(listIndexesRes1.cursor.firstBatch.some( + elem => elem.name === "TenantMigrationRecipientTTLIndex" && elem.key.expireAt === 1)); + + jsTest.log("Drop the TTL index"); + assert.commandWorked(recipientPrimary.getDB("config").runCommand( + {dropIndexes: "tenantMigrationRecipients", index: "TenantMigrationRecipientTTLIndex"})); + + jsTest.log("Confirm the TTL index no longer exists"); + const listIndexesRes2 = assert.commandWorked( + recipientPrimary.getDB("config").runCommand({listIndexes: "tenantMigrationRecipients"})); + assert(listIndexesRes2.cursor.firstBatch.every(elem => elem.key.expireAt == null)); + + jsTest.log("Confirm serverStatus does not show any active or completed tenant migrations."); + let recipientStats = tmt.getTenantMigrationStats(recipientPrimary); + jsTest.log("recipientStats: " + tojson(recipientStats)); + assert.eq(0, recipientStats.currentMigrationsReceiving); + + jsTest.log("Start a tenant migration."); + const tenantId = "testTenantId"; + const migrationId = extractUUIDFromObject(UUID()); + const migrationOpts = { + migrationIdString: migrationId, + tenantId: tenantId, + recipientConnString: tmt.getRecipientConnString(), + }; + TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts)); + + jsTest.log("Wait for the instance to delete the state doc"); + assert.soon(() => { + return 0 == + recipientPrimary.getDB("config").getCollection("tenantMigrationRecipients").count(); + }); + + jsTest.log("Confirm the instance eventually stops showing up as active in serverStatus"); + assert.soon(() => { + recipientStats = tmt.getTenantMigrationStats(recipientPrimary); + return 0 == recipientStats.currentMigrationsReceiving; + }); + + // TODO (SERVER-61717): Confirm the instance eventually stops showing up in the POS map. + + tmt.stop(); +})(); +})(); diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp index 847b8283788..869670c6ca3 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp @@ -49,6 +49,7 @@ #include "mongo/db/namespace_string.h" #include "mongo/db/op_observer/op_observer.h" #include "mongo/db/ops/write_ops_exec.h" +#include "mongo/db/persistent_task_store.h" #include "mongo/db/pipeline/process_interface/mongo_process_interface.h" #include "mongo/db/repl/cloner_utils.h" #include "mongo/db/repl/data_replicator_external_state.h" @@ -146,6 +147,7 @@ MONGO_FAIL_POINT_DEFINE(skipComparingRecipientAndDonorFCV); MONGO_FAIL_POINT_DEFINE(autoRecipientForgetMigration); MONGO_FAIL_POINT_DEFINE(skipFetchingCommittedTransactions); MONGO_FAIL_POINT_DEFINE(skipFetchingRetryableWritesEntriesBeforeStartOpTime); +MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationRecipientBeforeDeletingStateDoc); // Fails before waiting for the state doc to be majority replicated. MONGO_FAIL_POINT_DEFINE(failWhilePersistingTenantMigrationRecipientInstanceStateDoc); @@ -3001,6 +3003,13 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run( return storageInterface->dropCollection(opCtx.get(), getOplogBufferNs(getMigrationUUID())); }) + .then([this, self = shared_from_this(), token] { + { + stdx::lock_guard lk(_mutex); + setPromiseOkifNotReady(lk, _forgetMigrationDurablePromise); + } + return _waitForGarbageCollectionDelayThenDeleteStateDoc(token); + }) .thenRunOn(_recipientService->getInstanceCleanupExecutor()) .onCompletion([this, self = shared_from_this(), @@ -3009,16 +3018,7 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run( // is safe even on shutDown/stepDown. stdx::lock_guard lk(_mutex); invariant(_dataSyncCompletionPromise.getFuture().isReady()); - if (status.isOK()) { - LOGV2(4881401, - "Migration marked to be garbage collectable due to " - "recipientForgetMigration " - "command", - "migrationId"_attr = getMigrationUUID(), - "tenantId"_attr = getTenantId(), - "expireAt"_attr = *_stateDoc.getExpireAt()); - setPromiseOkifNotReady(lk, _forgetMigrationDurablePromise); - } else { + if (!status.isOK()) { // We should only hit here on a stepDown/shutDown, or a 'conflicting migration' // error. LOGV2(4881402, @@ -3033,6 +3033,52 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run( .semi(); } +SemiFuture<void> TenantMigrationRecipientService::Instance::_removeStateDoc( + const CancellationToken& token) { + return AsyncTry([this, self = shared_from_this()] { + auto opCtxHolder = cc().makeOperationContext(); + auto opCtx = opCtxHolder.get(); + + pauseTenantMigrationRecipientBeforeDeletingStateDoc.pauseWhileSet(opCtx); + + PersistentTaskStore<TenantMigrationRecipientDocument> store(_stateDocumentsNS); + store.remove( + opCtx, + BSON(TenantMigrationRecipientDocument::kIdFieldName << _migrationUuid), + WriteConcernOptions(1, WriteConcernOptions::SyncMode::UNSET, Seconds(0))); + LOGV2(8423371, + "State document is deleted", + "migrationId"_attr = _migrationUuid, + "tenantId"_attr = _tenantId); + }) + .until([](Status status) { return status.isOK(); }) + .withBackoffBetweenIterations(kExponentialBackoff) + .on(**_scopedExecutor, token) + .semi(); +} + +SemiFuture<void> +TenantMigrationRecipientService::Instance::_waitForGarbageCollectionDelayThenDeleteStateDoc( + const CancellationToken& token) { + stdx::lock_guard<Latch> lg(_mutex); + LOGV2(8423369, + "Waiting for garbage collection delay before deleting state document", + "migrationId"_attr = _migrationUuid, + "tenantId"_attr = _tenantId, + "expireAt"_attr = *_stateDoc.getExpireAt()); + + return (**_scopedExecutor) + ->sleepUntil(*_stateDoc.getExpireAt(), token) + .then([this, self = shared_from_this(), token]() { + LOGV2(8423370, + "Deleting state document", + "migrationId"_attr = _migrationUuid, + "tenantId"_attr = _tenantId); + return _removeStateDoc(token); + }) + .semi(); +} + const UUID& TenantMigrationRecipientService::Instance::getMigrationUUID() const { return _migrationUuid; } diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h index bce11fb695d..22de9a00fd1 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.h +++ b/src/mongo/db/repl/tenant_migration_recipient_service.h @@ -214,6 +214,9 @@ public: private: friend class TenantMigrationRecipientServiceTest; + const NamespaceString _stateDocumentsNS = + NamespaceString::kTenantMigrationRecipientsNamespace; + using ConnectionPair = std::pair<std::unique_ptr<DBClientConnection>, std::unique_ptr<DBClientConnection>>; @@ -334,6 +337,16 @@ public: SemiFuture<void> _markStateDocAsGarbageCollectable(); /** + * Deletes the state document. Does not return the opTime for the delete, since it's not + * necessary to wait for this delete to be majority committed (this is one of the last steps + * in the chain, and if the delete rolls back, the new primary will re-do the delete). + */ + SemiFuture<void> _removeStateDoc(const CancellationToken& token); + + SemiFuture<void> _waitForGarbageCollectionDelayThenDeleteStateDoc( + const CancellationToken& token); + + /** * Creates a client, connects it to the donor. If '_transientSSLParams' is not none, uses * the migration certificate to do SSL authentication. Otherwise, uses the default * authentication mode. Throws a user assertion on failure. diff --git a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp index 55c295b5c3c..8612c951131 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp @@ -1326,6 +1326,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTime_NoTransaction) { stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -1360,6 +1362,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTi TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTime_Transaction) { stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); const OpTime txnStartOpTime(Timestamp(3, 1), 1); @@ -1403,6 +1407,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTi TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTimes_RemoteOplogQueryFails) { stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); @@ -1434,6 +1440,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartOplogFetcher) { stopFailPointEnableBlock fp("fpAfterStartingOplogFetcherMigrationRecipientInstance"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); auto taskFp = globalFailPointRegistry().find("hangBeforeTaskCompletion"); auto initialTimesEntered = taskFp->setMode(FailPoint::alwaysOn); @@ -1482,6 +1490,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartOplogFe TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartsCloner) { stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); auto taskFp = globalFailPointRegistry().find("hangBeforeTaskCompletion"); ScopeGuard taskFpGuard([&taskFp] { taskFp->setMode(FailPoint::off); }); @@ -1550,6 +1560,9 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartsCloner } TEST_F(TenantMigrationRecipientServiceTest, OplogFetcherFailsDuringOplogApplication) { + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -2385,6 +2398,9 @@ TEST_F(TenantMigrationRecipientServiceTest, } TEST_F(TenantMigrationRecipientServiceTest, OplogApplierFails) { + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); const OpTime injectedEntryOpTime(Timestamp(6, 1), 1); @@ -2453,6 +2469,9 @@ TEST_F(TenantMigrationRecipientServiceTest, OplogApplierFails) { } TEST_F(TenantMigrationRecipientServiceTest, StoppingApplierAllowsCompletion) { + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -2687,6 +2706,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_WaitUntilSt // state doc. auto autoForgetFp = globalFailPointRegistry().find("autoRecipientForgetMigration"); autoForgetFp->setMode(FailPoint::off); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -2772,7 +2793,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterStartO 0, BSON("action" << "hang")); - + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -2838,6 +2860,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterConsis 0, BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -2927,6 +2952,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterFail) auto autoForgetFp = globalFailPointRegistry().find("autoRecipientForgetMigration"); autoForgetFp->setMode(FailPoint::off); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3015,6 +3043,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_FailToMarkG stopFailPointEnableBlock fp("fpAfterPersistingTenantMigrationRecipientInstanceStateDoc"); const UUID migrationUUID = UUID::gen(); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */); getTopologyManager()->setTopologyDescription(replSet.getTopologyDescription(clock())); @@ -3066,6 +3097,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_FailToMarkG TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientServiceRecordsFCVAtStart) { stopFailPointEnableBlock fp("fpAfterRecordingRecipientPrimaryStartingFCV"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); const UUID migrationUUID = UUID::gen(); MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */); @@ -3102,6 +3135,9 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientServiceRecor TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientServiceAlreadyRecordedFCV_Match) { stopFailPointEnableBlock fp("fpAfterRecordingRecipientPrimaryStartingFCV"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */); @@ -3225,6 +3261,9 @@ TEST_F(TenantMigrationRecipientServiceTest, } TEST_F(TenantMigrationRecipientServiceTest, WaitUntilMigrationReachesReturnAfterReachingTimestamp) { + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3286,6 +3325,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableFetcherErr BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3351,6 +3393,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableFetcher BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3411,6 +3456,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientWillNotRetryOnExternalInter BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3471,6 +3519,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientWillNotRetryOnReceivingForg BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3543,6 +3594,10 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableClonerErro BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3616,6 +3671,10 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableClonerE BSON("action" << "hang")); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3672,6 +3731,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableClonerE TEST_F(TenantMigrationRecipientServiceTest, IncrementNumRestartsDueToRecipientFailureCounter) { FailPointEnableBlock createIndexesFailpointBlock("skipCreatingIndexDuringRebuildService"); stopFailPointEnableBlock fp("fpAfterPersistingTenantMigrationRecipientInstanceStateDoc"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(1, 1), 1); @@ -3725,6 +3787,9 @@ TEST_F(TenantMigrationRecipientServiceTest, IncrementNumRestartsDueToRecipientFa TEST_F(TenantMigrationRecipientServiceTest, RecipientFailureCounterNotIncrementedWhenMigrationForgotten) { FailPointEnableBlock createIndexesFailpointBlock("skipCreatingIndexDuringRebuildService"); + // Hang before deleting the state doc so that we can check the state doc was persisted. + FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc"); + const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(1, 1), 1); |