SERVER-67372 Make tenant migration recipient delete its state document in its run method

4 files changed, 268 insertions, 11 deletions

diff --git a/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js b/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js
new file mode 100644
index 00000000000..37a8bc95a41
--- /dev/null
+++ b/jstests/replsets/tenant_migration_recipient_directly_deletes_its_state_doc.js
@@ -0,0 +1,133 @@
+/**
+ * Tests that a tenant migration recipient instance shows up as active in serverStatus metrics until
+ * it has directly deleted its state doc (even if the state doc has actually already been deleted by
+ * the TTL monitor).
+ *
+ * @tags: [
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ *   # Uses pauseTenantMigrationRecipientBeforeDeletingStateDoc failpoint, which was added in 6.2
+ *   requires_fcv_62,
+ *   requires_majority_read_concern,
+ *   requires_persistence,
+ *   serverless,
+ * ]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/uuid_util.js");
+load("jstests/replsets/libs/tenant_migration_test.js");
+
+(() => {
+    jsTest.log("Test case where the TTL monitor deletes the state doc first");
+
+    const tmt = new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
+
+    const recipientPrimary = tmt.getRecipientPrimary();
+    const fp =
+        configureFailPoint(recipientPrimary, "pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
+    jsTest.log("Confirm serverStatus does not show any active or completed tenant migrations.");
+    let recipientStats = tmt.getTenantMigrationStats(recipientPrimary);
+    jsTest.log("recipientStats: " + tojson(recipientStats));
+    assert.eq(0, recipientStats.currentMigrationsReceiving);
+
+    jsTest.log("Start a tenant migration.");
+    const tenantId = "testTenantId";
+    const migrationId = extractUUIDFromObject(UUID());
+    const migrationOpts = {
+        migrationIdString: migrationId,
+        tenantId: tenantId,
+        recipientConnString: tmt.getRecipientConnString(),
+    };
+
+    TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts));
+
+    jsTest.log("Wait for the migration to reach the failpoint before deleting the state doc.");
+    fp.wait();
+
+    jsTest.log("Confirm the state doc has been deleted by the TTL monitor");
+    assert.soon(() => {
+        return 0 ==
+            recipientPrimary.getDB("config").getCollection("tenantMigrationRecipients").count();
+    });
+
+    jsTest.log("Confirm the instance still shows up as active in serverStatus.");
+    recipientStats = tmt.getTenantMigrationStats(recipientPrimary);
+    jsTest.log("recipientStats: " + tojson(recipientStats));
+    assert.eq(1, recipientStats.currentMigrationsReceiving);
+
+    // TODO (SERVER-61717): Confirm the instance still shows up in the POS map. Currently, the
+    // instance is removed from the map as soon as its' state doc is deleted by the TTL monitor.
+
+    jsTest.log("Turn off the failpoint.");
+    fp.off();
+
+    jsTest.log("Confirm the instance eventually stops showing up as active in serverStatus");
+    assert.soon(() => {
+        recipientStats = tmt.getTenantMigrationStats(recipientPrimary);
+        return 0 == recipientStats.currentMigrationsReceiving;
+    });
+
+    // TODO (SERVER-61717): Confirm the instance eventually stops showing up in the POS map.
+
+    tmt.stop();
+})();
+
+(() => {
+    jsTest.log("Test case where the instance deletes the state doc first");
+
+    const tmt = new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
+
+    const recipientPrimary = tmt.getRecipientPrimary();
+
+    jsTest.log("Confirm the TTL index exists");
+    const listIndexesRes1 = assert.commandWorked(
+        recipientPrimary.getDB("config").runCommand({listIndexes: "tenantMigrationRecipients"}));
+    assert(listIndexesRes1.cursor.firstBatch.some(
+        elem => elem.name === "TenantMigrationRecipientTTLIndex" && elem.key.expireAt === 1));
+
+    jsTest.log("Drop the TTL index");
+    assert.commandWorked(recipientPrimary.getDB("config").runCommand(
+        {dropIndexes: "tenantMigrationRecipients", index: "TenantMigrationRecipientTTLIndex"}));
+
+    jsTest.log("Confirm the TTL index no longer exists");
+    const listIndexesRes2 = assert.commandWorked(
+        recipientPrimary.getDB("config").runCommand({listIndexes: "tenantMigrationRecipients"}));
+    assert(listIndexesRes2.cursor.firstBatch.every(elem => elem.key.expireAt == null));
+
+    jsTest.log("Confirm serverStatus does not show any active or completed tenant migrations.");
+    let recipientStats = tmt.getTenantMigrationStats(recipientPrimary);
+    jsTest.log("recipientStats: " + tojson(recipientStats));
+    assert.eq(0, recipientStats.currentMigrationsReceiving);
+
+    jsTest.log("Start a tenant migration.");
+    const tenantId = "testTenantId";
+    const migrationId = extractUUIDFromObject(UUID());
+    const migrationOpts = {
+        migrationIdString: migrationId,
+        tenantId: tenantId,
+        recipientConnString: tmt.getRecipientConnString(),
+    };
+    TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts));
+
+    jsTest.log("Wait for the instance to delete the state doc");
+    assert.soon(() => {
+        return 0 ==
+            recipientPrimary.getDB("config").getCollection("tenantMigrationRecipients").count();
+    });
+
+    jsTest.log("Confirm the instance eventually stops showing up as active in serverStatus");
+    assert.soon(() => {
+        recipientStats = tmt.getTenantMigrationStats(recipientPrimary);
+        return 0 == recipientStats.currentMigrationsReceiving;
+    });
+
+    // TODO (SERVER-61717): Confirm the instance eventually stops showing up in the POS map.
+
+    tmt.stop();
+})();
+})();
diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp
index 847b8283788..869670c6ca3 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp
+++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/namespace_string.h"
 #include "mongo/db/op_observer/op_observer.h"
 #include "mongo/db/ops/write_ops_exec.h"
+#include "mongo/db/persistent_task_store.h"
 #include "mongo/db/pipeline/process_interface/mongo_process_interface.h"
 #include "mongo/db/repl/cloner_utils.h"
 #include "mongo/db/repl/data_replicator_external_state.h"
@@ -146,6 +147,7 @@ MONGO_FAIL_POINT_DEFINE(skipComparingRecipientAndDonorFCV);
 MONGO_FAIL_POINT_DEFINE(autoRecipientForgetMigration);
 MONGO_FAIL_POINT_DEFINE(skipFetchingCommittedTransactions);
 MONGO_FAIL_POINT_DEFINE(skipFetchingRetryableWritesEntriesBeforeStartOpTime);
+MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationRecipientBeforeDeletingStateDoc);
 
 // Fails before waiting for the state doc to be majority replicated.
 MONGO_FAIL_POINT_DEFINE(failWhilePersistingTenantMigrationRecipientInstanceStateDoc);
@@ -3001,6 +3003,13 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run(
             return storageInterface->dropCollection(opCtx.get(),
                                                     getOplogBufferNs(getMigrationUUID()));
         })
+        .then([this, self = shared_from_this(), token] {
+            {
+                stdx::lock_guard lk(_mutex);
+                setPromiseOkifNotReady(lk, _forgetMigrationDurablePromise);
+            }
+            return _waitForGarbageCollectionDelayThenDeleteStateDoc(token);
+        })
         .thenRunOn(_recipientService->getInstanceCleanupExecutor())
         .onCompletion([this,
                        self = shared_from_this(),
@@ -3009,16 +3018,7 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run(
             // is safe even on shutDown/stepDown.
             stdx::lock_guard lk(_mutex);
             invariant(_dataSyncCompletionPromise.getFuture().isReady());
-            if (status.isOK()) {
-                LOGV2(4881401,
-                      "Migration marked to be garbage collectable due to "
-                      "recipientForgetMigration "
-                      "command",
-                      "migrationId"_attr = getMigrationUUID(),
-                      "tenantId"_attr = getTenantId(),
-                      "expireAt"_attr = *_stateDoc.getExpireAt());
-                setPromiseOkifNotReady(lk, _forgetMigrationDurablePromise);
-            } else {
+            if (!status.isOK()) {
                 // We should only hit here on a stepDown/shutDown, or a 'conflicting migration'
                 // error.
                 LOGV2(4881402,
@@ -3033,6 +3033,52 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run(
         .semi();
 }
 
+SemiFuture<void> TenantMigrationRecipientService::Instance::_removeStateDoc(
+    const CancellationToken& token) {
+    return AsyncTry([this, self = shared_from_this()] {
+               auto opCtxHolder = cc().makeOperationContext();
+               auto opCtx = opCtxHolder.get();
+
+               pauseTenantMigrationRecipientBeforeDeletingStateDoc.pauseWhileSet(opCtx);
+
+               PersistentTaskStore<TenantMigrationRecipientDocument> store(_stateDocumentsNS);
+               store.remove(
+                   opCtx,
+                   BSON(TenantMigrationRecipientDocument::kIdFieldName << _migrationUuid),
+                   WriteConcernOptions(1, WriteConcernOptions::SyncMode::UNSET, Seconds(0)));
+               LOGV2(8423371,
+                     "State document is deleted",
+                     "migrationId"_attr = _migrationUuid,
+                     "tenantId"_attr = _tenantId);
+           })
+        .until([](Status status) { return status.isOK(); })
+        .withBackoffBetweenIterations(kExponentialBackoff)
+        .on(**_scopedExecutor, token)
+        .semi();
+}
+
+SemiFuture<void>
+TenantMigrationRecipientService::Instance::_waitForGarbageCollectionDelayThenDeleteStateDoc(
+    const CancellationToken& token) {
+    stdx::lock_guard<Latch> lg(_mutex);
+    LOGV2(8423369,
+          "Waiting for garbage collection delay before deleting state document",
+          "migrationId"_attr = _migrationUuid,
+          "tenantId"_attr = _tenantId,
+          "expireAt"_attr = *_stateDoc.getExpireAt());
+
+    return (**_scopedExecutor)
+        ->sleepUntil(*_stateDoc.getExpireAt(), token)
+        .then([this, self = shared_from_this(), token]() {
+            LOGV2(8423370,
+                  "Deleting state document",
+                  "migrationId"_attr = _migrationUuid,
+                  "tenantId"_attr = _tenantId);
+            return _removeStateDoc(token);
+        })
+        .semi();
+}
+
 const UUID& TenantMigrationRecipientService::Instance::getMigrationUUID() const {
     return _migrationUuid;
 }
diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h
index bce11fb695d..22de9a00fd1 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_service.h
+++ b/src/mongo/db/repl/tenant_migration_recipient_service.h
@@ -214,6 +214,9 @@ public:
     private:
         friend class TenantMigrationRecipientServiceTest;
 
+        const NamespaceString _stateDocumentsNS =
+            NamespaceString::kTenantMigrationRecipientsNamespace;
+
         using ConnectionPair =
             std::pair<std::unique_ptr<DBClientConnection>, std::unique_ptr<DBClientConnection>>;
 
@@ -334,6 +337,16 @@ public:
         SemiFuture<void> _markStateDocAsGarbageCollectable();
 
         /**
+         * Deletes the state document. Does not return the opTime for the delete, since it's not
+         * necessary to wait for this delete to be majority committed (this is one of the last steps
+         * in the chain, and if the delete rolls back, the new primary will re-do the delete).
+         */
+        SemiFuture<void> _removeStateDoc(const CancellationToken& token);
+
+        SemiFuture<void> _waitForGarbageCollectionDelayThenDeleteStateDoc(
+            const CancellationToken& token);
+
+        /**
          * Creates a client, connects it to the donor. If '_transientSSLParams' is not none, uses
          * the migration certificate to do SSL authentication. Otherwise, uses the default
          * authentication mode. Throws a user assertion on failure.
diff --git a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp
index 55c295b5c3c..8612c951131 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp
+++ b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp
@@ -1326,6 +1326,8 @@ TEST_F(TenantMigrationRecipientServiceTest,
 
 TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTime_NoTransaction) {
     stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
@@ -1360,6 +1362,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTi
 
 TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTime_Transaction) {
     stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
     const OpTime txnStartOpTime(Timestamp(3, 1), 1);
@@ -1403,6 +1407,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientGetStartOpTi
 TEST_F(TenantMigrationRecipientServiceTest,
        TenantMigrationRecipientGetStartOpTimes_RemoteOplogQueryFails) {
     stopFailPointEnableBlock fp("fpAfterRetrievingStartOpTimesMigrationRecipientInstance");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
 
@@ -1434,6 +1440,8 @@ TEST_F(TenantMigrationRecipientServiceTest,
 
 TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartOplogFetcher) {
     stopFailPointEnableBlock fp("fpAfterStartingOplogFetcherMigrationRecipientInstance");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     auto taskFp = globalFailPointRegistry().find("hangBeforeTaskCompletion");
     auto initialTimesEntered = taskFp->setMode(FailPoint::alwaysOn);
@@ -1482,6 +1490,8 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartOplogFe
 
 TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartsCloner) {
     stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     auto taskFp = globalFailPointRegistry().find("hangBeforeTaskCompletion");
     ScopeGuard taskFpGuard([&taskFp] { taskFp->setMode(FailPoint::off); });
@@ -1550,6 +1560,9 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartsCloner
 }
 
 TEST_F(TenantMigrationRecipientServiceTest, OplogFetcherFailsDuringOplogApplication) {
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -2385,6 +2398,9 @@ TEST_F(TenantMigrationRecipientServiceTest,
 }
 
 TEST_F(TenantMigrationRecipientServiceTest, OplogApplierFails) {
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
     const OpTime injectedEntryOpTime(Timestamp(6, 1), 1);
@@ -2453,6 +2469,9 @@ TEST_F(TenantMigrationRecipientServiceTest, OplogApplierFails) {
 }
 
 TEST_F(TenantMigrationRecipientServiceTest, StoppingApplierAllowsCompletion) {
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -2687,6 +2706,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_WaitUntilSt
     // state doc.
     auto autoForgetFp = globalFailPointRegistry().find("autoRecipientForgetMigration");
     autoForgetFp->setMode(FailPoint::off);
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
@@ -2772,7 +2793,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterStartO
                                            0,
                                            BSON("action"
                                                 << "hang"));
-
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
@@ -2838,6 +2860,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterConsis
                                                          0,
                                                          BSON("action"
                                                               << "hang"));
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
 
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
@@ -2927,6 +2952,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterFail)
     auto autoForgetFp = globalFailPointRegistry().find("autoRecipientForgetMigration");
     autoForgetFp->setMode(FailPoint::off);
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions");
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
@@ -3015,6 +3043,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_FailToMarkG
     stopFailPointEnableBlock fp("fpAfterPersistingTenantMigrationRecipientInstanceStateDoc");
     const UUID migrationUUID = UUID::gen();
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */);
     getTopologyManager()->setTopologyDescription(replSet.getTopologyDescription(clock()));
 
@@ -3066,6 +3097,8 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_FailToMarkG
 
 TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientServiceRecordsFCVAtStart) {
     stopFailPointEnableBlock fp("fpAfterRecordingRecipientPrimaryStartingFCV");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
 
     const UUID migrationUUID = UUID::gen();
     MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */);
@@ -3102,6 +3135,9 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientServiceRecor
 TEST_F(TenantMigrationRecipientServiceTest,
        TenantMigrationRecipientServiceAlreadyRecordedFCV_Match) {
     stopFailPointEnableBlock fp("fpAfterRecordingRecipientPrimaryStartingFCV");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
 
     const UUID migrationUUID = UUID::gen();
     MockReplicaSet replSet("donorSet", 3, true /* hasPrimary */, true /* dollarPrefixHosts */);
@@ -3225,6 +3261,9 @@ TEST_F(TenantMigrationRecipientServiceTest,
 }
 
 TEST_F(TenantMigrationRecipientServiceTest, WaitUntilMigrationReachesReturnAfterReachingTimestamp) {
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3286,6 +3325,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableFetcherErr
                                            BSON("action"
                                                 << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3351,6 +3393,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableFetcher
                                            BSON("action"
                                                 << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3411,6 +3456,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientWillNotRetryOnExternalInter
                                            BSON("action"
                                                 << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3471,6 +3519,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientWillNotRetryOnReceivingForg
                                                  BSON("action"
                                                       << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3543,6 +3594,10 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableClonerErro
                                            BSON("action"
                                                 << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3616,6 +3671,10 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableClonerE
                                            BSON("action"
                                                 << "hang"));
 
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(5, 1), 1);
 
@@ -3672,6 +3731,9 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableClonerE
 TEST_F(TenantMigrationRecipientServiceTest, IncrementNumRestartsDueToRecipientFailureCounter) {
     FailPointEnableBlock createIndexesFailpointBlock("skipCreatingIndexDuringRebuildService");
     stopFailPointEnableBlock fp("fpAfterPersistingTenantMigrationRecipientInstanceStateDoc");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(1, 1), 1);
 
@@ -3725,6 +3787,9 @@ TEST_F(TenantMigrationRecipientServiceTest, IncrementNumRestartsDueToRecipientFa
 TEST_F(TenantMigrationRecipientServiceTest,
        RecipientFailureCounterNotIncrementedWhenMigrationForgotten) {
     FailPointEnableBlock createIndexesFailpointBlock("skipCreatingIndexDuringRebuildService");
+    // Hang before deleting the state doc so that we can check the state doc was persisted.
+    FailPointEnableBlock fpDeletingStateDoc("pauseTenantMigrationRecipientBeforeDeletingStateDoc");
+
     const UUID migrationUUID = UUID::gen();
     const OpTime topOfOplogOpTime(Timestamp(1, 1), 1);