SERVER-57832 Tenant collection cloner checkIfDonorCollectionIsEmptyStage doesn't handle collection drops on the donor

3 files changed, 77 insertions, 11 deletions

diff --git a/jstests/replsets/tenant_migration_drop_collection.js b/jstests/replsets/tenant_migration_drop_collection.js
index 409dfc19e0e..1c82423d520 100644
--- a/jstests/replsets/tenant_migration_drop_collection.js
+++ b/jstests/replsets/tenant_migration_drop_collection.js
@@ -127,7 +127,25 @@ runDropTest({
     createNew: true
 });
 
-jsTestLog("[3] Testing dropping between count and listIndexes.");
+jsTestLog("[3] Testing dropping before checkIfDonorCollectionIsEmpty stage.");
+runDropTest({
+    failPointName: "hangBeforeClonerStage",
+    failPointData: {cloner: "TenantCollectionCloner", stage: "checkIfDonorCollectionIsEmpty"},
+    expectedLog:
+        '{code: 5289701, attr: { namespace: nss, uuid: (x)=>(x.uuid.$uuid === uuid), tenantId: tenantId}}'
+});
+
+jsTestLog(
+    "[4] Testing dropping before checkIfDonorCollectionIsEmpty stage, with same-name collection created");
+runDropTest({
+    failPointName: "hangBeforeClonerStage",
+    failPointData: {cloner: "TenantCollectionCloner", stage: "checkIfDonorCollectionIsEmpty"},
+    expectedLog:
+        '{code: 5289701, attr: { namespace: nss, uuid: (x)=>(x.uuid.$uuid === uuid), tenantId: tenantId}}',
+    createNew: true
+});
+
+jsTestLog("[5] Testing dropping between checkIfDonorCollectionIsEmpty and listIndexes.");
 runDropTest({
     failPointName: "hangBeforeClonerStage",
     failPointData: {cloner: "TenantCollectionCloner", stage: "listIndexes"},
@@ -135,7 +153,8 @@ runDropTest({
         '{code: 5289701, attr: { namespace: nss, uuid: (x)=>(x.uuid.$uuid === uuid), tenantId: tenantId}}'
 });
 
-jsTestLog("[4] Testing dropping between count and listIndexes, with same-name collection created");
+jsTestLog(
+    "[6] Testing dropping between checkIfDonorCollectionIsEmpty and listIndexes, with same-name collection created");
 runDropTest({
     failPointName: "hangBeforeClonerStage",
     failPointData: {cloner: "TenantCollectionCloner", stage: "listIndexes"},
@@ -144,7 +163,7 @@ runDropTest({
     createNew: true
 });
 
-jsTestLog("[5] Testing dropping between listIndexes and find.");
+jsTestLog("[7] Testing dropping between listIndexes and find.");
 runDropTest({
     failPointName: "hangBeforeClonerStage",
     failPointData: {cloner: "TenantCollectionCloner", stage: "query"},
@@ -152,7 +171,7 @@ runDropTest({
         '{code: 5289701, attr: { namespace: nss, uuid: (x)=>(x.uuid.$uuid === uuid), tenantId: tenantId}}'
 });
 
-jsTestLog("[6] Testing dropping between listIndexes and find, with same-name collection created");
+jsTestLog("[8] Testing dropping between listIndexes and find, with same-name collection created");
 runDropTest({
     failPointName: "hangBeforeClonerStage",
     failPointData: {cloner: "TenantCollectionCloner", stage: "query"},
@@ -161,7 +180,7 @@ runDropTest({
     createNew: true
 });
 
-jsTestLog("[7] Testing dropping between getMore calls.");
+jsTestLog("[9] Testing dropping between getMore calls.");
 runDropTest({
     // Will trigger right after the first batch.
     failPointName: "tenantMigrationHangCollectionClonerAfterHandlingBatchResponse",
@@ -169,7 +188,7 @@ runDropTest({
         '{code: 5289701, attr: { namespace: nss, uuid: (x)=>(x.uuid.$uuid === uuid), tenantId: tenantId}}',
 });
 
-jsTestLog("[8] Testing dropping between getMore calls, with same-name collection created");
+jsTestLog("[10] Testing dropping between getMore calls, with same-name collection created");
 runDropTest({
     // Will trigger right after the first batch.
     failPointName: "tenantMigrationHangCollectionClonerAfterHandlingBatchResponse",
diff --git a/src/mongo/db/repl/tenant_collection_cloner.h b/src/mongo/db/repl/tenant_collection_cloner.h
index 4615670690b..b9c22928917 100644
--- a/src/mongo/db/repl/tenant_collection_cloner.h
+++ b/src/mongo/db/repl/tenant_collection_cloner.h
@@ -247,11 +247,11 @@ private:
     // The size of the batches of documents returned in collection cloning.
     int _collectionClonerBatchSize;  // (R)
 
-    TenantCollectionClonerStage _countStage;                          // (R)
-    TenantCollectionClonerStage _checkIfDonorCollectionIsEmptyStage;  // (R)
-    TenantCollectionClonerStage _listIndexesStage;                    // (R)
-    TenantCollectionClonerStage _createCollectionStage;               // (R)
-    TenantCollectionClonerQueryStage _queryStage;                     // (R)
+    TenantCollectionClonerStage _countStage;                               // (R)
+    TenantCollectionClonerQueryStage _checkIfDonorCollectionIsEmptyStage;  // (R)
+    TenantCollectionClonerStage _listIndexesStage;                         // (R)
+    TenantCollectionClonerStage _createCollectionStage;                    // (R)
+    TenantCollectionClonerQueryStage _queryStage;                          // (R)
 
     ProgressMeter _progressMeter;           // (X) progress meter for this instance.
     std::vector<BSONObj> _readyIndexSpecs;  // (X) Except for _id_
diff --git a/src/mongo/db/repl/tenant_collection_cloner_test.cpp b/src/mongo/db/repl/tenant_collection_cloner_test.cpp
index fef84327578..3e14fb3ff1a 100644
--- a/src/mongo/db/repl/tenant_collection_cloner_test.cpp
+++ b/src/mongo/db/repl/tenant_collection_cloner_test.cpp
@@ -730,6 +730,53 @@ TEST_F(TenantCollectionClonerTest, QueryStageNamespaceNotFoundOnSubsequentBatch)
 
 // We receive a QueryPlanKilled error, then a NamespaceNotFound error, indicating that the
 // collection no longer exists in the database.
+TEST_F(TenantCollectionClonerTest, QueryPlanKilledCheckIfDonorCollectionIsEmptyStage) {
+    // Set up data for preliminary stages.
+    _mockServer->setCommandReply("count", createCountResponse(3));
+
+    // Set up failpoints.
+    auto beforeStageFailPoint = globalFailPointRegistry().find("hangBeforeClonerStage");
+    auto timesEnteredBeforeStage = beforeStageFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantCollectionCloner', stage: 'checkIfDonorCollectionIsEmpty'}"));
+    auto beforeRetryFailPoint = globalFailPointRegistry().find("hangBeforeRetryingClonerStage");
+    auto timesEnteredBeforeRetry = beforeRetryFailPoint->setMode(
+        FailPoint::alwaysOn,
+        0,
+        fromjson("{cloner: 'TenantCollectionCloner', stage: 'checkIfDonorCollectionIsEmpty'}"));
+
+    auto cloner = makeCollectionCloner();
+    cloner->setBatchSize_forTest(2);
+
+    stdx::thread clonerThread([&] {
+        Client::initThread("ClonerRunner");
+        ASSERT_OK(cloner->run());
+        ASSERT_EQ(0, cloner->getStats().documentsCopied);
+    });
+
+    // Wait until we get to the 'checkIfDonorCollectionIsEmpty' stage.
+    beforeStageFailPoint->waitForTimesEntered(timesEnteredBeforeStage + 1);
+
+    // Despite the name, this will also trigger on the initial batch.
+    auto failNextBatch = globalFailPointRegistry().find("mockCursorThrowErrorOnGetMore");
+    failNextBatch->setMode(FailPoint::nTimes, 1, fromjson("{errorType: 'QueryPlanKilled'}"));
+
+    // Proceed with the 'checkIfDonorCollectionIsEmpty' stage.
+    beforeStageFailPoint->setMode(FailPoint::off, 0);
+    beforeRetryFailPoint->waitForTimesEntered(timesEnteredBeforeRetry + 1);
+
+    // Follow-up the QueryPlanKilled error with a NamespaceNotFound.
+    failNextBatch->setMode(FailPoint::nTimes, 1, fromjson("{errorType: 'NamespaceNotFound'}"));
+
+    beforeRetryFailPoint->setMode(FailPoint::off, 0);
+    clonerThread.join();
+
+    ASSERT_EQUALS(0, _opObserver->numDocsInserted);
+}
+
+// We receive a QueryPlanKilled error, then a NamespaceNotFound error, indicating that the
+// collection no longer exists in the database.
 TEST_F(TenantCollectionClonerTest, QueryPlanKilledThenNamespaceNotFoundFirstBatch) {
     // Set up data for preliminary stages.
     _mockServer->setCommandReply("count", createCountResponse(3));