SERVER-72199 Shard merge handles initial sync, resync and unclean restarts.

3 files changed, 111 insertions, 1 deletions

diff --git a/jstests/replsets/tenant_migration_recipient_forget_migration.js b/jstests/replsets/tenant_migration_recipient_forget_migration.js
index 145d837c63a..2d34605ab77 100644
--- a/jstests/replsets/tenant_migration_recipient_forget_migration.js
+++ b/jstests/replsets/tenant_migration_recipient_forget_migration.js
@@ -38,12 +38,17 @@ const isShardMergeEnabledOnDonorPrimary =
 const oplogBufferCollectionName = (migrationIdString) =>
     `repl.migration.oplog_${migrationIdString}`;
 const donatedFilesCollectionName = (migrationIdString) => `donatedFiles.${migrationIdString}`;
+const importMarkerCollName = (migrationIdString) => `importDoneMarker.${migrationIdString}`;
 
 const assertTempCollectionsExist = (conn, migrationIdString) => {
     const collections = conn.getDB("config").getCollectionNames();
     assert(collections.includes(oplogBufferCollectionName(migrationIdString)), collections);
     if (isShardMergeEnabledOnDonorPrimary) {
         assert(collections.includes(donatedFilesCollectionName(migrationIdString)), collections);
+        assert.eq(1,
+                  conn.getDB("local")
+                      .getCollectionInfos({name: importMarkerCollName(migrationIdString)})
+                      .length);
     }
 };
 
@@ -52,6 +57,10 @@ const assertTempCollectionsDoNotExist = (conn, migrationIdString) => {
     assert(!collections.includes(oplogBufferCollectionName(migrationIdString)), collections);
     if (isShardMergeEnabledOnDonorPrimary) {
         assert(!collections.includes(donatedFilesCollectionName(migrationIdString)), collections);
+        assert.eq(0,
+                  conn.getDB("local")
+                      .getCollectionInfos({name: importMarkerCollName(migrationIdString)})
+                      .length);
     }
 };
 
diff --git a/jstests/replsets/tenant_migration_recipient_shard_merge_import_survives_unclean_restart.js b/jstests/replsets/tenant_migration_recipient_shard_merge_import_survives_unclean_restart.js
new file mode 100644
index 00000000000..3bb2281bb90
--- /dev/null
+++ b/jstests/replsets/tenant_migration_recipient_shard_merge_import_survives_unclean_restart.js
@@ -0,0 +1,92 @@
+/**
+ * Tests that the donor data copied via shard merge protocol by recipient is present even after
+ * unclean restarts.
+ *
+ * @tags: [
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ *   requires_majority_read_concern,
+ *   requires_persistence,
+ *   serverless,
+ *   featureFlagShardMerge,
+ * ]
+ */
+
+import {TenantMigrationTest} from "jstests/replsets/libs/tenant_migration_test.js";
+import {
+    isShardMergeEnabled,
+    makeX509OptionsForTest
+} from "jstests/replsets/libs/tenant_migration_util.js";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/uuid_util.js");
+
+// Setting syncdelay to 0 will disable the checkpoint on the recipient.
+const recipientRst = new ReplSetTest({
+    nodes: 1,
+    name: "recipient",
+    serverless: true,
+    nodeOptions: Object.assign(makeX509OptionsForTest().recipient, {
+        setParameter:
+            {syncdelay: 0, tenantMigrationGarbageCollectionDelayMS: 0, ttlMonitorSleepSecs: 1}
+    })
+});
+
+recipientRst.startSet();
+recipientRst.initiate();
+
+const recipientPrimary = recipientRst.getPrimary();
+
+// Note: Including this explicit early return here due to the fact that multiversion
+// suites will execute this test without featureFlagShardMerge enabled (despite the
+// presence of the featureFlagShardMerge tag above), which means the test will attempt
+// to run a multi-tenant migration and fail.
+if (!isShardMergeEnabled(recipientPrimary.getDB("admin"))) {
+    recipientRst.stop();
+    jsTestLog("Skipping Shard Merge-specific test.");
+    quit();
+}
+
+const tenantMigrationTest = new TenantMigrationTest(
+    {name: jsTestName(), sharedOptions: {nodes: 1}, recipientRst: recipientRst});
+const donorPrimary = tenantMigrationTest.getDonorPrimary();
+
+const tenantId = ObjectId();
+const tenantDB = tenantMigrationTest.tenantDB(tenantId.str, "DB");
+const collName = "testColl";
+
+// Do a majority write.
+tenantMigrationTest.insertDonorDB(tenantDB, collName);
+
+const migrationUuid = UUID();
+const migrationOpts = {
+    migrationIdString: extractUUIDFromObject(migrationUuid),
+    readPreference: {mode: 'primary'},
+    tenantIds: [tenantId],
+};
+
+// Start migration, and then wait for the migration to get committed and garbage collected.
+assert.commandWorked(
+    tenantMigrationTest.runMigration(migrationOpts, {automaticForgetMigration: true}));
+
+jsTestLog("Restart recipient primary.");
+// Do an unclean shutdown of the recipient primary, and then restart.
+recipientRst.restart(recipientPrimary, {allowedExitCode: MongoRunner.EXIT_SIGKILL}, 9);
+// Wait for the recipient primary to get elected.
+recipientRst.getPrimary();
+
+// Verify the imported donor data is still present on recipient even though the "Checkpointer"
+// thread didn’t take a checkpoint before node unclean restart.
+recipientRst.nodes.forEach(node => {
+    jsTestLog(`Checking ${tenantDB}.${collName} on ${node}`);
+    // Use "countDocuments" to check actual docs, "count" to check sizeStorer data.
+    assert.eq(donorPrimary.getDB(tenantDB)[collName].countDocuments({}),
+              node.getDB(tenantDB)[collName].countDocuments({}),
+              "countDocuments");
+    assert.eq(donorPrimary.getDB(tenantDB)[collName].count(),
+              node.getDB(tenantDB)[collName].count(),
+              "count");
+});
+
+tenantMigrationTest.stop();
+recipientRst.stopSet();
diff --git a/jstests/replsets/tenant_migration_recipient_shard_merge_learn_files.js b/jstests/replsets/tenant_migration_recipient_shard_merge_learn_files.js
index a4b8affa7a3..b1e8ab7d6c6 100644
--- a/jstests/replsets/tenant_migration_recipient_shard_merge_learn_files.js
+++ b/jstests/replsets/tenant_migration_recipient_shard_merge_learn_files.js
@@ -22,7 +22,7 @@ const tenantMigrationTest =
 
 const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
 
-// Note: including this explicit early return here due to the fact that multiversion
+// Note: Including this explicit early return here due to the fact that multiversion
 // suites will execute this test without featureFlagShardMerge enabled (despite the
 // presence of the featureFlagShardMerge tag above), which means the test will attempt
 // to run a multi-tenant migration and fail.
@@ -59,6 +59,14 @@ assert.commandWorked(
 
 waitInFailPoint.wait();
 
+// Before transitioning to `kConsistent` state, check that all recipient nodes have
+// "importDoneMarker" collection.
+const importMarkerCollName = "importDoneMarker." + extractUUIDFromObject(migrationUuid);
+tenantMigrationTest.getRecipientRst().nodes.forEach(node => {
+    jsTestLog(`Checking if the local.${importMarkerCollName} collection exists on ${node}`);
+    assert.eq(1, node.getDB("local").getCollectionInfos({name: importMarkerCollName}).length);
+});
+
 tenantMigrationTest.assertRecipientNodesInExpectedState({
     nodes: tenantMigrationTest.getRecipientRst().nodes,
     migrationId: migrationUuid,
@@ -75,6 +83,7 @@ const donorPrimaryCountDocumentsResult = donorPrimary.getDB(tenantDB)[collName].
 const donorPrimaryCountResult = donorPrimary.getDB(tenantDB)[collName].count();
 
 tenantMigrationTest.getRecipientRst().nodes.forEach(node => {
+    jsTestLog(`Checking ${tenantDB}.${collName} on ${node}`);
     // Use "countDocuments" to check actual docs, "count" to check sizeStorer data.
     assert.eq(donorPrimaryCountDocumentsResult,
               node.getDB(tenantDB)[collName].countDocuments({}),