SERVER-63397 Abort all index builds during shard merge

12 files changed, 206 insertions, 11 deletions

diff --git a/jstests/libs/fail_point_util.js b/jstests/libs/fail_point_util.js
index c01c1d593e0..7572a753a81 100644
--- a/jstests/libs/fail_point_util.js
+++ b/jstests/libs/fail_point_util.js
@@ -23,7 +23,7 @@ configureFailPoint = function(conn, failPointName, data = {}, failPointMode = "a
                               {configureFailPoint: failPointName, mode: failPointMode, data: data}))
                           .count,
         wait:
-            function(maxTimeMS = kDefaultWaitForFailPointTimeout, timesEntered = 1) {
+            function({maxTimeMS = kDefaultWaitForFailPointTimeout, timesEntered = 1} = {}) {
                 // Can only be called once because this function does not keep track of the
                 // number of times the fail point is entered between the time it returns
                 // and the next time it gets called.
diff --git a/jstests/multiVersion/genericSetFCVUsage/tenant_migration_save_fcv.js b/jstests/multiVersion/genericSetFCVUsage/tenant_migration_save_fcv.js
index 27a6794f47c..9a71b954785 100644
--- a/jstests/multiVersion/genericSetFCVUsage/tenant_migration_save_fcv.js
+++ b/jstests/multiVersion/genericSetFCVUsage/tenant_migration_save_fcv.js
@@ -56,8 +56,7 @@ function runTest(downgradeFCV) {
     migrationThread.start();
     hangAfterSavingFCV.wait();
 
-    const isRunningMergeProtocol =
-        (TenantMigrationUtil.isShardMergeEnabled(recipientDb)) ? true : false;
+    const isRunningMergeProtocol = TenantMigrationUtil.isShardMergeEnabled(recipientDb);
 
     // Downgrade the FCV for the recipient set.
     assert.commandWorked(
diff --git a/jstests/noPassthrough/change_stream_pre_image_time_based_expiration.js b/jstests/noPassthrough/change_stream_pre_image_time_based_expiration.js
index ca6a0adf865..6c4720d02fe 100644
--- a/jstests/noPassthrough/change_stream_pre_image_time_based_expiration.js
+++ b/jstests/noPassthrough/change_stream_pre_image_time_based_expiration.js
@@ -155,7 +155,7 @@ function testTimeBasedPreImageRetentionPolicy(conn, primary) {
                            {currentTimeForTimeBasedExpiration: currentTime});
 
     // Wait until at least 1 complete cycle of pre-image removal job is completed.
-    currentTimeFailPoint.wait(kDefaultWaitForFailPointTimeout, 2);
+    currentTimeFailPoint.wait({timesEntered: 2});
 
     // Verify that when time-based pre-image expiration disabled, no pre-images are not deleted.
     verifyPreImages(preImageColl, allDocs, collectionsInfo);
diff --git a/jstests/replsets/drop_collections_two_phase_create_index.js b/jstests/replsets/drop_collections_two_phase_create_index.js
index 858f268a76b..8dc47ead7b4 100644
--- a/jstests/replsets/drop_collections_two_phase_create_index.js
+++ b/jstests/replsets/drop_collections_two_phase_create_index.js
@@ -47,7 +47,7 @@ const createIndexesFailPoint =
 const createIdx = IndexBuildTest.startIndexBuild(primary, coll.getFullName(), {a: 1});
 
 try {
-    createIndexesFailPoint.wait(30 * 1000);
+    createIndexesFailPoint.wait({maxTimeMS: 30 * 1000});
 
     // PREPARE collection drop.
     twoPhaseDropTest.prepareDropCollection(collName);
diff --git a/jstests/replsets/libs/tenant_migration_util.js b/jstests/replsets/libs/tenant_migration_util.js
index 4bfe9d5385c..180786b395b 100644
--- a/jstests/replsets/libs/tenant_migration_util.js
+++ b/jstests/replsets/libs/tenant_migration_util.js
@@ -209,6 +209,8 @@ var TenantMigrationUtil = (function() {
                         primary = rst.getPrimary();
                         return false;
                     }
+                    const cmdName = Object.keys(localCmdObj)[0];
+                    jsTestLog(`Error from ${cmdName}: ${tojson(res)}`);
                     return true;
                 }
 
diff --git a/jstests/replsets/tenant_migration_buildindex.js b/jstests/replsets/tenant_migration_buildindex.js
index 324ee6f974d..6719ee58c44 100644
--- a/jstests/replsets/tenant_migration_buildindex.js
+++ b/jstests/replsets/tenant_migration_buildindex.js
@@ -21,6 +21,14 @@ load("jstests/replsets/libs/tenant_migration_test.js");
 load("jstests/replsets/libs/tenant_migration_util.js");
 
 const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+// TODO (SERVER-63517): This test assumes the donor blocks only some tenants. Replace this test with
+// tenant_migration_buildindex_shard_merge.js.
+if (TenantMigrationUtil.isShardMergeEnabled(
+        tenantMigrationTest.getDonorPrimary().getDB("adminDB"))) {
+    jsTestLog("Skip: incompatible with featureFlagShardMerge");
+    tenantMigrationTest.stop();
+    return;
+}
 
 const kTenantId = "testTenantId1";
 const kUnrelatedTenantId = "testTenantId2";
diff --git a/jstests/replsets/tenant_migration_buildindex_shard_merge.js b/jstests/replsets/tenant_migration_buildindex_shard_merge.js
new file mode 100644
index 00000000000..16536d37a2b
--- /dev/null
+++ b/jstests/replsets/tenant_migration_buildindex_shard_merge.js
@@ -0,0 +1,177 @@
+/**
+ * Tests that index building is properly blocked and/or aborted during migrations.
+ *
+ * TODO (SERVER-63517): Replace tenant_migration_buildindex.js with this test.
+ *
+ * @tags: [
+ *   incompatible_with_eft,
+ *   incompatible_with_macos,
+ *   incompatible_with_windows_tls,
+ *   requires_majority_read_concern,
+ *   requires_persistence,
+ *   serverless,
+ *   featureFlagShardMerge,
+ *   requires_fcv_53,
+ * ]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/parallelTester.js");
+load("jstests/libs/uuid_util.js");
+load("jstests/replsets/libs/tenant_migration_test.js");
+load("jstests/replsets/libs/tenant_migration_util.js");
+
+// Index builds should be blocked by the tenant access blocker, not maxNumActiveUserIndexBuilds.
+const tenantMigrationTest = new TenantMigrationTest(
+    {name: jsTestName(), sharedOptions: {setParameter: {maxNumActiveUserIndexBuilds: 100}}});
+const donorPrimary = tenantMigrationTest.getDonorPrimary();
+const kTenant1Id = "testTenantId1";
+const kTenant2Id = "testTenantId2";
+const kTenant1DbName = tenantMigrationTest.tenantDB(kTenant1Id, "testDB");
+const kTenant2DbName = tenantMigrationTest.tenantDB(kTenant2Id, "testDB");
+const kEmptyCollName = "testEmptyColl";
+const kNonEmptyCollName = "testNonEmptyColl";
+const kNewCollName1 = "testNewColl1";
+const kNewCollName2 = "testNewColl2";
+
+// Attempts to create an index on a collection and checks that it fails because a migration
+// committed.
+function createIndexShouldFail(primaryHost, dbName, collName, indexSpec) {
+    const donorPrimary = new Mongo(primaryHost);
+    const db = donorPrimary.getDB(dbName);
+    const res = db[collName].createIndex(indexSpec);
+    jsTestLog(`createIndex ${dbName}.${collName} spec: ${tojson(indexSpec)} reply: ${tojson(res)}`);
+    assert.commandFailedWithCode(
+        res, ErrorCodes.TenantMigrationCommitted, `${dbName.collName} ${tojson(indexSpec)}`);
+}
+
+const migrationId = UUID();
+// TODO (SERVER-63454): remove tenantId, and remove kTenant2DbName, db2, tenant2IndexThread, etc.
+const migrationOpts = {
+    migrationIdString: extractUUIDFromObject(migrationId),
+    recipientConnString: tenantMigrationTest.getRecipientConnString(),
+    tenantId: kTenant1Id,
+};
+const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst());
+
+// Put some data in the non-empty collections, and create the empty one.
+const db1 = donorPrimary.getDB(kTenant1DbName);
+const db2 = donorPrimary.getDB(kTenant2DbName);
+assert.commandWorked(db1[kNonEmptyCollName].insert([{a: 1, b: 1}, {a: 2, b: 2}, {a: 3, b: 3}]));
+assert.commandWorked(db2[kNonEmptyCollName].insert([{x: 1, y: 1}, {x: 2, b: 2}, {x: 3, y: 3}]));
+assert.commandWorked(db1.createCollection(kEmptyCollName));
+assert.commandWorked(db2.createCollection(kEmptyCollName));
+
+// Start index builds and have them hang in the builder thread.
+var initFpCount =
+    assert
+        .commandWorked(donorPrimary.adminCommand(
+            {configureFailPoint: "hangAfterInitializingIndexBuild", mode: "alwaysOn"}))
+        .count;
+const tenant1IndexThread =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant1DbName, kNonEmptyCollName, {b: 1});
+// Even though tenantId1 is passed to donorStartMigration, the donor aborts this index too
+// because protocol is "shard merge".
+// TODO (SERVER-63454): remove comment above.
+const tenant2IndexThread =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant2DbName, kNonEmptyCollName, {y: 1});
+tenant1IndexThread.start();
+tenant2IndexThread.start();
+assert.commandWorked(donorPrimary.adminCommand({
+    waitForFailPoint: "hangAfterInitializingIndexBuild",
+    timesEntered: initFpCount + 2,
+    maxTimeMS: kDefaultWaitForFailPointTimeout
+}));
+
+// Start an index build and pause it after acquiring a slot but before registering itself.
+const indexBuildSlotFp = configureFailPoint(donorPrimary, "hangAfterAcquiringIndexBuildSlot");
+jsTestLog("Starting the racy index build");
+const racyIndexThread1 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant1DbName, kNonEmptyCollName, {a: 1});
+const racyIndexThread2 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant2DbName, kNonEmptyCollName, {a: 1});
+racyIndexThread1.start();
+racyIndexThread2.start();
+indexBuildSlotFp.wait({timesEntered: 2});
+
+jsTestLog("Starting a migration and pausing after majority-committing the initial state doc.");
+// Start a migration, and pause it after the donor has majority-committed the initial state doc.
+const dataSyncFp =
+    configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingDataSyncState");
+const migrationThread =
+    new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs);
+migrationThread.start();
+dataSyncFp.wait();
+
+// Release the previously-started index build thread and allow the donor to abort index builds
+assert.commandWorked(donorPrimary.adminCommand(
+    {configureFailPoint: "hangAfterInitializingIndexBuild", mode: "off"}));
+
+// Release the racy thread; it should block.
+indexBuildSlotFp.off();
+
+// Should be able to create an index on a non-existent collection.  Since the collection is
+// guaranteed to be empty and to have always been empty, this is safe.
+assert.commandWorked(db1[kNewCollName1].createIndex({a: 1}));
+
+// Attempts to create indexes on existing collections should fail.
+const emptyIndexThread1 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant1DbName, kEmptyCollName, {a: 1});
+const emptyIndexThread2 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant2DbName, kEmptyCollName, {a: 1});
+emptyIndexThread1.start();
+emptyIndexThread2.start();
+const nonEmptyIndexThread1 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant1DbName, kNonEmptyCollName, {c: 1});
+const nonEmptyIndexThread2 =
+    new Thread(createIndexShouldFail, donorPrimary.host, kTenant2DbName, kNonEmptyCollName, {c: 1});
+nonEmptyIndexThread1.start();
+nonEmptyIndexThread2.start();
+
+jsTestLog("Allowing migration to commit");
+// Allow the migration to move to the blocking state and commit.
+dataSyncFp.off();
+assert.soon(() => {
+    const state =
+        tenantMigrationTest
+            .getTenantMigrationAccessBlocker({donorNode: donorPrimary, tenantId: kTenant1Id})
+            .donor.state;
+    return state === TenantMigrationTest.DonorAccessState.kBlockWritesAndReads ||
+        state === TenantMigrationTest.DonorAccessState.kReject;
+});
+TenantMigrationTest.assertCommitted(migrationThread.returnData());
+
+// The index creation threads should be done.
+racyIndexThread1.join();
+// TODO: remove, search everywhere
+racyIndexThread2.join();
+tenant1IndexThread.join();
+tenant2IndexThread.join();
+emptyIndexThread1.join();
+emptyIndexThread2.join();
+nonEmptyIndexThread1.join();
+nonEmptyIndexThread2.join();
+
+// Should not be able to create an index on any collection.
+assert.commandFailedWithCode(db1[kEmptyCollName].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+assert.commandFailedWithCode(db2[kEmptyCollName].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+assert.commandFailedWithCode(db1[kNonEmptyCollName].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+assert.commandFailedWithCode(db2[kNonEmptyCollName].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+// Creating an index on a non-existent collection should fail because we can't create the
+// collection, but it's the same error code.
+assert.commandFailedWithCode(db1[kNewCollName2].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+assert.commandFailedWithCode(db2[kNewCollName2].createIndex({d: 1}),
+                             ErrorCodes.TenantMigrationCommitted);
+
+assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+
+tenantMigrationTest.stop();
+})();
diff --git a/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js b/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js
index 61cf1f9db03..bbb3640c806 100644
--- a/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js
+++ b/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js
@@ -29,7 +29,7 @@ function assertCanFindWithReadConcern(conn, dbName, collName, expectedDoc, readC
 
 let counter = 0;
 let makeTenantId = function() {
-    return "tenant_" + counter++;
+    return "tenant-" + counter++;
 };
 
 // Local read concern case.
diff --git a/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js b/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js
index d1b3df133ff..731eb2a5ebe 100644
--- a/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js
+++ b/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js
@@ -32,6 +32,14 @@ const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
 const donorRst = tenantMigrationTest.getDonorRst();
 const donorTestColl = donorPrimary.getDB(dbName).getCollection(collName);
 
+// TODO (SERVER-63517): Remove this test, it requires failover, and the ability to write to the
+// donor during migration, and it tests a cloning method superseded by Shard Merge.
+if (TenantMigrationUtil.isShardMergeEnabled(donorRst.getPrimary().getDB("adminDB"))) {
+    jsTestLog("Skip: featureFlagShardMerge enabled, but shard merge does not survive failover");
+    tenantMigrationTest.stop();
+    return;
+}
+
 // The default WC is majority and stopReplicationOnSecondaries will prevent satisfying any majority
 assert.commandWorked(recipientPrimary.adminCommand(
     {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}}));
diff --git a/jstests/replsets/tenant_migration_collection_ttl.js b/jstests/replsets/tenant_migration_collection_ttl.js
index 495451e3bbc..e76196b4fa4 100644
--- a/jstests/replsets/tenant_migration_collection_ttl.js
+++ b/jstests/replsets/tenant_migration_collection_ttl.js
@@ -99,7 +99,7 @@ function assertTTLDeleteExpiredDocs(dbName, node) {
 (() => {
     jsTest.log("Test that the TTL does not delete documents on recipient during cloning");
 
-    const tenantId = "testTenantId_duringCloning";
+    const tenantId = "testTenantId-duringCloning";
     const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB");
 
     const migrationId = UUID();
@@ -160,7 +160,7 @@ function assertTTLDeleteExpiredDocs(dbName, node) {
 (() => {
     jsTest.log("Test that the TTL does not delete documents on recipient after cloning");
 
-    const tenantId = "testTenantId_afterCloning";
+    const tenantId = "testTenantId-afterCloning";
     const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB");
 
     const migrationId = UUID();
diff --git a/jstests/replsets/tenant_migration_external_keys_ttl.js b/jstests/replsets/tenant_migration_external_keys_ttl.js
index ffc313329fd..145296badc1 100644
--- a/jstests/replsets/tenant_migration_external_keys_ttl.js
+++ b/jstests/replsets/tenant_migration_external_keys_ttl.js
@@ -31,7 +31,7 @@ const ttlMonitorOptions = {
 
 let counter = 0;
 let makeTenantId = function() {
-    return kTenantIdPrefix + "_" + counter++;
+    return kTenantIdPrefix + "-" + counter++;
 };
 
 function waitForExternalKeysTTLIndex(conn) {
diff --git a/jstests/replsets/tenant_migrations_noop_writes.js b/jstests/replsets/tenant_migrations_noop_writes.js
index 591f7f5c05a..c4f391c2add 100644
--- a/jstests/replsets/tenant_migrations_noop_writes.js
+++ b/jstests/replsets/tenant_migrations_noop_writes.js
@@ -22,13 +22,14 @@ load("jstests/replsets/libs/tenant_migration_test.js");
 load('jstests/libs/parallel_shell_helpers.js');
 
 const kTenantIdPrefix = "testTenantId";
-const kUnrelatedDbName = "unrelatedDB";
+// During "shard merge" tenant migrations, writes to internal DBs are still allowed.
+const kUnrelatedDbName = "admin";
 const collName = "foo";
 const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
 
 let counter = 0;
 let makeTenantId = function() {
-    return kTenantIdPrefix + "_" + counter++;
+    return kTenantIdPrefix + "-" + counter++;
 };
 
 function makeTestParams() {