SERVER-50572 Make initial sync clear and recover a tenant migration donor's in-memory state

3 files changed, 178 insertions, 14 deletions

diff --git a/jstests/replsets/libs/tenant_migration_util.js b/jstests/replsets/libs/tenant_migration_util.js
index a2313a56041..4adf50028bb 100644
--- a/jstests/replsets/libs/tenant_migration_util.js
+++ b/jstests/replsets/libs/tenant_migration_util.js
@@ -142,6 +142,14 @@ var TenantMigrationUtil = (function() {
         });
     }
 
+    /**
+     * Returns the TenantMigrationAccessBlocker associated with given the database prefix on the
+     * node.
+     */
+    function getTenantMigrationAccessBlocker(node, dbPrefix) {
+        return node.adminCommand({serverStatus: 1}).tenantMigrationAccessBlocker[dbPrefix];
+    }
+
     return {
         accessState,
         startMigration,
@@ -150,6 +158,7 @@ var TenantMigrationUtil = (function() {
         forgetMigrationRetryOnNotPrimaryErrors,
         assertMigrationCommitted,
         waitForMigrationToCommit,
-        waitForMigrationGarbageCollection
+        waitForMigrationGarbageCollection,
+        getTenantMigrationAccessBlocker
     };
 })();
diff --git a/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js b/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js
new file mode 100644
index 00000000000..4f23e4ef9b0
--- /dev/null
+++ b/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js
@@ -0,0 +1,131 @@
+/**
+ * Tests initial sync's recovery to a tenant migration's in-memory state.
+ *
+ * Tenant migrations are not expected to be run on servers with ephemeralForTest.
+ *
+ * @tags: [requires_fcv_47, requires_majority_read_concern, requires_persistence,
+ * incompatible_with_eft]
+ */
+
+(function() {
+"use strict";
+
+load("jstests/libs/fail_point_util.js");
+load("jstests/libs/parallelTester.js");
+load("jstests/replsets/libs/tenant_migration_util.js");
+
+const donorRst = new ReplSetTest(
+    {nodes: 1, name: 'donor', nodeOptions: {setParameter: {enableTenantMigrations: true}}});
+const recipientRst = new ReplSetTest(
+    {nodes: 1, name: 'recipient', nodeOptions: {setParameter: {enableTenantMigrations: true}}});
+
+donorRst.startSet();
+donorRst.initiate();
+
+recipientRst.startSet();
+recipientRst.initiate();
+
+const kMaxSleepTimeMS = 1000;
+const kDBPrefix = 'testDb';
+const kConfigDonorsNS = "config.tenantMigrationDonors";
+
+let donorPrimary = donorRst.getPrimary();
+let kRecipientConnString = recipientRst.getURL();
+
+function startMigration(host, recipientConnString, dbPrefix) {
+    const primary = new Mongo(host);
+    assert.commandWorked(primary.adminCommand({
+        donorStartMigration: 1,
+        migrationId: UUID(),
+        recipientConnectionString: recipientConnString,
+        databasePrefix: dbPrefix,
+        readPreference: {mode: "primary"}
+    }));
+}
+
+let migrationThread =
+    new Thread(startMigration, donorPrimary.host, kRecipientConnString, kDBPrefix);
+
+// Force the migration to pause after entering a randomly selected state to simulate a failure.
+Random.setRandomSeed();
+const kMigrationFpNames = [
+    "pauseTenantMigrationAfterDataSync",
+    "pauseTenantMigrationAfterBlockingStarts",
+    "abortTenantMigrationAfterBlockingStarts"
+];
+const index = Random.randInt(kMigrationFpNames.length + 1);
+if (index < kMigrationFpNames.length) {
+    configureFailPoint(donorPrimary, kMigrationFpNames[index]);
+}
+
+migrationThread.start();
+sleep(Math.random() * kMaxSleepTimeMS);
+
+// Add the initial sync node and make sure that it does not step up.
+var initialSyncNode =
+    donorRst.add({rsConfig: {priority: 0, votes: 0}, setParameter: {enableTenantMigrations: true}});
+
+donorRst.reInitiate();
+jsTestLog("Waiting for initial sync to finish.");
+donorRst.awaitSecondaryNodes();
+
+let configDonorsColl = initialSyncNode.getCollection(kConfigDonorsNS);
+let donorDoc = configDonorsColl.findOne({databasePrefix: kDBPrefix});
+if (donorDoc) {
+    let state = donorDoc.state;
+    switch (state) {
+        case "data sync":
+            assert.soon(() => TenantMigrationUtil
+                                  .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                  .access == TenantMigrationUtil.accessState.kAllow);
+            break;
+        case "blocking":
+            assert.soon(
+                () =>
+                    TenantMigrationUtil.getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                        .access == TenantMigrationUtil.accessState.kBlockingReadsAndWrites);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
+            break;
+        case "committed":
+            assert.soon(() => TenantMigrationUtil
+                                  .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                  .access == TenantMigrationUtil.accessState.kReject);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                        .commitOrAbortOpTime,
+                                    donorDoc.commitOrAbortOpTime) == 0);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
+            break;
+        case "aborted":
+            assert.soon(() => TenantMigrationUtil
+                                  .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                  .access == TenantMigrationUtil.accessState.kAllow);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                        .commitOrAbortOpTime,
+                                    donorDoc.commitOrAbortOpTime) == 0);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(initialSyncNode, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
+            break;
+        default:
+            throw new Error(`Invalid state "${state}" from donor doc.`);
+    }
+}
+
+migrationThread.join();
+donorRst.stopSet();
+recipientRst.stopSet();
+})();
diff --git a/jstests/replsets/tenant_migration_donor_startup_recovery.js b/jstests/replsets/tenant_migration_donor_startup_recovery.js
index 12fc5c8e5ce..9527570ab13 100644
--- a/jstests/replsets/tenant_migration_donor_startup_recovery.js
+++ b/jstests/replsets/tenant_migration_donor_startup_recovery.js
@@ -13,6 +13,7 @@
 
 load("jstests/libs/fail_point_util.js");
 load("jstests/libs/parallelTester.js");
+load("jstests/replsets/libs/tenant_migration_util.js");
 
 // An object that mirrors the access states for the TenantMigrationAccessBlocker.
 const accessState = {
@@ -93,40 +94,63 @@ donorRst.startSet({
         "failpoint.PrimaryOnlyServiceSkipRebuildingInstances": "{'mode':'alwaysOn'}"
     }
 });
-donorPrimary = donorRst.getPrimary();
 
+donorPrimary = donorRst.getPrimary();
 configDonorsColl = donorPrimary.getCollection(kConfigDonorsNS);
 let donorDoc = configDonorsColl.findOne({databasePrefix: kDBPrefix});
-let mtab = donorPrimary.adminCommand({serverStatus: 1}).tenantMigrationAccessBlocker;
 if (donorDoc) {
     let state = donorDoc.state;
     switch (state) {
         case "data sync":
-            assert.soon(() => mtab[kDBPrefix].access == accessState.kAllow);
+            assert.soon(
+                () => TenantMigrationUtil.getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                          .access == TenantMigrationUtil.accessState.kAllow);
             break;
         case "blocking":
-            assert.soon(() => mtab[kDBPrefix].access == accessState.kBlockingReadsAndWrites);
             assert.soon(
-                () => bsonWoCompare(mtab[kDBPrefix].blockTimestamp, donorDoc.blockTimestamp) == 0);
+                () => TenantMigrationUtil.getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                          .access == TenantMigrationUtil.accessState.kBlockingReadsAndWrites);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
             break;
         case "committed":
-            assert.soon(() => mtab[kDBPrefix].access == accessState.kReject);
-            assert.soon(() => bsonWoCompare(mtab[kDBPrefix].commitOrAbortOpTime,
-                                            donorDoc.commitOrAbortOpTime) == 0);
             assert.soon(
-                () => bsonWoCompare(mtab[kDBPrefix].blockTimestamp, donorDoc.blockTimestamp) == 0);
+                () => TenantMigrationUtil.getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                          .access == TenantMigrationUtil.accessState.kReject);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                                        .commitOrAbortOpTime,
+                                    donorDoc.commitOrAbortOpTime) == 0);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
             break;
         case "aborted":
-            assert.soon(() => mtab[kDBPrefix].access == accessState.kAllow);
-            assert.soon(() => bsonWoCompare(mtab[kDBPrefix].commitOrAbortOpTime,
-                                            donorDoc.commitOrAbortOpTime) == 0);
             assert.soon(
-                () => bsonWoCompare(mtab[kDBPrefix].blockTimestamp, donorDoc.blockTimestamp) == 0);
+                () => TenantMigrationUtil.getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                          .access == TenantMigrationUtil.accessState.kAllow);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                                        .commitOrAbortOpTime,
+                                    donorDoc.commitOrAbortOpTime) == 0);
+            assert.soon(
+                () => bsonWoCompare(TenantMigrationUtil
+                                        .getTenantMigrationAccessBlocker(donorPrimary, kDBPrefix)
+                                        .blockTimestamp,
+                                    donorDoc.blockTimestamp) == 0);
             break;
         default:
             throw new Error(`Invalid state "${state}" from donor doc.`);
     }
 }
+
 migrationThread.join();
 donorRst.stopSet();
 recipientRst.stopSet();