SERVER-65315 Enforce mutual exclusion between serverless operations

5 files changed, 52 insertions, 4 deletions

diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js
index 04619679e33..bd79c51b6bb 100644
--- a/jstests/replsets/libs/tenant_migration_test.js
+++ b/jstests/replsets/libs/tenant_migration_test.js
@@ -136,7 +136,7 @@ function TenantMigrationTest({
         nodeOptions["setParameter"] = setParameterOpts;
 
         const rstName = `${name}_${(isDonor ? "donor" : "recipient")}`;
-        const rst = new ReplSetTest({name: rstName, nodes, nodeOptions});
+        const rst = new ReplSetTest({name: rstName, nodes, serverless: true, nodeOptions});
         rst.startSet();
         if (initiateRstWithHighElectionTimeout) {
             rst.initiateWithHighElectionTimeout();
@@ -231,6 +231,7 @@ function TenantMigrationTest({
     this.runDonorStartMigration = function({
         migrationIdString,
         tenantId,
+        protocol,
         recipientConnectionString = recipientRst.getURL(),
         readPreference = {mode: "primary"},
         donorCertificateForRecipient = migrationCertificates.donorCertificateForRecipient,
@@ -251,6 +252,7 @@ function TenantMigrationTest({
             readPreference,
             donorCertificateForRecipient,
             recipientCertificateForDonor,
+            protocol
         };
 
         const stateRes = TenantMigrationUtil.runTenantMigrationCommand(cmdObj, this.getDonorRst(), {
diff --git a/jstests/replsets/libs/tenant_migration_util.js b/jstests/replsets/libs/tenant_migration_util.js
index d0e9354f1c1..bfad63c2cd0 100644
--- a/jstests/replsets/libs/tenant_migration_util.js
+++ b/jstests/replsets/libs/tenant_migration_util.js
@@ -262,6 +262,19 @@ var TenantMigrationUtil = (function() {
         return res;
     }
 
+    const ServerlessLockType = {
+        ShardSplitDonor: 'shard split donor',
+        TenantMigrationDonor: 'tenant migration donor',
+        TenantMigrationRecipient: 'tenant migration recipient'
+    };
+
+    /**
+     * Return the active serverless operation lock, if one is acquired.
+     */
+    function getServerlessOperationLock(node) {
+        return assert.commandWorked(node.adminCommand({serverStatus: 1})).serverless.operationLock;
+    }
+
     /**
      * Returns the TenantMigrationAccessBlocker serverStatus output for the multi-tenant migration
      * or shard merge for the given node.
@@ -561,6 +574,8 @@ var TenantMigrationUtil = (function() {
         makeMigrationCertificatesForTest,
         makeX509OptionsForTest,
         isMigrationCompleted,
+        ServerlessLockType,
+        getServerlessOperationLock,
         getTenantMigrationAccessBlocker,
         getTenantMigrationAccessBlockers,
         getNumBlockedReads,
diff --git a/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js b/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js
index 85429af3ddd..8209ee570e2 100644
--- a/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js
+++ b/jstests/replsets/tenant_migration_donor_initial_sync_recovery.js
@@ -5,6 +5,7 @@
  * @tags: [
  *   incompatible_with_macos,
  *   incompatible_with_windows_tls,
+ *   requires_fcv_62,
  *   requires_majority_read_concern,
  *   requires_persistence,
  *   serverless,
@@ -19,6 +20,7 @@ load("jstests/libs/uuid_util.js");
 load("jstests/libs/parallelTester.js");
 load("jstests/libs/write_concern_util.js");
 load("jstests/replsets/libs/tenant_migration_test.js");
+const {ServerlessLockType, getServerlessOperationLock} = TenantMigrationUtil;
 
 const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
 
@@ -47,6 +49,7 @@ const migrationOpts = {
     migrationIdString: extractUUIDFromObject(UUID()),
     tenantId: kTenantId
 };
+
 assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
 // We must wait for the migration to have finished replicating the recipient keys on the donor set
 // before starting initial sync, otherwise the migration will hang while waiting for initial sync to
@@ -83,7 +86,8 @@ donorRst.awaitReplication();
 stopServerReplication(initialSyncNode);
 
 let configDonorsColl = initialSyncNode.getCollection(TenantMigrationTest.kConfigDonorsNS);
-let donorDoc = configDonorsColl.findOne({tenantId: kTenantId});
+assert.lte(configDonorsColl.count(), 1);
+let donorDoc = configDonorsColl.findOne();
 if (donorDoc) {
     jsTestLog("Initial sync completed while migration was in state: " + donorDoc.state);
     switch (donorDoc.state) {
@@ -148,6 +152,13 @@ if (donorDoc) {
     }
 }
 
+const activeServerlessLock = getServerlessOperationLock(initialSyncNode);
+if (donorDoc && !donorDoc.expireAt) {
+    assert.eq(activeServerlessLock, ServerlessLockType.TenantMigrationDonor);
+} else {
+    assert.eq(activeServerlessLock, null);
+}
+
 if (fp) {
     fp.off();
 }
diff --git a/jstests/replsets/tenant_migration_donor_startup_recovery.js b/jstests/replsets/tenant_migration_donor_startup_recovery.js
index 7a564e416a5..f10c1c90d45 100644
--- a/jstests/replsets/tenant_migration_donor_startup_recovery.js
+++ b/jstests/replsets/tenant_migration_donor_startup_recovery.js
@@ -8,6 +8,7 @@
  *   incompatible_with_macos,
  *   incompatible_with_shard_merge,
  *   incompatible_with_windows_tls,
+ *   requires_fcv_62,
  *   requires_majority_read_concern,
  *   requires_persistence,
  *   serverless,
@@ -20,6 +21,7 @@
 load("jstests/libs/fail_point_util.js");
 load("jstests/libs/uuid_util.js");
 load("jstests/replsets/libs/tenant_migration_test.js");
+const {ServerlessLockType, getServerlessOperationLock} = TenantMigrationUtil;
 
 const donorRst = new ReplSetTest({
     nodes: 1,
@@ -70,7 +72,8 @@ donorRst.startSet({
 
 donorPrimary = donorRst.getPrimary();
 const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
-const donorDoc = configDonorsColl.findOne({tenantId: kTenantId});
+assert.lte(configDonorsColl.count(), 1);
+const donorDoc = configDonorsColl.findOne();
 if (donorDoc) {
     switch (donorDoc.state) {
         case TenantMigrationTest.DonorState.kAbortingIndexBuilds:
@@ -133,6 +136,13 @@ if (donorDoc) {
     }
 }
 
+const activeServerlessLock = getServerlessOperationLock(donorPrimary);
+if (donorDoc && !donorDoc.expireAt) {
+    assert.eq(activeServerlessLock, ServerlessLockType.TenantMigrationDonor);
+} else {
+    assert.eq(activeServerlessLock, null);
+}
+
 tenantMigrationTest.stop();
 donorRst.stopSet();
 })();
diff --git a/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js b/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js
index 404bf0fa765..a0fc1bfacb3 100644
--- a/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js
+++ b/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js
@@ -6,6 +6,7 @@
  *   incompatible_with_macos,
  *   incompatible_with_shard_merge,
  *   incompatible_with_windows_tls,
+ *   requires_fcv_62,
  *   requires_majority_read_concern,
  *   requires_persistence,
  *   serverless,
@@ -19,6 +20,7 @@ load("jstests/libs/fail_point_util.js");
 load("jstests/libs/uuid_util.js");
 load("jstests/libs/write_concern_util.js");
 load("jstests/replsets/libs/tenant_migration_test.js");
+const {ServerlessLockType, getServerlessOperationLock} = TenantMigrationUtil;
 
 const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
 
@@ -59,7 +61,8 @@ recipientRst.awaitReplication();
 stopServerReplication(initialSyncNode);
 
 const configRecipientsColl = initialSyncNode.getCollection(TenantMigrationTest.kConfigRecipientsNS);
-const recipientDoc = configRecipientsColl.findOne({tenantId: kTenantId});
+assert.lte(configRecipientsColl.count(), 1);
+const recipientDoc = configRecipientsColl.findOne();
 if (recipientDoc) {
     switch (recipientDoc.state) {
         case TenantMigrationTest.RecipientState.kStarted:
@@ -97,6 +100,13 @@ if (recipientDoc) {
     }
 }
 
+const activeServerlessLock = getServerlessOperationLock(initialSyncNode);
+if (recipientDoc && !recipientDoc.expireAt) {
+    assert.eq(activeServerlessLock, ServerlessLockType.TenantMigrationRecipient);
+} else {
+    assert.eq(activeServerlessLock, null);
+}
+
 restartServerReplication(initialSyncNode);
 
 tenantMigrationTest.stop();