diff options
author | Christopher Caplinger <christopher.caplinger@mongodb.com> | 2021-12-02 18:54:45 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-12-02 19:45:48 +0000 |
commit | ac9a64634fb87fa4b065c0e83c50c7b4d1400688 (patch) | |
tree | ef303b5b02abadbc52084df05c75a1f09ee624bf | |
parent | 7f120e5dbf538ce29b637c5caa2d175a58e1afad (diff) | |
download | mongo-ac9a64634fb87fa4b065c0e83c50c7b4d1400688.tar.gz |
SERVER-59786: Prevent concurrent merges
29 files changed, 690 insertions, 422 deletions
diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js index 0c87a193aea..9cbb7523559 100644 --- a/jstests/replsets/libs/tenant_migration_test.js +++ b/jstests/replsets/libs/tenant_migration_test.js @@ -33,7 +33,8 @@ function TenantMigrationTest({ sharedOptions = {}, // Default this to true so it is easier for data consistency checks. allowStaleReadsOnDonor = true, - initiateRstWithHighElectionTimeout = true + initiateRstWithHighElectionTimeout = true, + quickGarbageCollection = false, }) { const donorPassedIn = (donorRst !== undefined); const recipientPassedIn = (recipientRst !== undefined); @@ -42,7 +43,11 @@ function TenantMigrationTest({ const migrationCertificates = TenantMigrationUtil.makeMigrationCertificatesForTest(); const nodes = sharedOptions.nodes || 2; - let setParameterOpts = sharedOptions.setParameter || {}; + const setParameterOpts = sharedOptions.setParameter || {}; + if (quickGarbageCollection) { + setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 3 * 1000; + setParameterOpts.ttlMonitorSleepSecs = 3; + } donorRst = donorPassedIn ? donorRst : performSetUp(true /* isDonor */); recipientRst = recipientPassedIn ? recipientRst : performSetUp(false /* isDonor */); diff --git a/jstests/replsets/tenant_migration_abort_forget_retry.js b/jstests/replsets/tenant_migration_abort_forget_retry.js index 823f204a79d..379fbfc4556 100644 --- a/jstests/replsets/tenant_migration_abort_forget_retry.js +++ b/jstests/replsets/tenant_migration_abort_forget_retry.js @@ -29,7 +29,8 @@ function makeTenantId() { return kTenantIdPrefix + testNum++; } -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); (() => { const migrationId1 = extractUUIDFromObject(UUID()); @@ -60,6 +61,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); migrationId2 + ", tenantId: " + tenantId); TenantMigrationTest.assertCommitted( tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); })(); (() => { @@ -111,6 +113,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); migrationId2 + ", tenantId: " + tenantId); TenantMigrationTest.assertCommitted( tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId})); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId); })(); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js index 0adcb568bcf..5e15855f214 100644 --- a/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js +++ b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js @@ -2,9 +2,12 @@ * Test that tenant migration donor and recipient correctly copy each other cluster time keys into * their config.external_validation_keys collection. * + * TODO (SERVER-61231): Adapt for shard merge. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, diff --git a/jstests/replsets/tenant_migration_collection_ttl.js b/jstests/replsets/tenant_migration_collection_ttl.js index 8047fca397f..e7bff842178 100644 --- a/jstests/replsets/tenant_migration_collection_ttl.js +++ b/jstests/replsets/tenant_migration_collection_ttl.js @@ -147,6 +147,8 @@ function assertTTLDeleteExpiredDocs(dbName, node) { assertTTLDeleteExpiredDocs(dbName, donorPrimary); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString, + migrationOpts.tenantId); })(); // Tests that: @@ -211,6 +213,9 @@ function assertTTLDeleteExpiredDocs(dbName, node) { // After the tenant migration is aborted, the TTL cleanup is restored. assertTTLDeleteExpiredDocs(dbName, recipientPrimary); assertTTLDeleteExpiredDocs(dbName, donorPrimary); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString, + migrationOpts.tenantId); })(); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_commit_transaction_retry.js b/jstests/replsets/tenant_migration_commit_transaction_retry.js index e07c12bb6ff..fba740959ea 100644 --- a/jstests/replsets/tenant_migration_commit_transaction_retry.js +++ b/jstests/replsets/tenant_migration_commit_transaction_retry.js @@ -28,7 +28,7 @@ const kGarbageCollectionParams = { }; const tenantMigrationTest = new TenantMigrationTest( - {name: jsTestName(), sharedOptions: {nodes: 1, setParameter: kGarbageCollectionParams}}); + {name: jsTestName(), sharedOptions: {nodes: 1}, quickGarbageCollection: true}); const kTenantId = "testTenantId"; const kDbName = tenantMigrationTest.tenantDB(kTenantId, "testDB"); diff --git a/jstests/replsets/tenant_migration_concurrent_bulk_writes.js b/jstests/replsets/tenant_migration_concurrent_bulk_writes.js index beda0b68c67..2390e07a5d4 100644 --- a/jstests/replsets/tenant_migration_concurrent_bulk_writes.js +++ b/jstests/replsets/tenant_migration_concurrent_bulk_writes.js @@ -37,40 +37,54 @@ const kBatchTypes = { remove: 3 }; -const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); -const donorRst = new ReplSetTest({ - nodes: 1, - name: 'donor', - nodeOptions: Object.assign(migrationX509Options.donor, { - setParameter: { - internalInsertMaxBatchSize: - kMaxBatchSize, /* Decrease internal max batch size so we can still show writes are - batched without inserting hundreds of documents. */ - // Allow non-timestamped reads on donor after migration completes for testing. - 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}), +function setup() { + const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); + const donorRst = new ReplSetTest({ + nodes: 1, + name: 'donor', + nodeOptions: Object.assign(migrationX509Options.donor, { + setParameter: { + internalInsertMaxBatchSize: + kMaxBatchSize, /* Decrease internal max batch size so we can still show writes + are batched without inserting hundreds of documents. */ + // Allow non-timestamped reads on donor after migration completes for testing. + 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': + tojson({mode: 'alwaysOn'}), + } + }) + }); + donorRst.startSet(); + donorRst.initiate(); + + const recipientRst = new ReplSetTest({ + nodes: 1, + name: 'recipient', + nodeOptions: Object.assign(migrationX509Options.recipient, { + setParameter: { + internalInsertMaxBatchSize: + kMaxBatchSize /* Decrease internal max batch size so we can + still show writes are batched without + inserting hundreds of documents. */ + }, + }) + }); + recipientRst.startSet(); + recipientRst.initiate(); + + const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + + return { + tenantMigrationTest, + donorRst, + recipientRst, + teardown: function() { + tenantMigrationTest.stop(); + donorRst.stopSet(); + recipientRst.stopSet(); } - }) -}); -donorRst.startSet(); -donorRst.initiate(); - -const recipientRst = new ReplSetTest({ - nodes: 1, - name: 'recipient', - nodeOptions: Object.assign(migrationX509Options.recipient, { - setParameter: { - internalInsertMaxBatchSize: kMaxBatchSize /* Decrease internal max batch size so we can - still show writes are batched without - inserting hundreds of documents. */ - }, - }) -}); -recipientRst.startSet(); -recipientRst.initiate(); - -const kRecipientConnString = recipientRst.getURL(); - -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + }; +} function bulkInsertDocsOrdered(primaryHost, dbName, collName, numDocs) { const primary = new Mongo(primaryHost); @@ -159,6 +173,8 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { (() => { jsTestLog("Testing unordered bulk insert against a tenant migration that commits."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkUnorderedInserts-committed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -201,16 +217,19 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { assert(!err.errmsg); } }); + teardown(); })(); (() => { jsTestLog( "Testing unordered bulk insert against a tenant migration that blocks a few inserts and commits."); + const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup(); + const tenantId = "bulkUnorderedInserts-blocks-committed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), - recipientConnString: kRecipientConnString, + recipientConnString: recipientRst.getURL(), tenantId, }; const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); @@ -264,11 +283,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { assert.eq(err.errmsg, ""); } }); + teardown(); })(); (() => { jsTestLog("Testing unordered bulk insert against a tenant migration that aborts."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkUnorderedInserts-aborted"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -332,11 +354,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { assert(!err.errmsg); } }); + teardown(); })(); (() => { jsTestLog("Testing ordered bulk inserts against a tenant migration that commits."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkOrderedInserts-committed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -373,12 +398,15 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { // started blocking writes. assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize); assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationCommitted); + teardown(); })(); (() => { jsTestLog( "Testing ordered bulk insert against a tenant migration that blocks a few inserts and commits."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkOrderedInserts-blocks-committed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -428,11 +456,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { // started blocking writes. assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize); assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationCommitted); + teardown(); })(); (() => { jsTestLog("Testing ordered bulk write against a tenant migration that aborts."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkOrderedInserts-aborted"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -490,15 +521,18 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { // started blocking writes. assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize); assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationAborted); + teardown(); })(); (() => { jsTestLog("Testing unordered bulk multi update that blocks."); + const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup(); + const tenantId = "bulkUnorderedMultiUpdates-blocks"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), - recipientConnString: kRecipientConnString, + recipientConnString: recipientRst.getURL(), tenantId, }; const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); @@ -540,15 +574,18 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { bulkWriteRes.res.errmsg, "Operation interrupted by an internal data migration and could not be automatically retried", tojson(bulkWriteRes)); + teardown(); })(); (() => { jsTestLog("Testing ordered bulk multi update that blocks."); + const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup(); + const tenantId = "bulkOrderedMultiUpdates-blocks"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), - recipientConnString: kRecipientConnString, + recipientConnString: recipientRst.getURL(), tenantId, }; const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); @@ -590,11 +627,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { bulkWriteRes.res.errmsg, "Operation interrupted by an internal data migration and could not be automatically retried", tojson(bulkWriteRes)); + teardown(); })(); (() => { jsTestLog("Testing unordered multi updates against a tenant migration that has completed."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkUnorderedMultiUpdates-completed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -626,11 +666,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { bulkWriteRes.res.errmsg, "Operation interrupted by an internal data migration and could not be automatically retried", tojson(bulkWriteRes)); + teardown(); })(); (() => { jsTestLog("Testing ordered multi updates against a tenant migration that has completed."); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const tenantId = "bulkOrderedMultiUpdates-completed"; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -662,9 +705,6 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) { bulkWriteRes.res.errmsg, "Operation interrupted by an internal data migration and could not be automatically retried", tojson(bulkWriteRes)); + teardown(); })(); - -tenantMigrationTest.stop(); -donorRst.stopSet(); -recipientRst.stopSet(); })(); diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js b/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js index b7e3e36c8c2..f346420d90d 100644 --- a/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js +++ b/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js @@ -27,14 +27,7 @@ load("jstests/replsets/libs/tenant_migration_util.js"); const tenantMigrationTest = new TenantMigrationTest({ name: jsTestName(), - sharedOptions: { - setParameter: { - // set ttlMonitorSleepSecs and tenantMigrationGarbageCollectionDelayMS to low values - // to speed up cleanup between tests for shard merge - ttlMonitorSleepSecs: 1, - tenantMigrationGarbageCollectionDelayMS: 500, - } - } + quickGarbageCollection: true, }); const kCollName = "testColl"; diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js index 75c24362eef..b7d77a7b68a 100644 --- a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js +++ b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js @@ -27,15 +27,6 @@ load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); load("jstests/replsets/rslib.js"); -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: { - setParameter: { - tenantMigrationGarbageCollectionDelayMS: 0, - } - } -}); - const kCollName = "testColl"; const kTenantDefinedDbName = "0"; @@ -63,7 +54,7 @@ function runCommand(db, cmd, expectedError) { /** * Tests that the recipient starts rejecting all reads after cloning is done. */ -function testRejectAllReadsAfterCloningDone(testCase, dbName, collName) { +function testRejectAllReadsAfterCloningDone({testCase, dbName, collName, tenantMigrationTest}) { const tenantId = dbName.split('_')[0]; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -107,7 +98,7 @@ function testRejectAllReadsAfterCloningDone(testCase, dbName, collName) { * rejectReadsBeforeTimestamp. */ function testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp( - testCase, dbName, collName) { + {testCase, dbName, collName, tenantMigrationTest}) { const tenantId = dbName.split('_')[0]; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -185,7 +176,7 @@ function testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp( * recipient keeps rejecting all reads until the state doc is marked as garbage collectable. */ function testDoNotRejectReadsAfterMigrationAbortedBeforeReachingRejectReadsBeforeTimestamp( - testCase, dbName, collName) { + {testCase, dbName, collName, tenantMigrationTest}) { const tenantId = dbName.split('_')[0]; const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), @@ -238,7 +229,7 @@ function testDoNotRejectReadsAfterMigrationAbortedBeforeReachingRejectReadsBefor * collected. */ function testDoNotRejectReadsAfterMigrationAbortedAfterReachingRejectReadsBeforeTimestamp( - testCase, dbName, collName) { + {testCase, dbName, collName, tenantMigrationTest}) { const tenantId = dbName.split('_')[0]; const migrationId = UUID(); const migrationOpts = { @@ -412,13 +403,6 @@ const testCases = { }, }; -// Force the recipient to preserve all snapshot history to ensure that snapshot reads do not fail -// with SnapshotTooOld due to snapshot being unavailable. -const recipientRst = tenantMigrationTest.getRecipientRst(); -recipientRst.nodes.forEach(node => { - configureFailPoint(node, "WTPreserveSnapshotHistoryIndefinitely"); -}); - const testFuncs = { afterCloningDone: testRejectAllReadsAfterCloningDone, afterReachingBlockTs: testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp, @@ -432,9 +416,19 @@ for (const [testName, testFunc] of Object.entries(testFuncs)) { for (const [testCaseName, testCase] of Object.entries(testCases)) { jsTest.log("Testing " + testName + " with testCase " + testCaseName); let dbName = testCaseName + "-" + testName + "_" + kTenantDefinedDbName; - testFunc(testCase, dbName, kCollName); + const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + quickGarbageCollection: true, + }); + + // Force the recipient to preserve all snapshot history to ensure that snapshot reads do not + // fail with SnapshotTooOld due to snapshot being unavailable. + tenantMigrationTest.getRecipientRst().nodes.forEach(node => { + configureFailPoint(node, "WTPreserveSnapshotHistoryIndefinitely"); + }); + + testFunc({testCase, dbName, collName: kCollName, tenantMigrationTest}); + tenantMigrationTest.stop(); } } - -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js b/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js index 4985b35ea32..2a963464b11 100644 --- a/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js +++ b/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js @@ -20,16 +20,8 @@ load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: { - setParameter: { - tenantMigrationGarbageCollectionDelayMS: 1, - ttlMonitorSleepSecs: 1, - } - }, - initiateRstWithHighElectionTimeout: false -}); +const tenantMigrationTest = new TenantMigrationTest( + {name: jsTestName(), quickGarbageCollection: true, initiateRstWithHighElectionTimeout: false}); const kTenantId = "testTenantId"; @@ -67,4 +59,4 @@ assert.commandFailedWithCode(forgetMigrationThread.returnData(), donorRst.stopSet(); tenantMigrationTest.stop(); -})();
\ No newline at end of file +})(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js index 96dbe3089a3..29fd1fb256e 100644 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js +++ b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js @@ -28,17 +28,7 @@ load("jstests/replsets/libs/tenant_migration_util.js"); const tenantMigrationTest = new TenantMigrationTest({ name: jsTestName(), - sharedOptions: { - setParameter: { - // set ttlMonitorSleepSecs and tenantMigrationGarbageCollectionDelayMS to low values - // to speed up cleanup between tests for shard merge - ttlMonitorSleepSecs: 1, - tenantMigrationGarbageCollectionDelayMS: 500, - - // Allow non-timestamped reads on donor after migration completes for testing. - 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}) - } - } + quickGarbageCollection: true, }); const donorRst = tenantMigrationTest.getDonorRst(); diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js index 87627d50481..0a3ef6c87cd 100644 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js +++ b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js @@ -20,7 +20,8 @@ load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); const donorRst = tenantMigrationTest.getDonorRst(); const donorPrimary = donorRst.getPrimary(); @@ -82,6 +83,9 @@ const kTenantId = "testTenantId"; // Write after the migration is forgotten. assert.commandWorked(tenantCollOnRecipient.remove({_id: 1})); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString, + migrationOpts.tenantId); })(); (() => { @@ -115,6 +119,9 @@ const kTenantId = "testTenantId"; // Write after the migration is forgotten. assert.commandWorked(tenantCollOnRecipient.remove({_id: 1})); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString, + migrationOpts.tenantId); })(); (() => { @@ -146,6 +153,9 @@ const kTenantId = "testTenantId"; // Write after the migration is forgotten. assert.commandWorked(tenantCollOnRecipient.remove({_id: 1})); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString, + migrationOpts.tenantId); })(); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js index 1d8c9717c77..4949ed3b71f 100644 --- a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js +++ b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js @@ -54,18 +54,39 @@ function generateUniqueTenantId() { return kTenantIdPrefix + tenantCounter++; } -const donorRst = new ReplSetTest( - {nodes: 1, name: 'donorRst', nodeOptions: TenantMigrationUtil.makeX509OptionsForTest().donor}); - -donorRst.startSet(); -donorRst.initiate(); - -const tenantMigrationTest0 = new TenantMigrationTest({name: jsTestName(), donorRst}); - -const donorPrimary = donorRst.getPrimary(); -const recipientPrimary = tenantMigrationTest0.getRecipientPrimary(); - -let numRecipientSyncDataCmdSent = 0; +function setup() { + const {donor: donorNodeOptions} = TenantMigrationUtil.makeX509OptionsForTest(); + donorNodeOptions.setParameter = donorNodeOptions.setParameter || {}; + Object.assign(donorNodeOptions.setParameter, { + tenantMigrationGarbageCollectionDelayMS: 1 * 1000, + ttlMonitorSleepSecs: 1, + }); + const donorRst = new ReplSetTest({ + nodes: 1, + name: 'donorRst', + nodeOptions: donorNodeOptions, + }); + + donorRst.startSet(); + donorRst.initiate(); + + const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), donorRst, quickGarbageCollection: true}); + + const donorPrimary = donorRst.getPrimary(); + const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + + return { + tenantMigrationTest, + donorRst, + donorPrimary, + recipientPrimary, + teardown: function() { + tenantMigrationTest.stop(); + donorRst.stopSet(); + }, + }; +} // Test that a retry of a donorStartMigration command joins the existing migration that has // completed but has not been garbage-collected. @@ -76,13 +97,15 @@ let numRecipientSyncDataCmdSent = 0; tenantId, }; - TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration(migrationOpts)); - TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration(migrationOpts)); + const {tenantMigrationTest, recipientPrimary, teardown} = setup(); + + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); // If the second donorStartMigration had started a duplicate migration, the recipient would have // received four recipientSyncData commands instead of two. - numRecipientSyncDataCmdSent += 2; - checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent); + checkNumRecipientSyncDataCmdExecuted(recipientPrimary, 2); + teardown(); })(); // Test that a retry of a donorStartMigration command joins the ongoing migration. @@ -93,20 +116,22 @@ let numRecipientSyncDataCmdSent = 0; tenantId, }; - assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts)); - assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts)); + const {tenantMigrationTest, recipientPrimary, teardown} = setup(); + + assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); + assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); TenantMigrationTest.assertCommitted( - tenantMigrationTest0.waitForMigrationToComplete(migrationOpts)); + tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); TenantMigrationTest.assertCommitted( - tenantMigrationTest0.waitForMigrationToComplete(migrationOpts)); + tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); // If the second donorStartMigration had started a duplicate migration, the recipient would have // received four recipientSyncData commands instead of two. - numRecipientSyncDataCmdSent += 2; - checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent); + checkNumRecipientSyncDataCmdExecuted(recipientPrimary, 2); - assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts.migrationIdString)); + assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + teardown(); })(); /** @@ -114,8 +139,13 @@ let numRecipientSyncDataCmdSent = 0; * donorStartMigration command to start a migration that conflicts with an existing migration that * has committed but not garbage-collected (i.e. the donor has not received donorForgetMigration). */ -function testStartingConflictingMigrationAfterInitialMigrationCommitted( - tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) { +function testStartingConflictingMigrationAfterInitialMigrationCommitted({ + tenantMigrationTest0, + migrationOpts0, + tenantMigrationTest1, + migrationOpts1, + donorPrimary, +}) { TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration( migrationOpts0, false /* retryOnRetryableErrors */, false /* automaticForgetMigration */)); const res1 = assert.commandFailedWithCode(tenantMigrationTest1.runMigration(migrationOpts1), @@ -141,14 +171,21 @@ function testStartingConflictingMigrationAfterInitialMigrationCommitted( }).length); } assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts0.migrationIdString)); + tenantMigrationTest0.waitForMigrationGarbageCollection(migrationOpts0.migrationIdString, + migrationOpts0.tenantId); } /** * Tests that if the client runs multiple donorStartMigration commands that would start conflicting * migrations, only one of the migrations will start and succeed. */ -function testConcurrentConflictingMigrations( - tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) { +function testConcurrentConflictingMigrations({ + tenantMigrationTest0, + migrationOpts0, + tenantMigrationTest1, + migrationOpts1, + donorPrimary, +}) { const res0 = tenantMigrationTest0.startMigration(migrationOpts0); const res1 = tenantMigrationTest1.startMigration(migrationOpts1); @@ -207,73 +244,112 @@ function testConcurrentConflictingMigrations( // Test migrations with different migrationIds but identical settings. (() => { - let makeTestParams = () => { + const {tenantMigrationTest, donorPrimary, teardown} = setup(); + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), tenantId: generateUniqueTenantId() + "DiffMigrationId", }; const migrationOpts1 = Object.extend({}, migrationOpts0, true); migrationOpts1.migrationIdString = extractUUIDFromObject(UUID()); - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + return { + tenantMigrationTest0: tenantMigrationTest, + migrationOpts0, + tenantMigrationTest1: tenantMigrationTest, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); + teardown(); })(); // Test reusing a migrationId for different migration settings. // Test different tenantIds. (() => { - let makeTestParams = () => { + const {tenantMigrationTest, donorPrimary, teardown} = setup(); + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), tenantId: generateUniqueTenantId() + "DiffTenantId", }; const migrationOpts1 = Object.extend({}, migrationOpts0, true); migrationOpts1.tenantId = generateUniqueTenantId() + "DiffTenantId"; - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + return { + tenantMigrationTest0: tenantMigrationTest, + migrationOpts0, + tenantMigrationTest1: tenantMigrationTest, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); + teardown(); })(); // Test different recipient connection strings. (() => { + const { + tenantMigrationTest: tenantMigrationTest0, + donorRst, + donorPrimary, + teardown, + } = setup(); + const tenantMigrationTest1 = new TenantMigrationTest({name: `${jsTestName()}1`, donorRst}); - let makeTestParams = () => { + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), - tenantId: generateUniqueTenantId() + "DiffRecipientConnString", + tenantId: `${generateUniqueTenantId()}DiffRecipientConnString`, }; // The recipient connection string will be populated by the TenantMigrationTest fixture, so // no need to set it here. const migrationOpts1 = Object.extend({}, migrationOpts0, true); - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1]; + return { + tenantMigrationTest0, + migrationOpts0, + tenantMigrationTest1, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); tenantMigrationTest1.stop(); + teardown(); })(); // Test different cloning read preference. (() => { - let makeTestParams = () => { + const {tenantMigrationTest, donorPrimary, teardown} = setup(); + + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), tenantId: generateUniqueTenantId() + "DiffReadPref", }; const migrationOpts1 = Object.extend({}, migrationOpts0, true); migrationOpts1.readPreference = {mode: "secondary"}; - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + return { + tenantMigrationTest0: tenantMigrationTest, + migrationOpts0, + tenantMigrationTest1: tenantMigrationTest, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); + teardown(); })(); const kDonorCertificateAndPrivateKey = @@ -287,7 +363,9 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi // Test different donor certificates. (() => { - let makeTestParams = () => { + const {tenantMigrationTest, donorPrimary, teardown} = setup(); + + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), tenantId: generateUniqueTenantId() + "DiffDonorCertificate", @@ -296,16 +374,25 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi }; const migrationOpts1 = Object.extend({}, migrationOpts0, true); migrationOpts1.donorCertificateForRecipient = kExpiredDonorCertificateAndPrivateKey; - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + return { + tenantMigrationTest0: tenantMigrationTest, + migrationOpts0, + tenantMigrationTest1: tenantMigrationTest, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); + teardown(); })(); // Test different recipient certificates. (() => { - let makeTestParams = () => { + const {tenantMigrationTest, donorPrimary, teardown} = setup(); + + const makeTestParams = () => { const migrationOpts0 = { migrationIdString: extractUUIDFromObject(UUID()), tenantId: generateUniqueTenantId() + "DiffRecipientCertificate", @@ -314,13 +401,17 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi }; const migrationOpts1 = Object.extend({}, migrationOpts0, true); migrationOpts1.recipientCertificateForDonor = kExpiredRecipientCertificateAndPrivateKey; - return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + return { + tenantMigrationTest0: tenantMigrationTest, + migrationOpts0, + tenantMigrationTest1: tenantMigrationTest, + migrationOpts1, + donorPrimary, + }; }; - testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); - testConcurrentConflictingMigrations(...makeTestParams()); + testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams()); + testConcurrentConflictingMigrations(makeTestParams()); + teardown(); })(); - -tenantMigrationTest0.stop(); -donorRst.stopSet(); })(); diff --git a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js index 6989a35ff99..4edff2c8f39 100644 --- a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js +++ b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js @@ -4,6 +4,7 @@ * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * requires_fcv_52, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, @@ -123,24 +124,24 @@ assert.commandWorked(primary.adminCommand({ recipientSyncDataThread0.start(); recipientSyncDataThread1.start(); - jsTestLog("Waiting until both conflicting instances get started and hit the failPoint."); + jsTestLog("Waiting until one gets started and hits the failPoint."); assert.commandWorked(primary.adminCommand({ waitForFailPoint: "pauseBeforeRunTenantMigrationRecipientInstance", - timesEntered: fpPauseBeforeRunTenantMigrationRecipientInstance.timesEntered + 2, + timesEntered: fpPauseBeforeRunTenantMigrationRecipientInstance.timesEntered + 1, maxTimeMS: kDefaultWaitForFailPointTimeout })); - // Two instances are expected as the tenantId conflict is still unresolved. + // One instance is expected as the tenantId conflict is still unresolved. jsTestLog("Fetching current operations before conflict is resolved."); const currentOpEntriesBeforeInsert = getTenantMigrationRecipientCurrentOpEntries( primary, {desc: "tenant recipient migration", tenantId}); - assert.eq(2, currentOpEntriesBeforeInsert.length, tojson(currentOpEntriesBeforeInsert)); + assert.eq(1, currentOpEntriesBeforeInsert.length, tojson(currentOpEntriesBeforeInsert)); jsTestLog("Unblocking the tenant migration instance from persisting the state doc."); fpPauseBeforeRunTenantMigrationRecipientInstance.off(); - // Wait for both the conflicting instances to complete. Although both will "complete", one will - // return with ErrorCodes.ConflictingOperationInProgress, and the other with a + // Check responses for both commands. One will return with + // ErrorCodes.ConflictingOperationInProgress, and the other with a // TestData.stopFailPointErrorCode (a failpoint indicating that we have persisted the document). const res0 = assert.commandFailed(recipientSyncDataThread0.returnData()); const res1 = assert.commandFailed(recipientSyncDataThread1.returnData()); diff --git a/jstests/replsets/tenant_migration_donor_abort_state_transition.js b/jstests/replsets/tenant_migration_donor_abort_state_transition.js index 1592d3554bb..cfbb2a73588 100644 --- a/jstests/replsets/tenant_migration_donor_abort_state_transition.js +++ b/jstests/replsets/tenant_migration_donor_abort_state_transition.js @@ -20,16 +20,17 @@ load("jstests/replsets/libs/tenant_migration_util.js"); const kTenantIdPrefix = "testTenantId"; -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: {setParameter: {tenantMigrationGarbageCollectionDelayMS: 0}} -}); - /** * Starts a migration and forces the write to insert the donor's state doc to abort on the first few * tries. Asserts that the migration still completes successfully. */ -function testAbortInitialState(donorRst) { +function testAbortInitialState() { + const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + quickGarbageCollection: true, + }); + const donorRst = tenantMigrationTest.getDonorRst(); + const donorPrimary = donorRst.getPrimary(); // Force the storage transaction for the insert to abort prior to inserting the WiredTiger @@ -66,6 +67,7 @@ function testAbortInitialState(donorRst) { donorRst.nodes, migrationId, tenantId, TenantMigrationTest.DonorState.kCommitted); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.stop(); } /** @@ -74,10 +76,14 @@ function testAbortInitialState(donorRst) { * reaching 'pauseFailPoint' to abort on the first few tries. Asserts that the migration still * completes successfully. */ -function testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState) { +function testAbortStateTransition(pauseFailPoint, setUpFailPoints, nextState) { jsTest.log(`Test aborting the write to transition to state "${ nextState}" after reaching failpoint "${pauseFailPoint}"`); + const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); + const donorRst = tenantMigrationTest.getDonorRst(); + const donorPrimary = donorRst.getPrimary(); const tenantId = `${kTenantIdPrefix}-${nextState}`; @@ -123,11 +129,11 @@ function testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nex }); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.stop(); } -const donorRst = tenantMigrationTest.getDonorRst(); jsTest.log("Test aborting donor's state doc insert"); -testAbortInitialState(donorRst); +testAbortInitialState(); jsTest.log("Test aborting donor's state doc update"); [{ @@ -143,8 +149,6 @@ jsTest.log("Test aborting donor's state doc update"); setUpFailPoints: ["abortTenantMigrationBeforeLeavingBlockingState"], nextState: TenantMigrationTest.DonorState.kAborted }].forEach(({pauseFailPoint, setUpFailPoints = [], nextState}) => { - testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState); + testAbortStateTransition(pauseFailPoint, setUpFailPoints, nextState); }); - -tenantMigrationTest.stop(); }()); diff --git a/jstests/replsets/tenant_migration_donor_kill_op_retry.js b/jstests/replsets/tenant_migration_donor_kill_op_retry.js index 41617b027a6..e81a1699212 100644 --- a/jstests/replsets/tenant_migration_donor_kill_op_retry.js +++ b/jstests/replsets/tenant_migration_donor_kill_op_retry.js @@ -37,8 +37,6 @@ function makeTenantId() { return kTenantIdPrefix + testNum++; } -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); - { // This section tests behavior in the middle of a tenant migration. let fpNames = [ @@ -51,6 +49,8 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); "\" to test that the migration will retry the " + "operation at the failpoint if a killOp is issued."); + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); + const migrationOpts = { migrationIdString: extractUUIDFromObject(UUID()), recipientConnString: tenantMigrationTest.getRecipientConnString(), @@ -82,6 +82,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); TenantMigrationTest.assertCommitted(runMigrationThread.returnData()); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.stop(); } } @@ -92,6 +93,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); "pauseTenantMigrationBeforeCreatingExternalKeysTTLIndex" ]; for (let fpName of fpNames) { + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); tenantMigrationTest.getDonorRst().stopSet(); tenantMigrationTest.getDonorRst().startSet(Object.assign({}, migrationX509Options.donor, { setParameter: {['failpoint.' + fpName]: tojson({mode: 'alwaysOn'})} @@ -137,30 +139,32 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); TenantMigrationTest.assertCommitted(runMigrationThread.returnData()); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + tenantMigrationTest.stop(); } } { - // This section is testing behavior during garbage collection. - tenantMigrationTest.getDonorRst().stopSet(); - tenantMigrationTest.getDonorRst().startSet( - Object.assign({}, migrationX509Options.donor, {setParameter: garbageCollectionOpts})); - tenantMigrationTest.getDonorRst().initiate(); - TenantMigrationUtil.createTenantMigrationRecipientRoleIfNotExist( - tenantMigrationTest.getDonorRst()); - - tenantMigrationTest.getRecipientRst().stopSet(); - tenantMigrationTest.getRecipientRst().startSet( - Object.assign({}, migrationX509Options.recipient, {setParameter: garbageCollectionOpts})); - tenantMigrationTest.getRecipientRst().initiate(); - TenantMigrationUtil.createTenantMigrationDonorRoleIfNotExist( - tenantMigrationTest.getRecipientRst()); - let fpNames = [ "pauseTenantMigrationDonorBeforeMarkingStateGarbageCollectable", "pauseTenantMigrationBeforeMarkingExternalKeysGarbageCollectable" ]; for (let fpName of fpNames) { + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); + // This section is testing behavior during garbage collection. + tenantMigrationTest.getDonorRst().stopSet(); + tenantMigrationTest.getDonorRst().startSet( + Object.assign({}, migrationX509Options.donor, {setParameter: garbageCollectionOpts})); + tenantMigrationTest.getDonorRst().initiate(); + TenantMigrationUtil.createTenantMigrationRecipientRoleIfNotExist( + tenantMigrationTest.getDonorRst()); + + tenantMigrationTest.getRecipientRst().stopSet(); + tenantMigrationTest.getRecipientRst().startSet(Object.assign( + {}, migrationX509Options.recipient, {setParameter: garbageCollectionOpts})); + tenantMigrationTest.getRecipientRst().initiate(); + TenantMigrationUtil.createTenantMigrationDonorRoleIfNotExist( + tenantMigrationTest.getRecipientRst()); + jsTestLog( "Setting failpoint \"" + fpName + "\" during migration garbage collection to test that the migration will retry the " + @@ -201,7 +205,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); fp.off(); forgetMigrationThread.join(); tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, tenantId); + tenantMigrationTest.stop(); } } -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_donor_retry.js b/jstests/replsets/tenant_migration_donor_retry.js index 902005cdd38..1fb1a36c526 100644 --- a/jstests/replsets/tenant_migration_donor_retry.js +++ b/jstests/replsets/tenant_migration_donor_retry.js @@ -31,20 +31,27 @@ const garbageCollectionOpts = { ttlMonitorSleepSecs: 1 }; -const donorRst = new ReplSetTest({ - name: "donorRst", - nodes: 1, - nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, - {setParameter: garbageCollectionOpts}) -}); - -donorRst.startSet(); -donorRst.initiate(); +function setup() { + const donorRst = new ReplSetTest({ + name: "donorRst", + nodes: 1, + nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, + {setParameter: garbageCollectionOpts}) + }); -const tenantMigrationTest = new TenantMigrationTest( - {name: jsTestName(), donorRst: donorRst, sharedOptions: {setParameter: garbageCollectionOpts}}); -const donorPrimary = tenantMigrationTest.getDonorPrimary(); -const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + donorRst.startSet(); + donorRst.initiate(); + + const tenantMigrationTest = new TenantMigrationTest( + {name: jsTestName(), donorRst: donorRst, quickGarbageCollection: true}); + return { + tenantMigrationTest, + teardown: function() { + donorRst.stopSet(); + tenantMigrationTest.stop(); + }, + }; +} function makeTenantId() { return kTenantIdPrefix + testNum++; @@ -55,7 +62,7 @@ function makeTenantId() { * recipientSyncData command to fail with the given 'errorCode', and asserts the donor retries on * that error and is able to commit. */ -function testDonorRetryRecipientSyncDataCmdOnError(errorCode, failMode) { +function testDonorRetryRecipientSyncDataCmdOnError(tenantMigrationTest, errorCode, failMode) { const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); const tenantId = makeTenantId(); @@ -95,7 +102,7 @@ function testDonorRetryRecipientSyncDataCmdOnError(errorCode, failMode) { * recipientForgetMigration command to fail with the given 'errorCode', and asserts the donor * retries on that error and commits. */ -function testDonorRetryRecipientForgetMigrationCmdOnError(errorCode) { +function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, errorCode) { const tenantId = makeTenantId(); const migrationId = UUID(); const migrationOpts = { @@ -127,68 +134,88 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(errorCode) { (() => { jsTest.log( "Test that the donor retries recipientSyncData (to make the recipient start cloning) on recipient stepdown errors"); + const {tenantMigrationTest, teardown} = setup(); - const migrationId = - testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.NotWritablePrimary, {times: 1}); + const migrationId = testDonorRetryRecipientSyncDataCmdOnError( + tenantMigrationTest, ErrorCodes.NotWritablePrimary, {times: 1}); - const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + const configDonorsColl = + tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); + teardown(); })(); (() => { jsTest.log( "Test that the donor retries recipientSyncData (to make the recipient start cloning) on recipient shutdown errors"); + const {tenantMigrationTest, teardown} = setup(); - const migrationId = - testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.ShutdownInProgress, {times: 1}); + const migrationId = testDonorRetryRecipientSyncDataCmdOnError( + tenantMigrationTest, ErrorCodes.ShutdownInProgress, {times: 1}); - const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + const configDonorsColl = + tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); + teardown(); })(); (() => { jsTest.log( "Test that the donor retries recipientSyncData (with returnAfterReachingDonorTimestamp) on stepdown errors"); + const {tenantMigrationTest, teardown} = setup(); - const migrationId = - testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.NotWritablePrimary, {skip: 1}); + const migrationId = testDonorRetryRecipientSyncDataCmdOnError( + tenantMigrationTest, ErrorCodes.NotWritablePrimary, {skip: 1}); - const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + const configDonorsColl = + tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); + teardown(); })(); (() => { jsTest.log( "Test that the donor retries recipientSyncData (with returnAfterReachingDonorTimestamp) on recipient shutdown errors"); + const {tenantMigrationTest, teardown} = setup(); - const migrationId = - testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.ShutdownInProgress, {skip: 1}); + const migrationId = testDonorRetryRecipientSyncDataCmdOnError( + tenantMigrationTest, ErrorCodes.ShutdownInProgress, {skip: 1}); - const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + const configDonorsColl = + tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); assert.eq(TenantMigrationTest.DonorState.kCommitted, configDonorsColl.findOne({_id: migrationId}).state); + teardown(); })(); (() => { jsTest.log("Test that the donor retries recipientForgetMigration on stepdown errors"); - testDonorRetryRecipientForgetMigrationCmdOnError(ErrorCodes.NotWritablePrimary); + const {tenantMigrationTest, teardown} = setup(); + testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, + ErrorCodes.NotWritablePrimary); + teardown(); })(); (() => { jsTest.log("Test that the donor retries recipientForgetMigration on shutdown errors"); - testDonorRetryRecipientForgetMigrationCmdOnError(ErrorCodes.ShutdownInProgress); + const {tenantMigrationTest, teardown} = setup(); + testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, + ErrorCodes.ShutdownInProgress); + teardown(); })(); (() => { jsTest.log("Test that the donor retries recipientForgetMigration on interruption errors"); + const {tenantMigrationTest, teardown} = setup(); // Test an error code that is in the 'Interruption' category but not in the 'isRetriable' // category. const interruptionErrorCode = ErrorCodes.MaxTimeMSExpired; assert(ErrorCodes.isInterruption(interruptionErrorCode)); - testDonorRetryRecipientForgetMigrationCmdOnError(interruptionErrorCode); + testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, interruptionErrorCode); + teardown(); })(); // Each donor state doc is updated three times throughout the lifetime of a tenant migration: @@ -201,6 +228,7 @@ const kWriteErrorTimeMS = 50; (() => { jsTest.log("Test that the donor retries state doc insert on retriable errors"); + const {tenantMigrationTest, teardown} = setup(); const tenantId = makeTenantId(); const migrationId = UUID(); @@ -210,7 +238,7 @@ const kWriteErrorTimeMS = 50; recipientConnString: tenantMigrationTest.getRecipientConnString(), }; - let fp = configureFailPoint(donorPrimary, "failCollectionInserts", { + let fp = configureFailPoint(tenantMigrationTest.getDonorPrimary(), "failCollectionInserts", { collectionNS: TenantMigrationTest.kConfigDonorsNS, }); @@ -230,15 +258,18 @@ const kWriteErrorTimeMS = 50; migrationThread.join(); TenantMigrationTest.assertCommitted(migrationThread.returnData()); - const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + const configDonorsColl = + tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS); const donorStateDoc = configDonorsColl.findOne({_id: migrationId}); assert.eq(TenantMigrationTest.DonorState.kCommitted, donorStateDoc.state); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); + teardown(); })(); (() => { jsTest.log("Test that the donor retries state doc update on retriable errors"); + const {tenantMigrationTest, teardown} = setup(); const tenantId = kTenantIdPrefix + "RetryOnStateDocUpdateError"; const migrationId = UUID(); @@ -251,7 +282,7 @@ const kWriteErrorTimeMS = 50; const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()); // Use a random number of skips to fail a random update to config.tenantMigrationDonors. - const fp = configureFailPoint(donorPrimary, + const fp = configureFailPoint(tenantMigrationTest.getDonorPrimary(), "failCollectionUpdates", { collectionNS: TenantMigrationTest.kConfigDonorsNS, @@ -274,19 +305,17 @@ const kWriteErrorTimeMS = 50; fp.off(); migrationThread.join(); - const donorStateDoc = - donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).findOne({_id: migrationId}); + const donorStateDoc = tenantMigrationTest.getDonorPrimary() + .getCollection(TenantMigrationTest.kConfigDonorsNS) + .findOne({_id: migrationId}); assert.eq(donorStateDoc.state, TenantMigrationTest.DonorState.kCommitted); assert(donorStateDoc.expireAt); // Check that the recipient state doc is also correctly marked as garbage collectable. - const recipientStateDoc = - recipientPrimary.getCollection(TenantMigrationTest.kConfigRecipientsNS).findOne({ - _id: migrationId - }); + const recipientStateDoc = tenantMigrationTest.getRecipientPrimary() + .getCollection(TenantMigrationTest.kConfigRecipientsNS) + .findOne({_id: migrationId}); assert(recipientStateDoc.expireAt); + teardown(); })(); - -donorRst.stopSet(); -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_drop_state_doc_collection.js b/jstests/replsets/tenant_migration_drop_state_doc_collection.js index 97606a49f2f..cc2a1b3d4ae 100644 --- a/jstests/replsets/tenant_migration_drop_state_doc_collection.js +++ b/jstests/replsets/tenant_migration_drop_state_doc_collection.js @@ -162,19 +162,14 @@ function testDroppingStateDocCollections(tenantMigrationTest, fpName, { } } -const tenantMigrationTest = new TenantMigrationTest({ - name: jsTestName(), - sharedOptions: { - setParameter: { - tenantMigrationGarbageCollectionDelayMS: 1, - ttlMonitorSleepSecs: 1, - } - }, - initiateRstWithHighElectionTimeout: false -}); - jsTest.log("Test dropping donor and recipient state doc collections during a migration."); kMigrationFpNames.forEach(fpName => { + const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + quickGarbageCollection: true, + initiateRstWithHighElectionTimeout: false + }); + testDroppingStateDocCollections( tenantMigrationTest, fpName, {dropDonorsCollection: true, dropRecipientsCollection: true}); @@ -219,7 +214,6 @@ kMigrationFpNames.forEach(fpName => { expectedForgetMigrationError: originalMigrationStartedOnRecipient ? ErrorCodes.ConflictingOperationInProgress : null }); + tenantMigrationTest.stop(); }); - -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_external_keys_ttl.js b/jstests/replsets/tenant_migration_external_keys_ttl.js index 65f06b338ac..60835a3ac2d 100644 --- a/jstests/replsets/tenant_migration_external_keys_ttl.js +++ b/jstests/replsets/tenant_migration_external_keys_ttl.js @@ -99,15 +99,32 @@ function makeTestParams() { // (() => { - const tmt = new TenantMigrationTest( - {name: jsTestName(), sharedOptions: {setParameter: ttlMonitorOptions}}); - - // Verify the external keys TTL index is created on both replica sets on stepup. - waitForExternalKeysTTLIndex(tmt.getDonorPrimary()); - waitForExternalKeysTTLIndex(tmt.getRecipientPrimary()); + function setup() { + const tmt = new TenantMigrationTest( + {name: jsTestName(), sharedOptions: {setParameter: ttlMonitorOptions}}); + + // Verify the external keys TTL index is created on both replica sets on stepup. + waitForExternalKeysTTLIndex(tmt.getDonorPrimary()); + waitForExternalKeysTTLIndex(tmt.getRecipientPrimary()); + + return { + tmt, + teardown: function() { + tmt.stop(); + }, + }; + } - jsTestLog("Basic case with multiple migrations"); - { + (() => { + jsTestLog("Basic case with multiple migrations"); + const {tmt, teardown} = setup(); + if (TenantMigrationUtil.isShardMergeEnabled(tmt.getDonorPrimary().getDB("admin"))) { + // This test runs multiple concurrent migrations, which shard merge can't handle. + jsTestLog( + "Skip: featureFlagShardMerge is enabled and this test runs multiple concurrent migrations, which shard merge can't handle."); + teardown(); + return; + } const [tenantId, migrationId, migrationOpts] = makeTestParams(); TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts, @@ -148,10 +165,12 @@ function makeTestParams() { {migrationId: otherMigrationId, expectTTLValue: false}); verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId: otherMigrationId, expectTTLValue: false}); - } + teardown(); + })(); jsTestLog("Verify the TTL value is respected and keys are eventually reaped"); { + const {tmt, teardown} = setup(); const [tenantId, migrationId, migrationOpts] = makeTestParams(); const lowerExternalKeysBufferSecs = 5; @@ -190,9 +209,8 @@ function makeTestParams() { setTenantMigrationExpirationParams(tmt.getRecipientPrimary(), origRecipientStateDocExpirationParam, origRecipientKeysExpirationParam); + teardown(); } - - tmt.stop(); })(); // @@ -200,28 +218,43 @@ function makeTestParams() { // (() => { - const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); - const donorRst = new ReplSetTest({ - nodes: 3, - name: "donorRst", - nodeOptions: Object.assign(migrationX509Options.donor, {setParameter: ttlMonitorOptions}) - }); - donorRst.startSet(); - donorRst.initiate(); - - const recipientRst = new ReplSetTest({ - nodes: 3, - name: "recipientRst", - nodeOptions: - Object.assign(migrationX509Options.recipient, {setParameter: ttlMonitorOptions}) - }); - recipientRst.startSet(); - recipientRst.initiate(); - - const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + function setup() { + const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); + const donorRst = new ReplSetTest({ + nodes: 3, + name: "donorRst", + nodeOptions: + Object.assign(migrationX509Options.donor, {setParameter: ttlMonitorOptions}) + }); + donorRst.startSet(); + donorRst.initiate(); + + const recipientRst = new ReplSetTest({ + nodes: 3, + name: "recipientRst", + nodeOptions: + Object.assign(migrationX509Options.recipient, {setParameter: ttlMonitorOptions}) + }); + recipientRst.startSet(); + recipientRst.initiate(); + + const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + + return { + tmt, + donorRst, + recipientRst, + teardown: function() { + donorRst.stopSet(); + recipientRst.stopSet(); + tmt.stop(); + }, + }; + } jsTestLog("Donor failover before receiving forgetMigration"); { + const {tmt, teardown} = setup(); const [tenantId, migrationId, migrationOpts] = makeTestParams(); const donorPrimary = tmt.getDonorPrimary(); const fp = @@ -248,10 +281,12 @@ function makeTestParams() { // buffer is 1 day so the keys will not have been deleted. verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true}); verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true}); + teardown(); } jsTestLog("Recipient failover before receiving forgetMigration"); { + const {tmt, teardown} = setup(); const [tenantId, migrationId, migrationOpts] = makeTestParams(); const recipientPrimary = tmt.getRecipientPrimary(); const fp = configureFailPoint(recipientPrimary, @@ -279,11 +314,13 @@ function makeTestParams() { // buffer is 1 day so the keys will not have been deleted. verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true}); verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true}); + teardown(); } jsTestLog( "Donor failover after receiving forgetMigration before marking keys garbage collectable"); { + const {tmt, donorRst, teardown} = setup(); const [tenantId, migrationId, migrationOpts] = makeTestParams(); const donorPrimary = tmt.getDonorPrimary(); @@ -315,11 +352,13 @@ function makeTestParams() { // buffer is 1 day so the keys will not have been deleted. verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true}); verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true}); + teardown(); } jsTestLog( "Recipient failover after receiving forgetMigration before marking keys garbage collectable"); { + const {tmt, donorRst, teardown} = setup(); const [tenantId, migrationId, migrationOpts] = makeTestParams(); const recipientPrimary = tmt.getRecipientPrimary(); @@ -349,18 +388,19 @@ function makeTestParams() { verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true}); verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true}); - } - - // The next two cases expect the external keys to expire, so lower the expiration timeouts. - const lowerExternalKeysBufferSecs = 5; - const lowerStateDocExpirationMS = 500; - for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) { - setTenantMigrationExpirationParams( - conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs); + teardown(); } jsTestLog("Donor failover after receiving forgetMigration after updating keys."); { + const {tmt, donorRst, recipientRst, teardown} = setup(); + // this test expects the external keys to expire, so lower the expiration timeouts. + const lowerExternalKeysBufferSecs = 5; + const lowerStateDocExpirationMS = 500; + for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) { + setTenantMigrationExpirationParams( + conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs); + } const [tenantId, migrationId, migrationOpts] = makeTestParams(); const donorPrimary = tmt.getDonorPrimary(); @@ -394,10 +434,19 @@ function makeTestParams() { fp.off(); assert.commandWorked(forgetMigrationThread.returnData()); + teardown(); } jsTestLog("Recipient failover after receiving forgetMigration after updating keys."); { + const {tmt, donorRst, recipientRst, teardown} = setup(); + // this test expects the external keys to expire, so lower the expiration timeouts. + const lowerExternalKeysBufferSecs = 5; + const lowerStateDocExpirationMS = 500; + for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) { + setTenantMigrationExpirationParams( + conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs); + } const [tenantId, migrationId, migrationOpts] = makeTestParams(); const recipientPrimary = tmt.getRecipientPrimary(); @@ -433,10 +482,7 @@ function makeTestParams() { // Eventually the donor's keys should be deleted too. waitForExternalKeysToBeDeleted(tmt.getDonorPrimary(), migrationId); + teardown(); } - - donorRst.stopSet(); - recipientRst.stopSet(); - tmt.stop(); })(); })(); diff --git a/jstests/replsets/tenant_migration_filters_tenant_id.js b/jstests/replsets/tenant_migration_filters_tenant_id.js index a897a65aa87..9a2b26cb5c9 100644 --- a/jstests/replsets/tenant_migration_filters_tenant_id.js +++ b/jstests/replsets/tenant_migration_filters_tenant_id.js @@ -17,8 +17,6 @@ load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); - const tenantIdPrefix = "tenantId"; const baseDBName = "testDB"; const collName = "testColl"; @@ -31,6 +29,8 @@ const makeBaseTenantId = () => { const runTest = (baseTenantId, dbName, shouldMatch) => { jsTestLog(`Running tenant migration with dbName ${dbName} and tenantId ${baseTenantId}`); + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); + assert.eq(shouldMatch, TenantMigrationUtil.isNamespaceForTenant(baseTenantId, dbName)); tenantMigrationTest.insertDonorDB(dbName, collName); @@ -45,6 +45,7 @@ const runTest = (baseTenantId, dbName, shouldMatch) => { // migrated, so we can directly call it here. tenantMigrationTest.verifyRecipientDB(baseTenantId, dbName, collName); tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString); + tenantMigrationTest.stop(); }; const testCases = [ @@ -64,6 +65,4 @@ for (const {makeTenantId, shouldMatch} of testCases) { const tenantId = makeTenantId(baseTenantId); runTest(baseTenantId, `${tenantId}_${baseDBName}`, shouldMatch); } - -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js index f7dc0fe4479..710d20f200d 100644 --- a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js +++ b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js @@ -21,18 +21,6 @@ load("jstests/replsets/libs/tenant_migration_test.js"); load('jstests/replsets/rslib.js'); // for waitForNewlyAddedRemovalForNodeToBeCommitted const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); -const donorRst = new ReplSetTest({ - name: "donorRst", - nodes: 1, - nodeOptions: Object.assign(migrationX509Options.donor, { - setParameter: { - // Allow non-timestamped reads on donor after migration completes for testing. - 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}), - } - }) -}); -donorRst.startSet(); -donorRst.initiate(); const testDBName = 'testDB'; const testCollName = 'testColl'; @@ -143,9 +131,23 @@ function restartNodeAndCheckStateDuringOplogApplication( // 5. Steps up the restarted node as the recipient primary, lifts the recipient failpoint, and // allows the migration to complete. function runTestCase(tenantId, recipientFailpoint, checkMtab, restartNodeAndCheckStateFunction) { + const donorRst = new ReplSetTest({ + name: "donorRst", + nodes: 1, + nodeOptions: Object.assign(migrationX509Options.donor, { + setParameter: { + // Allow non-timestamped reads on donor after migration completes for testing. + 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': + tojson({mode: 'alwaysOn'}), + } + }) + }); + donorRst.startSet(); + donorRst.initiate(); + const tenantMigrationTest = new TenantMigrationTest({ name: jsTestName(), - donorRst: donorRst, + donorRst, sharedOptions: {setParameter: {tenantApplierBatchSizeOps: 2}} }); @@ -171,6 +173,7 @@ function runTestCase(tenantId, recipientFailpoint, checkMtab, restartNodeAndChec assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); tenantMigrationTest.stop(); + donorRst.stopSet(); } // These two test cases are for before the mtab is created, and before the oplog applier has been @@ -196,6 +199,4 @@ runTestCase('tenantId4', "fpAfterWaitForRejectReadsBeforeTimestamp", true /* checkMtab */, restartNodeAndCheckStateWithoutOplogApplication); - -donorRst.stopSet(); })(); diff --git a/jstests/replsets/tenant_migration_recipient_rollback_recovery.js b/jstests/replsets/tenant_migration_recipient_rollback_recovery.js index f011d9a3a7a..0f694c40912 100644 --- a/jstests/replsets/tenant_migration_recipient_rollback_recovery.js +++ b/jstests/replsets/tenant_migration_recipient_rollback_recovery.js @@ -30,20 +30,6 @@ const kGarbageCollectionDelayMS = 30 * 1000; const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest(); -const donorRst = new ReplSetTest({ - name: "donorRst", - nodes: 1, - nodeOptions: Object.assign({}, migrationX509Options.donor, { - setParameter: { - tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, - ttlMonitorSleepSecs: 1, - } - }) -}); -donorRst.startSet(); -donorRst.initiate(); -const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); - function makeMigrationOpts(tenantMigrationTest, migrationId, tenantId) { return { migrationIdString: extractUUIDFromObject(migrationId), @@ -61,6 +47,21 @@ function makeMigrationOpts(tenantMigrationTest, migrationId, tenantId) { * replication steady state. */ function testRollBack(setUpFunc, rollbackOpsFunc, steadyStateFunc) { + const donorRst = new ReplSetTest({ + name: "donorRst", + nodes: 1, + nodeOptions: Object.assign({}, migrationX509Options.donor, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS, + ttlMonitorSleepSecs: 1, + } + }) + }); + donorRst.startSet(); + donorRst.initiate(); + + const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); + const recipientRst = new ReplSetTest({ name: "recipientRst", nodes: 3, @@ -106,6 +107,7 @@ function testRollBack(setUpFunc, rollbackOpsFunc, steadyStateFunc) { steadyStateFunc(tenantMigrationTest); + donorRst.stopSet(); recipientRst.stopSet(); } @@ -333,6 +335,4 @@ testRollBackMarkingStateGarbageCollectable(); jsTest.log("Test roll back random"); testRollBackRandom(); - -donorRst.stopSet(); }()); diff --git a/jstests/replsets/tenant_migration_recipient_ttl.js b/jstests/replsets/tenant_migration_recipient_ttl.js index 20100d9ebf8..afab2032942 100644 --- a/jstests/replsets/tenant_migration_recipient_ttl.js +++ b/jstests/replsets/tenant_migration_recipient_ttl.js @@ -17,16 +17,8 @@ load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). load("jstests/replsets/libs/tenant_migration_test.js"); load("jstests/replsets/libs/tenant_migration_util.js"); -const kGarbageCollectionParams = { - // Set the delay to 30s so that we can see the document vanish. - tenantMigrationGarbageCollectionDelayMS: 30 * 1000, - - // Set the TTL monitor to run at a smaller interval to speed up the test. - ttlMonitorSleepSecs: 1 -}; - -const tenantMigrationTest = new TenantMigrationTest( - {name: jsTestName(), sharedOptions: {setParameter: kGarbageCollectionParams}}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); const kRecipientTTLIndexName = "TenantMigrationRecipientTTLIndex"; @@ -83,4 +75,4 @@ sleep(30000); // The garbage collection delay is 30s. tenantMigrationTest.waitForMigrationGarbageCollection(kMigrationId, kTenantId); tenantMigrationTest.stop(); -})();
\ No newline at end of file +})(); diff --git a/jstests/replsets/tenant_migration_retry_session_migration.js b/jstests/replsets/tenant_migration_retry_session_migration.js index 27e7069bca4..2f82cd4ea34 100644 --- a/jstests/replsets/tenant_migration_retry_session_migration.js +++ b/jstests/replsets/tenant_migration_retry_session_migration.js @@ -19,17 +19,8 @@ load("jstests/replsets/libs/tenant_migration_util.js"); load("jstests/replsets/rslib.js"); load("jstests/libs/uuid_util.js"); -const kGarbageCollectionParams = { - // Set the delay before a donor state doc is garbage collected to be short to speed up - // the test. - tenantMigrationGarbageCollectionDelayMS: 3 * 1000, - - // Set the TTL monitor to run at a smaller interval to speed up the test. - ttlMonitorSleepSecs: 1, -}; - -const tenantMigrationTest = new TenantMigrationTest( - {name: jsTestName(), sharedOptions: {setParameter: kGarbageCollectionParams}}); +const tenantMigrationTest = + new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true}); const kTenantId = "testTenantId"; const kDbName = tenantMigrationTest.tenantDB(kTenantId, "testDB"); diff --git a/jstests/replsets/tenant_migration_x509.js b/jstests/replsets/tenant_migration_x509.js index e0a5169a3f5..32621531735 100644 --- a/jstests/replsets/tenant_migration_x509.js +++ b/jstests/replsets/tenant_migration_x509.js @@ -22,7 +22,15 @@ function makeTestNs(tenantId) { return {dbName: tenantId + "_testDb", collName: "testColl"}; } -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); +function setup() { + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); + return { + tenantMigrationTest, + teardown: function() { + tenantMigrationTest.stop(); + } + }; +} const kDonorCertificateAndPrivateKey = TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_donor.pem"); @@ -31,6 +39,7 @@ const kRecipientCertificateAndPrivateKey = (() => { jsTest.log("Test valid donor and recipient certificates"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "validCertificates"; const migrationOpts = { @@ -45,10 +54,12 @@ const kRecipientCertificateAndPrivateKey = TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, true /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid donor certificate, no header and trailer"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidDonorCertificateNoHeaderAndTrailer"; const migrationOpts = { @@ -67,10 +78,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid donor certificate, use private key as certificate"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidDonorCertificatePrivateKeyAsCertificate"; const migrationOpts = { @@ -89,10 +102,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid donor private key, no header and trailer"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidDonorPrivateKeyNoHeaderAndTrailer"; const migrationOpts = { @@ -111,10 +126,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid donor private key, use certificate as private key"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidDonorPrivateKeyCertificateAsPrivateKey"; const migrationOpts = { @@ -133,10 +150,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid donor certificate and private key pair"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidDonorCertificatePrivateKeyPair"; const migrationOpts = { @@ -155,10 +174,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test expired donor certificate and key"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "expiredDonorCertificate"; const migrationOpts = { @@ -175,10 +196,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid recipient certificate, no header and trailer"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidRecipientCertificateNoHeaderAndTrailer"; const migrationOpts = { @@ -197,10 +220,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid recipient certificate, use private key as certificate"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidRecipientCertificatePrivateKeyAsCertificate"; const migrationOpts = { @@ -219,10 +244,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid recipient private key, no header and trailer"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidRecipientPrivateKeyNoHeaderAndTrailer"; const migrationOpts = { @@ -241,10 +268,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid recipient private key, use certificate as private key"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidRecipientPrivateKeyCertificateAsPrivateKey"; const migrationOpts = { @@ -263,10 +292,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test expired recipient certificate and key"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "expiredRecipientCertificate"; const migrationOpts = { @@ -283,10 +314,12 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { jsTest.log("Test invalid recipient certificate and private key pair"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "invalidRecipientCertificatePrivateKeyPair"; const migrationOpts = { @@ -305,16 +338,17 @@ const kRecipientCertificateAndPrivateKey = ErrorCodes.InvalidSSLConfiguration); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); if (!TestData.auth) { jsTestLog("Skipping testing authorization since auth is not enabled"); - tenantMigrationTest.stop(); return; } (() => { jsTest.log("Test donor certificate without the required privileges"); + const {tenantMigrationTest, teardown} = setup(); const migrationId = UUID(); const tenantId = "donorCertificateInsufficientPrivileges"; const migrationOpts = { @@ -331,9 +365,11 @@ if (!TestData.auth) { ErrorCodes.Unauthorized); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); (() => { + const {tenantMigrationTest, teardown} = setup(); jsTest.log("Test recipient certificate without the required privileges"); const migrationId = UUID(); const tenantId = "recipientCertificateInsufficientPrivileges"; @@ -351,7 +387,6 @@ if (!TestData.auth) { ErrorCodes.Unauthorized); tenantMigrationTest.verifyRecipientDB( tenantId, dbName, collName, false /* migrationCommitted */); + teardown(); })(); - -tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migrations_noop_writes.js b/jstests/replsets/tenant_migrations_noop_writes.js index abae2350f4c..f4dd9f647e9 100644 --- a/jstests/replsets/tenant_migrations_noop_writes.js +++ b/jstests/replsets/tenant_migrations_noop_writes.js @@ -70,40 +70,52 @@ function runAfterClusterTimeRead(dbName, collName, operationTime, clusterTime, e } } -const donorRst = new ReplSetTest({ - nodes: 3, - name: "donor", - settings: {chainingAllowed: false}, - nodeOptions: Object.assign(migrationX509Options.donor, { - setParameter: { - // To allow after test hooks to run without errors. - "failpoint.tenantMigrationDonorAllowsNonTimestampedReads": tojson({mode: "alwaysOn"}), - } - }) -}); -donorRst.startSet(); -donorRst.initiate(); - -const recipientRst = new ReplSetTest({ - nodes: 3, - name: "recipient", - settings: {chainingAllowed: false}, - nodeOptions: migrationX509Options.recipient -}); -recipientRst.startSet(); -recipientRst.initiate(); - -const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); - -const donorPrimary = tmt.getDonorPrimary(); -const recipientPrimary = tmt.getRecipientPrimary(); +function setup() { + const donorRst = new ReplSetTest({ + nodes: 3, + name: "donor", + settings: {chainingAllowed: false}, + nodeOptions: Object.assign(migrationX509Options.donor, { + setParameter: { + // To allow after test hooks to run without errors. + "failpoint.tenantMigrationDonorAllowsNonTimestampedReads": + tojson({mode: "alwaysOn"}), + } + }) + }); + donorRst.startSet(); + donorRst.initiate(); + + const recipientRst = new ReplSetTest({ + nodes: 3, + name: "recipient", + settings: {chainingAllowed: false}, + nodeOptions: migrationX509Options.recipient + }); + recipientRst.startSet(); + recipientRst.initiate(); + + const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst}); + + return { + tmt, + teardown: function() { + donorRst.stopSet(); + recipientRst.stopSet(); + tmt.stop(); + }, + }; +} { jsTestLog("Testing noops on the recipient"); + const {tmt, teardown} = setup(); + const [tenantId, migrationId, migrationOpts, tenantDbName] = makeTestParams(); const laggedRecipientSecondary = tmt.getRecipientRst().getSecondary(); - const fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState"); + const fp = + configureFailPoint(tmt.getDonorPrimary(), "pauseTenantMigrationBeforeLeavingBlockingState"); // // Run a migration, pausing after selecting a block timestamp to advance cluster time beyond it @@ -120,14 +132,15 @@ const recipientPrimary = tmt.getRecipientPrimary(); // already been replicated by this point. stopServerReplication(laggedRecipientSecondary); - advanceClusterTime(donorPrimary, kUnrelatedDbName, collName); + advanceClusterTime(tmt.getDonorPrimary(), kUnrelatedDbName, collName); - const donorRes = - assert.commandWorked(donorPrimary.getDB(tenantDbName).runCommand({find: collName})); + const donorRes = assert.commandWorked( + tmt.getDonorPrimary().getDB(tenantDbName).runCommand({find: collName})); assert(donorRes.operationTime, tojson(donorRes)); - assert.eq(timestampCmp(donorRes.operationTime, getBlockTimestamp(donorPrimary, tenantId)), - 1, - tojson(donorRes)); + assert.eq( + timestampCmp(donorRes.operationTime, getBlockTimestamp(tmt.getDonorPrimary(), tenantId)), + 1, + tojson(donorRes)); fp.off(); TenantMigrationTest.assertCommitted(tmt.waitForMigrationToComplete( @@ -140,7 +153,7 @@ const recipientPrimary = tmt.getRecipientPrimary(); // profiled so we use a fail point to detect it. // - const hangInNoopFp = configureFailPoint(recipientPrimary, "hangInAppendOplogNote"); + const hangInNoopFp = configureFailPoint(tmt.getRecipientPrimary(), "hangInAppendOplogNote"); const awaitReadOnRecipient = startParallelShell(funWithArgs(runAfterClusterTimeRead, tenantDbName, collName, @@ -153,14 +166,18 @@ const recipientPrimary = tmt.getRecipientPrimary(); restartServerReplication(laggedRecipientSecondary); awaitReadOnRecipient(); + teardown(); } { jsTestLog("Testing noops on the donor"); + const {tmt, teardown} = setup(); + const [tenantId, migrationId, migrationOpts, tenantDbName] = makeTestParams(); const laggedDonorSecondary = tmt.getDonorRst().getSecondary(); - const fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState"); + const fp = + configureFailPoint(tmt.getDonorPrimary(), "pauseTenantMigrationBeforeLeavingBlockingState"); // // Commit a normal migration, but disable replication on a donor secondary before the commit so @@ -182,12 +199,13 @@ const recipientPrimary = tmt.getRecipientPrimary(); // Advance cluster time on the recipient beyond the block timestamp. // - advanceClusterTime(recipientPrimary, kUnrelatedDbName, collName); + advanceClusterTime(tmt.getRecipientPrimary(), kUnrelatedDbName, collName); - const recipientRes = - assert.commandWorked(recipientPrimary.getDB(tenantDbName).runCommand({find: collName})); + const recipientRes = assert.commandWorked( + tmt.getRecipientPrimary().getDB(tenantDbName).runCommand({find: collName})); assert(recipientRes.operationTime, tojson(recipientRes)); - assert.eq(timestampCmp(recipientRes.operationTime, getBlockTimestamp(donorPrimary, tenantId)), + assert.eq(timestampCmp(recipientRes.operationTime, + getBlockTimestamp(tmt.getDonorPrimary(), tenantId)), 1, tojson(recipientRes)); @@ -199,7 +217,7 @@ const recipientPrimary = tmt.getRecipientPrimary(); // necessary to unblock tenant operations waiting for a cluster time > the block timestamp. // - const hangInNoopFp = configureFailPoint(donorPrimary, "hangInAppendOplogNote"); + const hangInNoopFp = configureFailPoint(tmt.getDonorPrimary(), "hangInAppendOplogNote"); const awaitReadOnDonor = startParallelShell(funWithArgs(runAfterClusterTimeRead, tenantDbName, collName, @@ -213,9 +231,6 @@ const recipientPrimary = tmt.getRecipientPrimary(); restartServerReplication(laggedDonorSecondary); awaitReadOnDonor(); + teardown(); } - -donorRst.stopSet(); -recipientRst.stopSet(); -tmt.stop(); })(); diff --git a/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp b/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp index ebd71c3b423..9952d301d47 100644 --- a/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp +++ b/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp @@ -117,24 +117,13 @@ public: auto returnAfterReachingDonorTs = cmd.getReturnAfterReachingDonorTimestamp(); - try { - if (!returnAfterReachingDonorTs) { - return Response( - recipientInstance->waitUntilMigrationReachesConsistentState(opCtx)); - } - - return Response( - recipientInstance->waitUntilMigrationReachesReturnAfterReachingTimestamp( - opCtx, *returnAfterReachingDonorTs)); - } catch (ExceptionFor<ErrorCodes::ConflictingOperationInProgress>& ex) { - // A conflict may arise when inserting the recipientInstance's state document. - // Since the conflict occurred at the insert stage, that means this instance's - // tenantId conflicts with an existing instance's tenantId. Therefore, remove the - // instance that was just created. - // The status from this exception will be passed to the instance interrupt() method. - recipientService->releaseInstance(stateDocBson["_id"].wrap(), ex.toStatus()); - throw; + if (!returnAfterReachingDonorTs) { + return Response(recipientInstance->waitUntilMigrationReachesConsistentState(opCtx)); } + + return Response( + recipientInstance->waitUntilMigrationReachesReturnAfterReachingTimestamp( + opCtx, *returnAfterReachingDonorTs)); } private: diff --git a/src/mongo/db/repl/tenant_migration_donor_service.cpp b/src/mongo/db/repl/tenant_migration_donor_service.cpp index 510e24e8772..70fc5c1384f 100644 --- a/src/mongo/db/repl/tenant_migration_donor_service.cpp +++ b/src/mongo/db/repl/tenant_migration_donor_service.cpp @@ -34,6 +34,7 @@ #include "mongo/client/connection_string.h" #include "mongo/client/replica_set_monitor.h" #include "mongo/config.h" +#include "mongo/db/commands/tenant_migration_donor_cmds_gen.h" #include "mongo/db/commands/tenant_migration_recipient_cmds_gen.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" @@ -148,20 +149,32 @@ void TenantMigrationDonorService::checkIfConflictsWithOtherInstances( OperationContext* opCtx, BSONObj initialState, const std::vector<const repl::PrimaryOnlyService::Instance*>& existingInstances) { - auto tenantId = initialState["tenantId"].valueStringData(); - // Any existing migration for this tenant must be aborted and garbage-collectable. - for (auto& instance : existingInstances) { - auto typedInstance = checked_cast<const TenantMigrationDonorService::Instance*>(instance); - auto durableState = typedInstance->getDurableState(); + auto stateDoc = tenant_migration_access_blocker::parseDonorStateDocument(initialState); + auto isNewShardMerge = stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge; - if (typedInstance->getTenantId() == tenantId) { - // If a migration with the same tenantId exists, it needs to be already in kAborted - // state + for (auto& instance : existingInstances) { + auto existingTypedInstance = + checked_cast<const TenantMigrationDonorService::Instance*>(instance); + auto existingState = existingTypedInstance->getDurableState(); + auto existingIsAborted = existingState && + existingState->state == TenantMigrationDonorStateEnum::kAborted && + existingState->expireAt; + + uassert(ErrorCodes::ConflictingOperationInProgress, + str::stream() << "Cannot start a shard merge with existing migrations in progress", + !isNewShardMerge || existingIsAborted); + + uassert( + ErrorCodes::ConflictingOperationInProgress, + str::stream() << "Cannot start a migration with an existing shard merge in progress", + existingTypedInstance->getProtocol() != MigrationProtocolEnum::kShardMerge || + existingIsAborted); + + // Any existing migration for this tenant must be aborted and garbage-collectable. + if (existingTypedInstance->getTenantId() == stateDoc.getTenantId()) { uassert(ErrorCodes::ConflictingOperationInProgress, - str::stream() << "tenant " << tenantId << " is already migrating", - durableState && - durableState->state == TenantMigrationDonorStateEnum::kAborted && - durableState->expireAt); + str::stream() << "tenant " << stateDoc.getTenantId() << " is already migrating", + existingIsAborted); } } } diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp index 25312c71eb4..9b154211be1 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp @@ -40,6 +40,7 @@ #include "mongo/client/replica_set_monitor_manager.h" #include "mongo/config.h" #include "mongo/db/client.h" +#include "mongo/db/commands/tenant_migration_donor_cmds_gen.h" #include "mongo/db/commands/test_commands_enabled.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" @@ -315,7 +316,25 @@ void TenantMigrationRecipientService::checkIfConflictsWithOtherInstances( OperationContext* opCtx, BSONObj initialStateDoc, const std::vector<const PrimaryOnlyService::Instance*>& existingInstances) { - // TODO (SERVER-59786): check for conflicts here, not RecipientSyncDataCmd::typedRun. + auto tenantId = initialStateDoc["tenantId"].valueStringData(); + + for (auto& instance : existingInstances) { + auto existingTypedInstance = + checked_cast<const TenantMigrationRecipientService::Instance*>(instance); + auto existingState = existingTypedInstance->getState(); + auto isDone = existingState.getState() == TenantMigrationRecipientStateEnum::kDone && + existingState.getExpireAt(); + + uassert(ErrorCodes::ConflictingOperationInProgress, + "an existing shard merge is in progress", + isDone || + (existingTypedInstance->getProtocol() != MigrationProtocolEnum::kShardMerge && + existingState.getProtocol() != MigrationProtocolEnum::kShardMerge)); + + uassert(ErrorCodes::ConflictingOperationInProgress, + str::stream() << "tenant " << tenantId << " is already migrating", + isDone || existingTypedInstance->getTenantId() != tenantId); + } } std::shared_ptr<PrimaryOnlyService::Instance> TenantMigrationRecipientService::constructInstance( @@ -2670,5 +2689,10 @@ const MigrationProtocolEnum& TenantMigrationRecipientService::Instance::getProto return _protocol; } +const TenantMigrationRecipientDocument TenantMigrationRecipientService::Instance::getState() const { + stdx::lock_guard lk(_mutex); + return _stateDoc; +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h index 2de82942985..f35f1a93f19 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.h +++ b/src/mongo/db/repl/tenant_migration_recipient_service.h @@ -148,6 +148,11 @@ public: */ const MigrationProtocolEnum& getProtocol() const; + /* + * Returns the recipient document state + */ + const TenantMigrationRecipientDocument getState() const; + /** * To be called on the instance returned by PrimaryOnlyService::getOrCreate(). Returns an * error if the options this Instance was created with are incompatible with the options |