summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChristopher Caplinger <christopher.caplinger@mongodb.com>2021-12-02 18:54:45 +0000
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2021-12-02 19:45:48 +0000
commitac9a64634fb87fa4b065c0e83c50c7b4d1400688 (patch)
treeef303b5b02abadbc52084df05c75a1f09ee624bf
parent7f120e5dbf538ce29b637c5caa2d175a58e1afad (diff)
downloadmongo-ac9a64634fb87fa4b065c0e83c50c7b4d1400688.tar.gz
SERVER-59786: Prevent concurrent merges
-rw-r--r--jstests/replsets/libs/tenant_migration_test.js9
-rw-r--r--jstests/replsets/tenant_migration_abort_forget_retry.js5
-rw-r--r--jstests/replsets/tenant_migration_cluster_time_keys_cloning.js3
-rw-r--r--jstests/replsets/tenant_migration_collection_ttl.js5
-rw-r--r--jstests/replsets/tenant_migration_commit_transaction_retry.js2
-rw-r--r--jstests/replsets/tenant_migration_concurrent_bulk_writes.js120
-rw-r--r--jstests/replsets/tenant_migration_concurrent_reads_on_donor.js9
-rw-r--r--jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js40
-rw-r--r--jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js14
-rw-r--r--jstests/replsets/tenant_migration_concurrent_writes_on_donor.js12
-rw-r--r--jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js12
-rw-r--r--jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js201
-rw-r--r--jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js13
-rw-r--r--jstests/replsets/tenant_migration_donor_abort_state_transition.js28
-rw-r--r--jstests/replsets/tenant_migration_donor_kill_op_retry.js40
-rw-r--r--jstests/replsets/tenant_migration_donor_retry.js113
-rw-r--r--jstests/replsets/tenant_migration_drop_state_doc_collection.js20
-rw-r--r--jstests/replsets/tenant_migration_external_keys_ttl.js130
-rw-r--r--jstests/replsets/tenant_migration_filters_tenant_id.js7
-rw-r--r--jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js31
-rw-r--r--jstests/replsets/tenant_migration_recipient_rollback_recovery.js32
-rw-r--r--jstests/replsets/tenant_migration_recipient_ttl.js14
-rw-r--r--jstests/replsets/tenant_migration_retry_session_migration.js13
-rw-r--r--jstests/replsets/tenant_migration_x509.js43
-rw-r--r--jstests/replsets/tenant_migrations_noop_writes.js105
-rw-r--r--src/mongo/db/commands/tenant_migration_recipient_cmds.cpp23
-rw-r--r--src/mongo/db/repl/tenant_migration_donor_service.cpp37
-rw-r--r--src/mongo/db/repl/tenant_migration_recipient_service.cpp26
-rw-r--r--src/mongo/db/repl/tenant_migration_recipient_service.h5
29 files changed, 690 insertions, 422 deletions
diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js
index 0c87a193aea..9cbb7523559 100644
--- a/jstests/replsets/libs/tenant_migration_test.js
+++ b/jstests/replsets/libs/tenant_migration_test.js
@@ -33,7 +33,8 @@ function TenantMigrationTest({
sharedOptions = {},
// Default this to true so it is easier for data consistency checks.
allowStaleReadsOnDonor = true,
- initiateRstWithHighElectionTimeout = true
+ initiateRstWithHighElectionTimeout = true,
+ quickGarbageCollection = false,
}) {
const donorPassedIn = (donorRst !== undefined);
const recipientPassedIn = (recipientRst !== undefined);
@@ -42,7 +43,11 @@ function TenantMigrationTest({
const migrationCertificates = TenantMigrationUtil.makeMigrationCertificatesForTest();
const nodes = sharedOptions.nodes || 2;
- let setParameterOpts = sharedOptions.setParameter || {};
+ const setParameterOpts = sharedOptions.setParameter || {};
+ if (quickGarbageCollection) {
+ setParameterOpts.tenantMigrationGarbageCollectionDelayMS = 3 * 1000;
+ setParameterOpts.ttlMonitorSleepSecs = 3;
+ }
donorRst = donorPassedIn ? donorRst : performSetUp(true /* isDonor */);
recipientRst = recipientPassedIn ? recipientRst : performSetUp(false /* isDonor */);
diff --git a/jstests/replsets/tenant_migration_abort_forget_retry.js b/jstests/replsets/tenant_migration_abort_forget_retry.js
index 823f204a79d..379fbfc4556 100644
--- a/jstests/replsets/tenant_migration_abort_forget_retry.js
+++ b/jstests/replsets/tenant_migration_abort_forget_retry.js
@@ -29,7 +29,8 @@ function makeTenantId() {
return kTenantIdPrefix + testNum++;
}
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
(() => {
const migrationId1 = extractUUIDFromObject(UUID());
@@ -60,6 +61,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
migrationId2 + ", tenantId: " + tenantId);
TenantMigrationTest.assertCommitted(
tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId}));
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId);
})();
(() => {
@@ -111,6 +113,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
migrationId2 + ", tenantId: " + tenantId);
TenantMigrationTest.assertCommitted(
tenantMigrationTest.runMigration({migrationIdString: migrationId2, tenantId: tenantId}));
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationId2, tenantId);
})();
tenantMigrationTest.stop();
diff --git a/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js
index 0adcb568bcf..5e15855f214 100644
--- a/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js
+++ b/jstests/replsets/tenant_migration_cluster_time_keys_cloning.js
@@ -2,9 +2,12 @@
* Test that tenant migration donor and recipient correctly copy each other cluster time keys into
* their config.external_validation_keys collection.
*
+ * TODO (SERVER-61231): Adapt for shard merge.
+ *
* @tags: [
* incompatible_with_eft,
* incompatible_with_macos,
+ * incompatible_with_shard_merge,
* incompatible_with_windows_tls,
* requires_majority_read_concern,
* requires_persistence,
diff --git a/jstests/replsets/tenant_migration_collection_ttl.js b/jstests/replsets/tenant_migration_collection_ttl.js
index 8047fca397f..e7bff842178 100644
--- a/jstests/replsets/tenant_migration_collection_ttl.js
+++ b/jstests/replsets/tenant_migration_collection_ttl.js
@@ -147,6 +147,8 @@ function assertTTLDeleteExpiredDocs(dbName, node) {
assertTTLDeleteExpiredDocs(dbName, donorPrimary);
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString,
+ migrationOpts.tenantId);
})();
// Tests that:
@@ -211,6 +213,9 @@ function assertTTLDeleteExpiredDocs(dbName, node) {
// After the tenant migration is aborted, the TTL cleanup is restored.
assertTTLDeleteExpiredDocs(dbName, recipientPrimary);
assertTTLDeleteExpiredDocs(dbName, donorPrimary);
+
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString,
+ migrationOpts.tenantId);
})();
tenantMigrationTest.stop();
diff --git a/jstests/replsets/tenant_migration_commit_transaction_retry.js b/jstests/replsets/tenant_migration_commit_transaction_retry.js
index e07c12bb6ff..fba740959ea 100644
--- a/jstests/replsets/tenant_migration_commit_transaction_retry.js
+++ b/jstests/replsets/tenant_migration_commit_transaction_retry.js
@@ -28,7 +28,7 @@ const kGarbageCollectionParams = {
};
const tenantMigrationTest = new TenantMigrationTest(
- {name: jsTestName(), sharedOptions: {nodes: 1, setParameter: kGarbageCollectionParams}});
+ {name: jsTestName(), sharedOptions: {nodes: 1}, quickGarbageCollection: true});
const kTenantId = "testTenantId";
const kDbName = tenantMigrationTest.tenantDB(kTenantId, "testDB");
diff --git a/jstests/replsets/tenant_migration_concurrent_bulk_writes.js b/jstests/replsets/tenant_migration_concurrent_bulk_writes.js
index beda0b68c67..2390e07a5d4 100644
--- a/jstests/replsets/tenant_migration_concurrent_bulk_writes.js
+++ b/jstests/replsets/tenant_migration_concurrent_bulk_writes.js
@@ -37,40 +37,54 @@ const kBatchTypes = {
remove: 3
};
-const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
-const donorRst = new ReplSetTest({
- nodes: 1,
- name: 'donor',
- nodeOptions: Object.assign(migrationX509Options.donor, {
- setParameter: {
- internalInsertMaxBatchSize:
- kMaxBatchSize, /* Decrease internal max batch size so we can still show writes are
- batched without inserting hundreds of documents. */
- // Allow non-timestamped reads on donor after migration completes for testing.
- 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}),
+function setup() {
+ const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
+ const donorRst = new ReplSetTest({
+ nodes: 1,
+ name: 'donor',
+ nodeOptions: Object.assign(migrationX509Options.donor, {
+ setParameter: {
+ internalInsertMaxBatchSize:
+ kMaxBatchSize, /* Decrease internal max batch size so we can still show writes
+ are batched without inserting hundreds of documents. */
+ // Allow non-timestamped reads on donor after migration completes for testing.
+ 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads':
+ tojson({mode: 'alwaysOn'}),
+ }
+ })
+ });
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const recipientRst = new ReplSetTest({
+ nodes: 1,
+ name: 'recipient',
+ nodeOptions: Object.assign(migrationX509Options.recipient, {
+ setParameter: {
+ internalInsertMaxBatchSize:
+ kMaxBatchSize /* Decrease internal max batch size so we can
+ still show writes are batched without
+ inserting hundreds of documents. */
+ },
+ })
+ });
+ recipientRst.startSet();
+ recipientRst.initiate();
+
+ const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+
+ return {
+ tenantMigrationTest,
+ donorRst,
+ recipientRst,
+ teardown: function() {
+ tenantMigrationTest.stop();
+ donorRst.stopSet();
+ recipientRst.stopSet();
}
- })
-});
-donorRst.startSet();
-donorRst.initiate();
-
-const recipientRst = new ReplSetTest({
- nodes: 1,
- name: 'recipient',
- nodeOptions: Object.assign(migrationX509Options.recipient, {
- setParameter: {
- internalInsertMaxBatchSize: kMaxBatchSize /* Decrease internal max batch size so we can
- still show writes are batched without
- inserting hundreds of documents. */
- },
- })
-});
-recipientRst.startSet();
-recipientRst.initiate();
-
-const kRecipientConnString = recipientRst.getURL();
-
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+ };
+}
function bulkInsertDocsOrdered(primaryHost, dbName, collName, numDocs) {
const primary = new Mongo(primaryHost);
@@ -159,6 +173,8 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
(() => {
jsTestLog("Testing unordered bulk insert against a tenant migration that commits.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkUnorderedInserts-committed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -201,16 +217,19 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
assert(!err.errmsg);
}
});
+ teardown();
})();
(() => {
jsTestLog(
"Testing unordered bulk insert against a tenant migration that blocks a few inserts and commits.");
+ const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup();
+
const tenantId = "bulkUnorderedInserts-blocks-committed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
- recipientConnString: kRecipientConnString,
+ recipientConnString: recipientRst.getURL(),
tenantId,
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
@@ -264,11 +283,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
assert.eq(err.errmsg, "");
}
});
+ teardown();
})();
(() => {
jsTestLog("Testing unordered bulk insert against a tenant migration that aborts.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkUnorderedInserts-aborted";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -332,11 +354,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
assert(!err.errmsg);
}
});
+ teardown();
})();
(() => {
jsTestLog("Testing ordered bulk inserts against a tenant migration that commits.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkOrderedInserts-committed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -373,12 +398,15 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
// started blocking writes.
assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize);
assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationCommitted);
+ teardown();
})();
(() => {
jsTestLog(
"Testing ordered bulk insert against a tenant migration that blocks a few inserts and commits.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkOrderedInserts-blocks-committed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -428,11 +456,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
// started blocking writes.
assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize);
assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationCommitted);
+ teardown();
})();
(() => {
jsTestLog("Testing ordered bulk write against a tenant migration that aborts.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkOrderedInserts-aborted";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -490,15 +521,18 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
// started blocking writes.
assert.eq(writeErrors[0].index, kNumWriteBatchesWithoutMigrationConflict * kMaxBatchSize);
assert.eq(writeErrors[0].code, ErrorCodes.TenantMigrationAborted);
+ teardown();
})();
(() => {
jsTestLog("Testing unordered bulk multi update that blocks.");
+ const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup();
+
const tenantId = "bulkUnorderedMultiUpdates-blocks";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
- recipientConnString: kRecipientConnString,
+ recipientConnString: recipientRst.getURL(),
tenantId,
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
@@ -540,15 +574,18 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
bulkWriteRes.res.errmsg,
"Operation interrupted by an internal data migration and could not be automatically retried",
tojson(bulkWriteRes));
+ teardown();
})();
(() => {
jsTestLog("Testing ordered bulk multi update that blocks.");
+ const {tenantMigrationTest, donorRst, recipientRst, teardown} = setup();
+
const tenantId = "bulkOrderedMultiUpdates-blocks";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
- recipientConnString: kRecipientConnString,
+ recipientConnString: recipientRst.getURL(),
tenantId,
};
const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
@@ -590,11 +627,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
bulkWriteRes.res.errmsg,
"Operation interrupted by an internal data migration and could not be automatically retried",
tojson(bulkWriteRes));
+ teardown();
})();
(() => {
jsTestLog("Testing unordered multi updates against a tenant migration that has completed.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkUnorderedMultiUpdates-completed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -626,11 +666,14 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
bulkWriteRes.res.errmsg,
"Operation interrupted by an internal data migration and could not be automatically retried",
tojson(bulkWriteRes));
+ teardown();
})();
(() => {
jsTestLog("Testing ordered multi updates against a tenant migration that has completed.");
+ const {tenantMigrationTest, donorRst, teardown} = setup();
+
const tenantId = "bulkOrderedMultiUpdates-completed";
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -662,9 +705,6 @@ function bulkMultiUpdateDocsUnordered(primaryHost, dbName, collName, numDocs) {
bulkWriteRes.res.errmsg,
"Operation interrupted by an internal data migration and could not be automatically retried",
tojson(bulkWriteRes));
+ teardown();
})();
-
-tenantMigrationTest.stop();
-donorRst.stopSet();
-recipientRst.stopSet();
})();
diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js b/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js
index b7e3e36c8c2..f346420d90d 100644
--- a/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js
+++ b/jstests/replsets/tenant_migration_concurrent_reads_on_donor.js
@@ -27,14 +27,7 @@ load("jstests/replsets/libs/tenant_migration_util.js");
const tenantMigrationTest = new TenantMigrationTest({
name: jsTestName(),
- sharedOptions: {
- setParameter: {
- // set ttlMonitorSleepSecs and tenantMigrationGarbageCollectionDelayMS to low values
- // to speed up cleanup between tests for shard merge
- ttlMonitorSleepSecs: 1,
- tenantMigrationGarbageCollectionDelayMS: 500,
- }
- }
+ quickGarbageCollection: true,
});
const kCollName = "testColl";
diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js
index 75c24362eef..b7d77a7b68a 100644
--- a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js
+++ b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js
@@ -27,15 +27,6 @@ load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
load("jstests/replsets/rslib.js");
-const tenantMigrationTest = new TenantMigrationTest({
- name: jsTestName(),
- sharedOptions: {
- setParameter: {
- tenantMigrationGarbageCollectionDelayMS: 0,
- }
- }
-});
-
const kCollName = "testColl";
const kTenantDefinedDbName = "0";
@@ -63,7 +54,7 @@ function runCommand(db, cmd, expectedError) {
/**
* Tests that the recipient starts rejecting all reads after cloning is done.
*/
-function testRejectAllReadsAfterCloningDone(testCase, dbName, collName) {
+function testRejectAllReadsAfterCloningDone({testCase, dbName, collName, tenantMigrationTest}) {
const tenantId = dbName.split('_')[0];
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -107,7 +98,7 @@ function testRejectAllReadsAfterCloningDone(testCase, dbName, collName) {
* rejectReadsBeforeTimestamp.
*/
function testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp(
- testCase, dbName, collName) {
+ {testCase, dbName, collName, tenantMigrationTest}) {
const tenantId = dbName.split('_')[0];
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -185,7 +176,7 @@ function testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp(
* recipient keeps rejecting all reads until the state doc is marked as garbage collectable.
*/
function testDoNotRejectReadsAfterMigrationAbortedBeforeReachingRejectReadsBeforeTimestamp(
- testCase, dbName, collName) {
+ {testCase, dbName, collName, tenantMigrationTest}) {
const tenantId = dbName.split('_')[0];
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
@@ -238,7 +229,7 @@ function testDoNotRejectReadsAfterMigrationAbortedBeforeReachingRejectReadsBefor
* collected.
*/
function testDoNotRejectReadsAfterMigrationAbortedAfterReachingRejectReadsBeforeTimestamp(
- testCase, dbName, collName) {
+ {testCase, dbName, collName, tenantMigrationTest}) {
const tenantId = dbName.split('_')[0];
const migrationId = UUID();
const migrationOpts = {
@@ -412,13 +403,6 @@ const testCases = {
},
};
-// Force the recipient to preserve all snapshot history to ensure that snapshot reads do not fail
-// with SnapshotTooOld due to snapshot being unavailable.
-const recipientRst = tenantMigrationTest.getRecipientRst();
-recipientRst.nodes.forEach(node => {
- configureFailPoint(node, "WTPreserveSnapshotHistoryIndefinitely");
-});
-
const testFuncs = {
afterCloningDone: testRejectAllReadsAfterCloningDone,
afterReachingBlockTs: testRejectOnlyReadsWithAtClusterTimeLessThanRejectReadsBeforeTimestamp,
@@ -432,9 +416,19 @@ for (const [testName, testFunc] of Object.entries(testFuncs)) {
for (const [testCaseName, testCase] of Object.entries(testCases)) {
jsTest.log("Testing " + testName + " with testCase " + testCaseName);
let dbName = testCaseName + "-" + testName + "_" + kTenantDefinedDbName;
- testFunc(testCase, dbName, kCollName);
+ const tenantMigrationTest = new TenantMigrationTest({
+ name: jsTestName(),
+ quickGarbageCollection: true,
+ });
+
+ // Force the recipient to preserve all snapshot history to ensure that snapshot reads do not
+ // fail with SnapshotTooOld due to snapshot being unavailable.
+ tenantMigrationTest.getRecipientRst().nodes.forEach(node => {
+ configureFailPoint(node, "WTPreserveSnapshotHistoryIndefinitely");
+ });
+
+ testFunc({testCase, dbName, collName: kCollName, tenantMigrationTest});
+ tenantMigrationTest.stop();
}
}
-
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js b/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js
index 4985b35ea32..2a963464b11 100644
--- a/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js
+++ b/jstests/replsets/tenant_migration_concurrent_state_doc_removal_and_stepdown.js
@@ -20,16 +20,8 @@ load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
-const tenantMigrationTest = new TenantMigrationTest({
- name: jsTestName(),
- sharedOptions: {
- setParameter: {
- tenantMigrationGarbageCollectionDelayMS: 1,
- ttlMonitorSleepSecs: 1,
- }
- },
- initiateRstWithHighElectionTimeout: false
-});
+const tenantMigrationTest = new TenantMigrationTest(
+ {name: jsTestName(), quickGarbageCollection: true, initiateRstWithHighElectionTimeout: false});
const kTenantId = "testTenantId";
@@ -67,4 +59,4 @@ assert.commandFailedWithCode(forgetMigrationThread.returnData(),
donorRst.stopSet();
tenantMigrationTest.stop();
-})(); \ No newline at end of file
+})();
diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
index 96dbe3089a3..29fd1fb256e 100644
--- a/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
+++ b/jstests/replsets/tenant_migration_concurrent_writes_on_donor.js
@@ -28,17 +28,7 @@ load("jstests/replsets/libs/tenant_migration_util.js");
const tenantMigrationTest = new TenantMigrationTest({
name: jsTestName(),
- sharedOptions: {
- setParameter: {
- // set ttlMonitorSleepSecs and tenantMigrationGarbageCollectionDelayMS to low values
- // to speed up cleanup between tests for shard merge
- ttlMonitorSleepSecs: 1,
- tenantMigrationGarbageCollectionDelayMS: 500,
-
- // Allow non-timestamped reads on donor after migration completes for testing.
- 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'})
- }
- }
+ quickGarbageCollection: true,
});
const donorRst = tenantMigrationTest.getDonorRst();
diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js
index 87627d50481..0a3ef6c87cd 100644
--- a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js
+++ b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js
@@ -20,7 +20,8 @@ load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
const donorRst = tenantMigrationTest.getDonorRst();
const donorPrimary = donorRst.getPrimary();
@@ -82,6 +83,9 @@ const kTenantId = "testTenantId";
// Write after the migration is forgotten.
assert.commandWorked(tenantCollOnRecipient.remove({_id: 1}));
+
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString,
+ migrationOpts.tenantId);
})();
(() => {
@@ -115,6 +119,9 @@ const kTenantId = "testTenantId";
// Write after the migration is forgotten.
assert.commandWorked(tenantCollOnRecipient.remove({_id: 1}));
+
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString,
+ migrationOpts.tenantId);
})();
(() => {
@@ -146,6 +153,9 @@ const kTenantId = "testTenantId";
// Write after the migration is forgotten.
assert.commandWorked(tenantCollOnRecipient.remove({_id: 1}));
+
+ tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString,
+ migrationOpts.tenantId);
})();
tenantMigrationTest.stop();
diff --git a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js
index 1d8c9717c77..4949ed3b71f 100644
--- a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js
+++ b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js
@@ -54,18 +54,39 @@ function generateUniqueTenantId() {
return kTenantIdPrefix + tenantCounter++;
}
-const donorRst = new ReplSetTest(
- {nodes: 1, name: 'donorRst', nodeOptions: TenantMigrationUtil.makeX509OptionsForTest().donor});
-
-donorRst.startSet();
-donorRst.initiate();
-
-const tenantMigrationTest0 = new TenantMigrationTest({name: jsTestName(), donorRst});
-
-const donorPrimary = donorRst.getPrimary();
-const recipientPrimary = tenantMigrationTest0.getRecipientPrimary();
-
-let numRecipientSyncDataCmdSent = 0;
+function setup() {
+ const {donor: donorNodeOptions} = TenantMigrationUtil.makeX509OptionsForTest();
+ donorNodeOptions.setParameter = donorNodeOptions.setParameter || {};
+ Object.assign(donorNodeOptions.setParameter, {
+ tenantMigrationGarbageCollectionDelayMS: 1 * 1000,
+ ttlMonitorSleepSecs: 1,
+ });
+ const donorRst = new ReplSetTest({
+ nodes: 1,
+ name: 'donorRst',
+ nodeOptions: donorNodeOptions,
+ });
+
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), donorRst, quickGarbageCollection: true});
+
+ const donorPrimary = donorRst.getPrimary();
+ const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
+
+ return {
+ tenantMigrationTest,
+ donorRst,
+ donorPrimary,
+ recipientPrimary,
+ teardown: function() {
+ tenantMigrationTest.stop();
+ donorRst.stopSet();
+ },
+ };
+}
// Test that a retry of a donorStartMigration command joins the existing migration that has
// completed but has not been garbage-collected.
@@ -76,13 +97,15 @@ let numRecipientSyncDataCmdSent = 0;
tenantId,
};
- TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration(migrationOpts));
- TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration(migrationOpts));
+ const {tenantMigrationTest, recipientPrimary, teardown} = setup();
+
+ TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts));
+ TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts));
// If the second donorStartMigration had started a duplicate migration, the recipient would have
// received four recipientSyncData commands instead of two.
- numRecipientSyncDataCmdSent += 2;
- checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent);
+ checkNumRecipientSyncDataCmdExecuted(recipientPrimary, 2);
+ teardown();
})();
// Test that a retry of a donorStartMigration command joins the ongoing migration.
@@ -93,20 +116,22 @@ let numRecipientSyncDataCmdSent = 0;
tenantId,
};
- assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts));
- assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts));
+ const {tenantMigrationTest, recipientPrimary, teardown} = setup();
+
+ assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
+ assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts));
TenantMigrationTest.assertCommitted(
- tenantMigrationTest0.waitForMigrationToComplete(migrationOpts));
+ tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
TenantMigrationTest.assertCommitted(
- tenantMigrationTest0.waitForMigrationToComplete(migrationOpts));
+ tenantMigrationTest.waitForMigrationToComplete(migrationOpts));
// If the second donorStartMigration had started a duplicate migration, the recipient would have
// received four recipientSyncData commands instead of two.
- numRecipientSyncDataCmdSent += 2;
- checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent);
+ checkNumRecipientSyncDataCmdExecuted(recipientPrimary, 2);
- assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts.migrationIdString));
+ assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ teardown();
})();
/**
@@ -114,8 +139,13 @@ let numRecipientSyncDataCmdSent = 0;
* donorStartMigration command to start a migration that conflicts with an existing migration that
* has committed but not garbage-collected (i.e. the donor has not received donorForgetMigration).
*/
-function testStartingConflictingMigrationAfterInitialMigrationCommitted(
- tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) {
+function testStartingConflictingMigrationAfterInitialMigrationCommitted({
+ tenantMigrationTest0,
+ migrationOpts0,
+ tenantMigrationTest1,
+ migrationOpts1,
+ donorPrimary,
+}) {
TenantMigrationTest.assertCommitted(tenantMigrationTest0.runMigration(
migrationOpts0, false /* retryOnRetryableErrors */, false /* automaticForgetMigration */));
const res1 = assert.commandFailedWithCode(tenantMigrationTest1.runMigration(migrationOpts1),
@@ -141,14 +171,21 @@ function testStartingConflictingMigrationAfterInitialMigrationCommitted(
}).length);
}
assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts0.migrationIdString));
+ tenantMigrationTest0.waitForMigrationGarbageCollection(migrationOpts0.migrationIdString,
+ migrationOpts0.tenantId);
}
/**
* Tests that if the client runs multiple donorStartMigration commands that would start conflicting
* migrations, only one of the migrations will start and succeed.
*/
-function testConcurrentConflictingMigrations(
- tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) {
+function testConcurrentConflictingMigrations({
+ tenantMigrationTest0,
+ migrationOpts0,
+ tenantMigrationTest1,
+ migrationOpts1,
+ donorPrimary,
+}) {
const res0 = tenantMigrationTest0.startMigration(migrationOpts0);
const res1 = tenantMigrationTest1.startMigration(migrationOpts1);
@@ -207,73 +244,112 @@ function testConcurrentConflictingMigrations(
// Test migrations with different migrationIds but identical settings.
(() => {
- let makeTestParams = () => {
+ const {tenantMigrationTest, donorPrimary, teardown} = setup();
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
tenantId: generateUniqueTenantId() + "DiffMigrationId",
};
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
migrationOpts1.migrationIdString = extractUUIDFromObject(UUID());
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
+ return {
+ tenantMigrationTest0: tenantMigrationTest,
+ migrationOpts0,
+ tenantMigrationTest1: tenantMigrationTest,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
+ teardown();
})();
// Test reusing a migrationId for different migration settings.
// Test different tenantIds.
(() => {
- let makeTestParams = () => {
+ const {tenantMigrationTest, donorPrimary, teardown} = setup();
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
tenantId: generateUniqueTenantId() + "DiffTenantId",
};
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
migrationOpts1.tenantId = generateUniqueTenantId() + "DiffTenantId";
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
+ return {
+ tenantMigrationTest0: tenantMigrationTest,
+ migrationOpts0,
+ tenantMigrationTest1: tenantMigrationTest,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
+ teardown();
})();
// Test different recipient connection strings.
(() => {
+ const {
+ tenantMigrationTest: tenantMigrationTest0,
+ donorRst,
+ donorPrimary,
+ teardown,
+ } = setup();
+
const tenantMigrationTest1 = new TenantMigrationTest({name: `${jsTestName()}1`, donorRst});
- let makeTestParams = () => {
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
- tenantId: generateUniqueTenantId() + "DiffRecipientConnString",
+ tenantId: `${generateUniqueTenantId()}DiffRecipientConnString`,
};
// The recipient connection string will be populated by the TenantMigrationTest fixture, so
// no need to set it here.
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1];
+ return {
+ tenantMigrationTest0,
+ migrationOpts0,
+ tenantMigrationTest1,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
tenantMigrationTest1.stop();
+ teardown();
})();
// Test different cloning read preference.
(() => {
- let makeTestParams = () => {
+ const {tenantMigrationTest, donorPrimary, teardown} = setup();
+
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
tenantId: generateUniqueTenantId() + "DiffReadPref",
};
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
migrationOpts1.readPreference = {mode: "secondary"};
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
+ return {
+ tenantMigrationTest0: tenantMigrationTest,
+ migrationOpts0,
+ tenantMigrationTest1: tenantMigrationTest,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
+ teardown();
})();
const kDonorCertificateAndPrivateKey =
@@ -287,7 +363,9 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi
// Test different donor certificates.
(() => {
- let makeTestParams = () => {
+ const {tenantMigrationTest, donorPrimary, teardown} = setup();
+
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
tenantId: generateUniqueTenantId() + "DiffDonorCertificate",
@@ -296,16 +374,25 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi
};
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
migrationOpts1.donorCertificateForRecipient = kExpiredDonorCertificateAndPrivateKey;
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
+ return {
+ tenantMigrationTest0: tenantMigrationTest,
+ migrationOpts0,
+ tenantMigrationTest1: tenantMigrationTest,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
+ teardown();
})();
// Test different recipient certificates.
(() => {
- let makeTestParams = () => {
+ const {tenantMigrationTest, donorPrimary, teardown} = setup();
+
+ const makeTestParams = () => {
const migrationOpts0 = {
migrationIdString: extractUUIDFromObject(UUID()),
tenantId: generateUniqueTenantId() + "DiffRecipientCertificate",
@@ -314,13 +401,17 @@ const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertifi
};
const migrationOpts1 = Object.extend({}, migrationOpts0, true);
migrationOpts1.recipientCertificateForDonor = kExpiredRecipientCertificateAndPrivateKey;
- return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
+ return {
+ tenantMigrationTest0: tenantMigrationTest,
+ migrationOpts0,
+ tenantMigrationTest1: tenantMigrationTest,
+ migrationOpts1,
+ donorPrimary,
+ };
};
- testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
- testConcurrentConflictingMigrations(...makeTestParams());
+ testStartingConflictingMigrationAfterInitialMigrationCommitted(makeTestParams());
+ testConcurrentConflictingMigrations(makeTestParams());
+ teardown();
})();
-
-tenantMigrationTest0.stop();
-donorRst.stopSet();
})();
diff --git a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js
index 6989a35ff99..4edff2c8f39 100644
--- a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js
+++ b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js
@@ -4,6 +4,7 @@
* @tags: [
* incompatible_with_eft,
* incompatible_with_macos,
+ * requires_fcv_52,
* incompatible_with_windows_tls,
* requires_majority_read_concern,
* requires_persistence,
@@ -123,24 +124,24 @@ assert.commandWorked(primary.adminCommand({
recipientSyncDataThread0.start();
recipientSyncDataThread1.start();
- jsTestLog("Waiting until both conflicting instances get started and hit the failPoint.");
+ jsTestLog("Waiting until one gets started and hits the failPoint.");
assert.commandWorked(primary.adminCommand({
waitForFailPoint: "pauseBeforeRunTenantMigrationRecipientInstance",
- timesEntered: fpPauseBeforeRunTenantMigrationRecipientInstance.timesEntered + 2,
+ timesEntered: fpPauseBeforeRunTenantMigrationRecipientInstance.timesEntered + 1,
maxTimeMS: kDefaultWaitForFailPointTimeout
}));
- // Two instances are expected as the tenantId conflict is still unresolved.
+ // One instance is expected as the tenantId conflict is still unresolved.
jsTestLog("Fetching current operations before conflict is resolved.");
const currentOpEntriesBeforeInsert = getTenantMigrationRecipientCurrentOpEntries(
primary, {desc: "tenant recipient migration", tenantId});
- assert.eq(2, currentOpEntriesBeforeInsert.length, tojson(currentOpEntriesBeforeInsert));
+ assert.eq(1, currentOpEntriesBeforeInsert.length, tojson(currentOpEntriesBeforeInsert));
jsTestLog("Unblocking the tenant migration instance from persisting the state doc.");
fpPauseBeforeRunTenantMigrationRecipientInstance.off();
- // Wait for both the conflicting instances to complete. Although both will "complete", one will
- // return with ErrorCodes.ConflictingOperationInProgress, and the other with a
+ // Check responses for both commands. One will return with
+ // ErrorCodes.ConflictingOperationInProgress, and the other with a
// TestData.stopFailPointErrorCode (a failpoint indicating that we have persisted the document).
const res0 = assert.commandFailed(recipientSyncDataThread0.returnData());
const res1 = assert.commandFailed(recipientSyncDataThread1.returnData());
diff --git a/jstests/replsets/tenant_migration_donor_abort_state_transition.js b/jstests/replsets/tenant_migration_donor_abort_state_transition.js
index 1592d3554bb..cfbb2a73588 100644
--- a/jstests/replsets/tenant_migration_donor_abort_state_transition.js
+++ b/jstests/replsets/tenant_migration_donor_abort_state_transition.js
@@ -20,16 +20,17 @@ load("jstests/replsets/libs/tenant_migration_util.js");
const kTenantIdPrefix = "testTenantId";
-const tenantMigrationTest = new TenantMigrationTest({
- name: jsTestName(),
- sharedOptions: {setParameter: {tenantMigrationGarbageCollectionDelayMS: 0}}
-});
-
/**
* Starts a migration and forces the write to insert the donor's state doc to abort on the first few
* tries. Asserts that the migration still completes successfully.
*/
-function testAbortInitialState(donorRst) {
+function testAbortInitialState() {
+ const tenantMigrationTest = new TenantMigrationTest({
+ name: jsTestName(),
+ quickGarbageCollection: true,
+ });
+ const donorRst = tenantMigrationTest.getDonorRst();
+
const donorPrimary = donorRst.getPrimary();
// Force the storage transaction for the insert to abort prior to inserting the WiredTiger
@@ -66,6 +67,7 @@ function testAbortInitialState(donorRst) {
donorRst.nodes, migrationId, tenantId, TenantMigrationTest.DonorState.kCommitted);
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ tenantMigrationTest.stop();
}
/**
@@ -74,10 +76,14 @@ function testAbortInitialState(donorRst) {
* reaching 'pauseFailPoint' to abort on the first few tries. Asserts that the migration still
* completes successfully.
*/
-function testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState) {
+function testAbortStateTransition(pauseFailPoint, setUpFailPoints, nextState) {
jsTest.log(`Test aborting the write to transition to state "${
nextState}" after reaching failpoint "${pauseFailPoint}"`);
+ const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
+ const donorRst = tenantMigrationTest.getDonorRst();
+
const donorPrimary = donorRst.getPrimary();
const tenantId = `${kTenantIdPrefix}-${nextState}`;
@@ -123,11 +129,11 @@ function testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nex
});
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ tenantMigrationTest.stop();
}
-const donorRst = tenantMigrationTest.getDonorRst();
jsTest.log("Test aborting donor's state doc insert");
-testAbortInitialState(donorRst);
+testAbortInitialState();
jsTest.log("Test aborting donor's state doc update");
[{
@@ -143,8 +149,6 @@ jsTest.log("Test aborting donor's state doc update");
setUpFailPoints: ["abortTenantMigrationBeforeLeavingBlockingState"],
nextState: TenantMigrationTest.DonorState.kAborted
}].forEach(({pauseFailPoint, setUpFailPoints = [], nextState}) => {
- testAbortStateTransition(donorRst, pauseFailPoint, setUpFailPoints, nextState);
+ testAbortStateTransition(pauseFailPoint, setUpFailPoints, nextState);
});
-
-tenantMigrationTest.stop();
}());
diff --git a/jstests/replsets/tenant_migration_donor_kill_op_retry.js b/jstests/replsets/tenant_migration_donor_kill_op_retry.js
index 41617b027a6..e81a1699212 100644
--- a/jstests/replsets/tenant_migration_donor_kill_op_retry.js
+++ b/jstests/replsets/tenant_migration_donor_kill_op_retry.js
@@ -37,8 +37,6 @@ function makeTenantId() {
return kTenantIdPrefix + testNum++;
}
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
-
{
// This section tests behavior in the middle of a tenant migration.
let fpNames = [
@@ -51,6 +49,8 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
"\" to test that the migration will retry the " +
"operation at the failpoint if a killOp is issued.");
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+
const migrationOpts = {
migrationIdString: extractUUIDFromObject(UUID()),
recipientConnString: tenantMigrationTest.getRecipientConnString(),
@@ -82,6 +82,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
TenantMigrationTest.assertCommitted(runMigrationThread.returnData());
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ tenantMigrationTest.stop();
}
}
@@ -92,6 +93,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
"pauseTenantMigrationBeforeCreatingExternalKeysTTLIndex"
];
for (let fpName of fpNames) {
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
tenantMigrationTest.getDonorRst().stopSet();
tenantMigrationTest.getDonorRst().startSet(Object.assign({}, migrationX509Options.donor, {
setParameter: {['failpoint.' + fpName]: tojson({mode: 'alwaysOn'})}
@@ -137,30 +139,32 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
TenantMigrationTest.assertCommitted(runMigrationThread.returnData());
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ tenantMigrationTest.stop();
}
}
{
- // This section is testing behavior during garbage collection.
- tenantMigrationTest.getDonorRst().stopSet();
- tenantMigrationTest.getDonorRst().startSet(
- Object.assign({}, migrationX509Options.donor, {setParameter: garbageCollectionOpts}));
- tenantMigrationTest.getDonorRst().initiate();
- TenantMigrationUtil.createTenantMigrationRecipientRoleIfNotExist(
- tenantMigrationTest.getDonorRst());
-
- tenantMigrationTest.getRecipientRst().stopSet();
- tenantMigrationTest.getRecipientRst().startSet(
- Object.assign({}, migrationX509Options.recipient, {setParameter: garbageCollectionOpts}));
- tenantMigrationTest.getRecipientRst().initiate();
- TenantMigrationUtil.createTenantMigrationDonorRoleIfNotExist(
- tenantMigrationTest.getRecipientRst());
-
let fpNames = [
"pauseTenantMigrationDonorBeforeMarkingStateGarbageCollectable",
"pauseTenantMigrationBeforeMarkingExternalKeysGarbageCollectable"
];
for (let fpName of fpNames) {
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+ // This section is testing behavior during garbage collection.
+ tenantMigrationTest.getDonorRst().stopSet();
+ tenantMigrationTest.getDonorRst().startSet(
+ Object.assign({}, migrationX509Options.donor, {setParameter: garbageCollectionOpts}));
+ tenantMigrationTest.getDonorRst().initiate();
+ TenantMigrationUtil.createTenantMigrationRecipientRoleIfNotExist(
+ tenantMigrationTest.getDonorRst());
+
+ tenantMigrationTest.getRecipientRst().stopSet();
+ tenantMigrationTest.getRecipientRst().startSet(Object.assign(
+ {}, migrationX509Options.recipient, {setParameter: garbageCollectionOpts}));
+ tenantMigrationTest.getRecipientRst().initiate();
+ TenantMigrationUtil.createTenantMigrationDonorRoleIfNotExist(
+ tenantMigrationTest.getRecipientRst());
+
jsTestLog(
"Setting failpoint \"" + fpName +
"\" during migration garbage collection to test that the migration will retry the " +
@@ -201,7 +205,7 @@ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
fp.off();
forgetMigrationThread.join();
tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, tenantId);
+ tenantMigrationTest.stop();
}
}
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migration_donor_retry.js b/jstests/replsets/tenant_migration_donor_retry.js
index 902005cdd38..1fb1a36c526 100644
--- a/jstests/replsets/tenant_migration_donor_retry.js
+++ b/jstests/replsets/tenant_migration_donor_retry.js
@@ -31,20 +31,27 @@ const garbageCollectionOpts = {
ttlMonitorSleepSecs: 1
};
-const donorRst = new ReplSetTest({
- name: "donorRst",
- nodes: 1,
- nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor,
- {setParameter: garbageCollectionOpts})
-});
-
-donorRst.startSet();
-donorRst.initiate();
+function setup() {
+ const donorRst = new ReplSetTest({
+ name: "donorRst",
+ nodes: 1,
+ nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor,
+ {setParameter: garbageCollectionOpts})
+ });
-const tenantMigrationTest = new TenantMigrationTest(
- {name: jsTestName(), donorRst: donorRst, sharedOptions: {setParameter: garbageCollectionOpts}});
-const donorPrimary = tenantMigrationTest.getDonorPrimary();
-const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const tenantMigrationTest = new TenantMigrationTest(
+ {name: jsTestName(), donorRst: donorRst, quickGarbageCollection: true});
+ return {
+ tenantMigrationTest,
+ teardown: function() {
+ donorRst.stopSet();
+ tenantMigrationTest.stop();
+ },
+ };
+}
function makeTenantId() {
return kTenantIdPrefix + testNum++;
@@ -55,7 +62,7 @@ function makeTenantId() {
* recipientSyncData command to fail with the given 'errorCode', and asserts the donor retries on
* that error and is able to commit.
*/
-function testDonorRetryRecipientSyncDataCmdOnError(errorCode, failMode) {
+function testDonorRetryRecipientSyncDataCmdOnError(tenantMigrationTest, errorCode, failMode) {
const recipientPrimary = tenantMigrationTest.getRecipientPrimary();
const tenantId = makeTenantId();
@@ -95,7 +102,7 @@ function testDonorRetryRecipientSyncDataCmdOnError(errorCode, failMode) {
* recipientForgetMigration command to fail with the given 'errorCode', and asserts the donor
* retries on that error and commits.
*/
-function testDonorRetryRecipientForgetMigrationCmdOnError(errorCode) {
+function testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, errorCode) {
const tenantId = makeTenantId();
const migrationId = UUID();
const migrationOpts = {
@@ -127,68 +134,88 @@ function testDonorRetryRecipientForgetMigrationCmdOnError(errorCode) {
(() => {
jsTest.log(
"Test that the donor retries recipientSyncData (to make the recipient start cloning) on recipient stepdown errors");
+ const {tenantMigrationTest, teardown} = setup();
- const migrationId =
- testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.NotWritablePrimary, {times: 1});
+ const migrationId = testDonorRetryRecipientSyncDataCmdOnError(
+ tenantMigrationTest, ErrorCodes.NotWritablePrimary, {times: 1});
- const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
+ const configDonorsColl =
+ tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
assert.eq(TenantMigrationTest.DonorState.kCommitted,
configDonorsColl.findOne({_id: migrationId}).state);
+ teardown();
})();
(() => {
jsTest.log(
"Test that the donor retries recipientSyncData (to make the recipient start cloning) on recipient shutdown errors");
+ const {tenantMigrationTest, teardown} = setup();
- const migrationId =
- testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.ShutdownInProgress, {times: 1});
+ const migrationId = testDonorRetryRecipientSyncDataCmdOnError(
+ tenantMigrationTest, ErrorCodes.ShutdownInProgress, {times: 1});
- const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
+ const configDonorsColl =
+ tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
assert.eq(TenantMigrationTest.DonorState.kCommitted,
configDonorsColl.findOne({_id: migrationId}).state);
+ teardown();
})();
(() => {
jsTest.log(
"Test that the donor retries recipientSyncData (with returnAfterReachingDonorTimestamp) on stepdown errors");
+ const {tenantMigrationTest, teardown} = setup();
- const migrationId =
- testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.NotWritablePrimary, {skip: 1});
+ const migrationId = testDonorRetryRecipientSyncDataCmdOnError(
+ tenantMigrationTest, ErrorCodes.NotWritablePrimary, {skip: 1});
- const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
+ const configDonorsColl =
+ tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
assert.eq(TenantMigrationTest.DonorState.kCommitted,
configDonorsColl.findOne({_id: migrationId}).state);
+ teardown();
})();
(() => {
jsTest.log(
"Test that the donor retries recipientSyncData (with returnAfterReachingDonorTimestamp) on recipient shutdown errors");
+ const {tenantMigrationTest, teardown} = setup();
- const migrationId =
- testDonorRetryRecipientSyncDataCmdOnError(ErrorCodes.ShutdownInProgress, {skip: 1});
+ const migrationId = testDonorRetryRecipientSyncDataCmdOnError(
+ tenantMigrationTest, ErrorCodes.ShutdownInProgress, {skip: 1});
- const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
+ const configDonorsColl =
+ tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
assert.eq(TenantMigrationTest.DonorState.kCommitted,
configDonorsColl.findOne({_id: migrationId}).state);
+ teardown();
})();
(() => {
jsTest.log("Test that the donor retries recipientForgetMigration on stepdown errors");
- testDonorRetryRecipientForgetMigrationCmdOnError(ErrorCodes.NotWritablePrimary);
+ const {tenantMigrationTest, teardown} = setup();
+ testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest,
+ ErrorCodes.NotWritablePrimary);
+ teardown();
})();
(() => {
jsTest.log("Test that the donor retries recipientForgetMigration on shutdown errors");
- testDonorRetryRecipientForgetMigrationCmdOnError(ErrorCodes.ShutdownInProgress);
+ const {tenantMigrationTest, teardown} = setup();
+ testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest,
+ ErrorCodes.ShutdownInProgress);
+ teardown();
})();
(() => {
jsTest.log("Test that the donor retries recipientForgetMigration on interruption errors");
+ const {tenantMigrationTest, teardown} = setup();
// Test an error code that is in the 'Interruption' category but not in the 'isRetriable'
// category.
const interruptionErrorCode = ErrorCodes.MaxTimeMSExpired;
assert(ErrorCodes.isInterruption(interruptionErrorCode));
- testDonorRetryRecipientForgetMigrationCmdOnError(interruptionErrorCode);
+ testDonorRetryRecipientForgetMigrationCmdOnError(tenantMigrationTest, interruptionErrorCode);
+ teardown();
})();
// Each donor state doc is updated three times throughout the lifetime of a tenant migration:
@@ -201,6 +228,7 @@ const kWriteErrorTimeMS = 50;
(() => {
jsTest.log("Test that the donor retries state doc insert on retriable errors");
+ const {tenantMigrationTest, teardown} = setup();
const tenantId = makeTenantId();
const migrationId = UUID();
@@ -210,7 +238,7 @@ const kWriteErrorTimeMS = 50;
recipientConnString: tenantMigrationTest.getRecipientConnString(),
};
- let fp = configureFailPoint(donorPrimary, "failCollectionInserts", {
+ let fp = configureFailPoint(tenantMigrationTest.getDonorPrimary(), "failCollectionInserts", {
collectionNS: TenantMigrationTest.kConfigDonorsNS,
});
@@ -230,15 +258,18 @@ const kWriteErrorTimeMS = 50;
migrationThread.join();
TenantMigrationTest.assertCommitted(migrationThread.returnData());
- const configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
+ const configDonorsColl =
+ tenantMigrationTest.getDonorPrimary().getCollection(TenantMigrationTest.kConfigDonorsNS);
const donorStateDoc = configDonorsColl.findOne({_id: migrationId});
assert.eq(TenantMigrationTest.DonorState.kCommitted, donorStateDoc.state);
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
+ teardown();
})();
(() => {
jsTest.log("Test that the donor retries state doc update on retriable errors");
+ const {tenantMigrationTest, teardown} = setup();
const tenantId = kTenantIdPrefix + "RetryOnStateDocUpdateError";
const migrationId = UUID();
@@ -251,7 +282,7 @@ const kWriteErrorTimeMS = 50;
const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst());
// Use a random number of skips to fail a random update to config.tenantMigrationDonors.
- const fp = configureFailPoint(donorPrimary,
+ const fp = configureFailPoint(tenantMigrationTest.getDonorPrimary(),
"failCollectionUpdates",
{
collectionNS: TenantMigrationTest.kConfigDonorsNS,
@@ -274,19 +305,17 @@ const kWriteErrorTimeMS = 50;
fp.off();
migrationThread.join();
- const donorStateDoc =
- donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS).findOne({_id: migrationId});
+ const donorStateDoc = tenantMigrationTest.getDonorPrimary()
+ .getCollection(TenantMigrationTest.kConfigDonorsNS)
+ .findOne({_id: migrationId});
assert.eq(donorStateDoc.state, TenantMigrationTest.DonorState.kCommitted);
assert(donorStateDoc.expireAt);
// Check that the recipient state doc is also correctly marked as garbage collectable.
- const recipientStateDoc =
- recipientPrimary.getCollection(TenantMigrationTest.kConfigRecipientsNS).findOne({
- _id: migrationId
- });
+ const recipientStateDoc = tenantMigrationTest.getRecipientPrimary()
+ .getCollection(TenantMigrationTest.kConfigRecipientsNS)
+ .findOne({_id: migrationId});
assert(recipientStateDoc.expireAt);
+ teardown();
})();
-
-donorRst.stopSet();
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migration_drop_state_doc_collection.js b/jstests/replsets/tenant_migration_drop_state_doc_collection.js
index 97606a49f2f..cc2a1b3d4ae 100644
--- a/jstests/replsets/tenant_migration_drop_state_doc_collection.js
+++ b/jstests/replsets/tenant_migration_drop_state_doc_collection.js
@@ -162,19 +162,14 @@ function testDroppingStateDocCollections(tenantMigrationTest, fpName, {
}
}
-const tenantMigrationTest = new TenantMigrationTest({
- name: jsTestName(),
- sharedOptions: {
- setParameter: {
- tenantMigrationGarbageCollectionDelayMS: 1,
- ttlMonitorSleepSecs: 1,
- }
- },
- initiateRstWithHighElectionTimeout: false
-});
-
jsTest.log("Test dropping donor and recipient state doc collections during a migration.");
kMigrationFpNames.forEach(fpName => {
+ const tenantMigrationTest = new TenantMigrationTest({
+ name: jsTestName(),
+ quickGarbageCollection: true,
+ initiateRstWithHighElectionTimeout: false
+ });
+
testDroppingStateDocCollections(
tenantMigrationTest, fpName, {dropDonorsCollection: true, dropRecipientsCollection: true});
@@ -219,7 +214,6 @@ kMigrationFpNames.forEach(fpName => {
expectedForgetMigrationError:
originalMigrationStartedOnRecipient ? ErrorCodes.ConflictingOperationInProgress : null
});
+ tenantMigrationTest.stop();
});
-
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migration_external_keys_ttl.js b/jstests/replsets/tenant_migration_external_keys_ttl.js
index 65f06b338ac..60835a3ac2d 100644
--- a/jstests/replsets/tenant_migration_external_keys_ttl.js
+++ b/jstests/replsets/tenant_migration_external_keys_ttl.js
@@ -99,15 +99,32 @@ function makeTestParams() {
//
(() => {
- const tmt = new TenantMigrationTest(
- {name: jsTestName(), sharedOptions: {setParameter: ttlMonitorOptions}});
-
- // Verify the external keys TTL index is created on both replica sets on stepup.
- waitForExternalKeysTTLIndex(tmt.getDonorPrimary());
- waitForExternalKeysTTLIndex(tmt.getRecipientPrimary());
+ function setup() {
+ const tmt = new TenantMigrationTest(
+ {name: jsTestName(), sharedOptions: {setParameter: ttlMonitorOptions}});
+
+ // Verify the external keys TTL index is created on both replica sets on stepup.
+ waitForExternalKeysTTLIndex(tmt.getDonorPrimary());
+ waitForExternalKeysTTLIndex(tmt.getRecipientPrimary());
+
+ return {
+ tmt,
+ teardown: function() {
+ tmt.stop();
+ },
+ };
+ }
- jsTestLog("Basic case with multiple migrations");
- {
+ (() => {
+ jsTestLog("Basic case with multiple migrations");
+ const {tmt, teardown} = setup();
+ if (TenantMigrationUtil.isShardMergeEnabled(tmt.getDonorPrimary().getDB("admin"))) {
+ // This test runs multiple concurrent migrations, which shard merge can't handle.
+ jsTestLog(
+ "Skip: featureFlagShardMerge is enabled and this test runs multiple concurrent migrations, which shard merge can't handle.");
+ teardown();
+ return;
+ }
const [tenantId, migrationId, migrationOpts] = makeTestParams();
TenantMigrationTest.assertCommitted(tmt.runMigration(migrationOpts,
@@ -148,10 +165,12 @@ function makeTestParams() {
{migrationId: otherMigrationId, expectTTLValue: false});
verifyExternalKeys(tmt.getRecipientPrimary(),
{migrationId: otherMigrationId, expectTTLValue: false});
- }
+ teardown();
+ })();
jsTestLog("Verify the TTL value is respected and keys are eventually reaped");
{
+ const {tmt, teardown} = setup();
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const lowerExternalKeysBufferSecs = 5;
@@ -190,9 +209,8 @@ function makeTestParams() {
setTenantMigrationExpirationParams(tmt.getRecipientPrimary(),
origRecipientStateDocExpirationParam,
origRecipientKeysExpirationParam);
+ teardown();
}
-
- tmt.stop();
})();
//
@@ -200,28 +218,43 @@ function makeTestParams() {
//
(() => {
- const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
- const donorRst = new ReplSetTest({
- nodes: 3,
- name: "donorRst",
- nodeOptions: Object.assign(migrationX509Options.donor, {setParameter: ttlMonitorOptions})
- });
- donorRst.startSet();
- donorRst.initiate();
-
- const recipientRst = new ReplSetTest({
- nodes: 3,
- name: "recipientRst",
- nodeOptions:
- Object.assign(migrationX509Options.recipient, {setParameter: ttlMonitorOptions})
- });
- recipientRst.startSet();
- recipientRst.initiate();
-
- const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+ function setup() {
+ const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
+ const donorRst = new ReplSetTest({
+ nodes: 3,
+ name: "donorRst",
+ nodeOptions:
+ Object.assign(migrationX509Options.donor, {setParameter: ttlMonitorOptions})
+ });
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const recipientRst = new ReplSetTest({
+ nodes: 3,
+ name: "recipientRst",
+ nodeOptions:
+ Object.assign(migrationX509Options.recipient, {setParameter: ttlMonitorOptions})
+ });
+ recipientRst.startSet();
+ recipientRst.initiate();
+
+ const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+
+ return {
+ tmt,
+ donorRst,
+ recipientRst,
+ teardown: function() {
+ donorRst.stopSet();
+ recipientRst.stopSet();
+ tmt.stop();
+ },
+ };
+ }
jsTestLog("Donor failover before receiving forgetMigration");
{
+ const {tmt, teardown} = setup();
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const donorPrimary = tmt.getDonorPrimary();
const fp =
@@ -248,10 +281,12 @@ function makeTestParams() {
// buffer is 1 day so the keys will not have been deleted.
verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true});
verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true});
+ teardown();
}
jsTestLog("Recipient failover before receiving forgetMigration");
{
+ const {tmt, teardown} = setup();
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const recipientPrimary = tmt.getRecipientPrimary();
const fp = configureFailPoint(recipientPrimary,
@@ -279,11 +314,13 @@ function makeTestParams() {
// buffer is 1 day so the keys will not have been deleted.
verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true});
verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true});
+ teardown();
}
jsTestLog(
"Donor failover after receiving forgetMigration before marking keys garbage collectable");
{
+ const {tmt, donorRst, teardown} = setup();
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const donorPrimary = tmt.getDonorPrimary();
@@ -315,11 +352,13 @@ function makeTestParams() {
// buffer is 1 day so the keys will not have been deleted.
verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true});
verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true});
+ teardown();
}
jsTestLog(
"Recipient failover after receiving forgetMigration before marking keys garbage collectable");
{
+ const {tmt, donorRst, teardown} = setup();
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const recipientPrimary = tmt.getRecipientPrimary();
@@ -349,18 +388,19 @@ function makeTestParams() {
verifyExternalKeys(tmt.getDonorPrimary(), {migrationId, expectTTLValue: true});
verifyExternalKeys(tmt.getRecipientPrimary(), {migrationId, expectTTLValue: true});
- }
-
- // The next two cases expect the external keys to expire, so lower the expiration timeouts.
- const lowerExternalKeysBufferSecs = 5;
- const lowerStateDocExpirationMS = 500;
- for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) {
- setTenantMigrationExpirationParams(
- conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs);
+ teardown();
}
jsTestLog("Donor failover after receiving forgetMigration after updating keys.");
{
+ const {tmt, donorRst, recipientRst, teardown} = setup();
+ // this test expects the external keys to expire, so lower the expiration timeouts.
+ const lowerExternalKeysBufferSecs = 5;
+ const lowerStateDocExpirationMS = 500;
+ for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) {
+ setTenantMigrationExpirationParams(
+ conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs);
+ }
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const donorPrimary = tmt.getDonorPrimary();
@@ -394,10 +434,19 @@ function makeTestParams() {
fp.off();
assert.commandWorked(forgetMigrationThread.returnData());
+ teardown();
}
jsTestLog("Recipient failover after receiving forgetMigration after updating keys.");
{
+ const {tmt, donorRst, recipientRst, teardown} = setup();
+ // this test expects the external keys to expire, so lower the expiration timeouts.
+ const lowerExternalKeysBufferSecs = 5;
+ const lowerStateDocExpirationMS = 500;
+ for (let conn of [...donorRst.nodes, ...recipientRst.nodes]) {
+ setTenantMigrationExpirationParams(
+ conn, lowerStateDocExpirationMS, lowerExternalKeysBufferSecs);
+ }
const [tenantId, migrationId, migrationOpts] = makeTestParams();
const recipientPrimary = tmt.getRecipientPrimary();
@@ -433,10 +482,7 @@ function makeTestParams() {
// Eventually the donor's keys should be deleted too.
waitForExternalKeysToBeDeleted(tmt.getDonorPrimary(), migrationId);
+ teardown();
}
-
- donorRst.stopSet();
- recipientRst.stopSet();
- tmt.stop();
})();
})();
diff --git a/jstests/replsets/tenant_migration_filters_tenant_id.js b/jstests/replsets/tenant_migration_filters_tenant_id.js
index a897a65aa87..9a2b26cb5c9 100644
--- a/jstests/replsets/tenant_migration_filters_tenant_id.js
+++ b/jstests/replsets/tenant_migration_filters_tenant_id.js
@@ -17,8 +17,6 @@ load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
-
const tenantIdPrefix = "tenantId";
const baseDBName = "testDB";
const collName = "testColl";
@@ -31,6 +29,8 @@ const makeBaseTenantId = () => {
const runTest = (baseTenantId, dbName, shouldMatch) => {
jsTestLog(`Running tenant migration with dbName ${dbName} and tenantId ${baseTenantId}`);
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+
assert.eq(shouldMatch, TenantMigrationUtil.isNamespaceForTenant(baseTenantId, dbName));
tenantMigrationTest.insertDonorDB(dbName, collName);
@@ -45,6 +45,7 @@ const runTest = (baseTenantId, dbName, shouldMatch) => {
// migrated, so we can directly call it here.
tenantMigrationTest.verifyRecipientDB(baseTenantId, dbName, collName);
tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString);
+ tenantMigrationTest.stop();
};
const testCases = [
@@ -64,6 +65,4 @@ for (const {makeTenantId, shouldMatch} of testCases) {
const tenantId = makeTenantId(baseTenantId);
runTest(baseTenantId, `${tenantId}_${baseDBName}`, shouldMatch);
}
-
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js
index f7dc0fe4479..710d20f200d 100644
--- a/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js
+++ b/jstests/replsets/tenant_migration_recipient_initial_sync_cloning.js
@@ -21,18 +21,6 @@ load("jstests/replsets/libs/tenant_migration_test.js");
load('jstests/replsets/rslib.js'); // for waitForNewlyAddedRemovalForNodeToBeCommitted
const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
-const donorRst = new ReplSetTest({
- name: "donorRst",
- nodes: 1,
- nodeOptions: Object.assign(migrationX509Options.donor, {
- setParameter: {
- // Allow non-timestamped reads on donor after migration completes for testing.
- 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads': tojson({mode: 'alwaysOn'}),
- }
- })
-});
-donorRst.startSet();
-donorRst.initiate();
const testDBName = 'testDB';
const testCollName = 'testColl';
@@ -143,9 +131,23 @@ function restartNodeAndCheckStateDuringOplogApplication(
// 5. Steps up the restarted node as the recipient primary, lifts the recipient failpoint, and
// allows the migration to complete.
function runTestCase(tenantId, recipientFailpoint, checkMtab, restartNodeAndCheckStateFunction) {
+ const donorRst = new ReplSetTest({
+ name: "donorRst",
+ nodes: 1,
+ nodeOptions: Object.assign(migrationX509Options.donor, {
+ setParameter: {
+ // Allow non-timestamped reads on donor after migration completes for testing.
+ 'failpoint.tenantMigrationDonorAllowsNonTimestampedReads':
+ tojson({mode: 'alwaysOn'}),
+ }
+ })
+ });
+ donorRst.startSet();
+ donorRst.initiate();
+
const tenantMigrationTest = new TenantMigrationTest({
name: jsTestName(),
- donorRst: donorRst,
+ donorRst,
sharedOptions: {setParameter: {tenantApplierBatchSizeOps: 2}}
});
@@ -171,6 +173,7 @@ function runTestCase(tenantId, recipientFailpoint, checkMtab, restartNodeAndChec
assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString));
tenantMigrationTest.stop();
+ donorRst.stopSet();
}
// These two test cases are for before the mtab is created, and before the oplog applier has been
@@ -196,6 +199,4 @@ runTestCase('tenantId4',
"fpAfterWaitForRejectReadsBeforeTimestamp",
true /* checkMtab */,
restartNodeAndCheckStateWithoutOplogApplication);
-
-donorRst.stopSet();
})();
diff --git a/jstests/replsets/tenant_migration_recipient_rollback_recovery.js b/jstests/replsets/tenant_migration_recipient_rollback_recovery.js
index f011d9a3a7a..0f694c40912 100644
--- a/jstests/replsets/tenant_migration_recipient_rollback_recovery.js
+++ b/jstests/replsets/tenant_migration_recipient_rollback_recovery.js
@@ -30,20 +30,6 @@ const kGarbageCollectionDelayMS = 30 * 1000;
const migrationX509Options = TenantMigrationUtil.makeX509OptionsForTest();
-const donorRst = new ReplSetTest({
- name: "donorRst",
- nodes: 1,
- nodeOptions: Object.assign({}, migrationX509Options.donor, {
- setParameter: {
- tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS,
- ttlMonitorSleepSecs: 1,
- }
- })
-});
-donorRst.startSet();
-donorRst.initiate();
-const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
-
function makeMigrationOpts(tenantMigrationTest, migrationId, tenantId) {
return {
migrationIdString: extractUUIDFromObject(migrationId),
@@ -61,6 +47,21 @@ function makeMigrationOpts(tenantMigrationTest, migrationId, tenantId) {
* replication steady state.
*/
function testRollBack(setUpFunc, rollbackOpsFunc, steadyStateFunc) {
+ const donorRst = new ReplSetTest({
+ name: "donorRst",
+ nodes: 1,
+ nodeOptions: Object.assign({}, migrationX509Options.donor, {
+ setParameter: {
+ tenantMigrationGarbageCollectionDelayMS: kGarbageCollectionDelayMS,
+ ttlMonitorSleepSecs: 1,
+ }
+ })
+ });
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst);
+
const recipientRst = new ReplSetTest({
name: "recipientRst",
nodes: 3,
@@ -106,6 +107,7 @@ function testRollBack(setUpFunc, rollbackOpsFunc, steadyStateFunc) {
steadyStateFunc(tenantMigrationTest);
+ donorRst.stopSet();
recipientRst.stopSet();
}
@@ -333,6 +335,4 @@ testRollBackMarkingStateGarbageCollectable();
jsTest.log("Test roll back random");
testRollBackRandom();
-
-donorRst.stopSet();
}());
diff --git a/jstests/replsets/tenant_migration_recipient_ttl.js b/jstests/replsets/tenant_migration_recipient_ttl.js
index 20100d9ebf8..afab2032942 100644
--- a/jstests/replsets/tenant_migration_recipient_ttl.js
+++ b/jstests/replsets/tenant_migration_recipient_ttl.js
@@ -17,16 +17,8 @@ load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject().
load("jstests/replsets/libs/tenant_migration_test.js");
load("jstests/replsets/libs/tenant_migration_util.js");
-const kGarbageCollectionParams = {
- // Set the delay to 30s so that we can see the document vanish.
- tenantMigrationGarbageCollectionDelayMS: 30 * 1000,
-
- // Set the TTL monitor to run at a smaller interval to speed up the test.
- ttlMonitorSleepSecs: 1
-};
-
-const tenantMigrationTest = new TenantMigrationTest(
- {name: jsTestName(), sharedOptions: {setParameter: kGarbageCollectionParams}});
+const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
const kRecipientTTLIndexName = "TenantMigrationRecipientTTLIndex";
@@ -83,4 +75,4 @@ sleep(30000); // The garbage collection delay is 30s.
tenantMigrationTest.waitForMigrationGarbageCollection(kMigrationId, kTenantId);
tenantMigrationTest.stop();
-})(); \ No newline at end of file
+})();
diff --git a/jstests/replsets/tenant_migration_retry_session_migration.js b/jstests/replsets/tenant_migration_retry_session_migration.js
index 27e7069bca4..2f82cd4ea34 100644
--- a/jstests/replsets/tenant_migration_retry_session_migration.js
+++ b/jstests/replsets/tenant_migration_retry_session_migration.js
@@ -19,17 +19,8 @@ load("jstests/replsets/libs/tenant_migration_util.js");
load("jstests/replsets/rslib.js");
load("jstests/libs/uuid_util.js");
-const kGarbageCollectionParams = {
- // Set the delay before a donor state doc is garbage collected to be short to speed up
- // the test.
- tenantMigrationGarbageCollectionDelayMS: 3 * 1000,
-
- // Set the TTL monitor to run at a smaller interval to speed up the test.
- ttlMonitorSleepSecs: 1,
-};
-
-const tenantMigrationTest = new TenantMigrationTest(
- {name: jsTestName(), sharedOptions: {setParameter: kGarbageCollectionParams}});
+const tenantMigrationTest =
+ new TenantMigrationTest({name: jsTestName(), quickGarbageCollection: true});
const kTenantId = "testTenantId";
const kDbName = tenantMigrationTest.tenantDB(kTenantId, "testDB");
diff --git a/jstests/replsets/tenant_migration_x509.js b/jstests/replsets/tenant_migration_x509.js
index e0a5169a3f5..32621531735 100644
--- a/jstests/replsets/tenant_migration_x509.js
+++ b/jstests/replsets/tenant_migration_x509.js
@@ -22,7 +22,15 @@ function makeTestNs(tenantId) {
return {dbName: tenantId + "_testDb", collName: "testColl"};
}
-const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+function setup() {
+ const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()});
+ return {
+ tenantMigrationTest,
+ teardown: function() {
+ tenantMigrationTest.stop();
+ }
+ };
+}
const kDonorCertificateAndPrivateKey =
TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_donor.pem");
@@ -31,6 +39,7 @@ const kRecipientCertificateAndPrivateKey =
(() => {
jsTest.log("Test valid donor and recipient certificates");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "validCertificates";
const migrationOpts = {
@@ -45,10 +54,12 @@ const kRecipientCertificateAndPrivateKey =
TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts));
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, true /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid donor certificate, no header and trailer");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidDonorCertificateNoHeaderAndTrailer";
const migrationOpts = {
@@ -67,10 +78,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid donor certificate, use private key as certificate");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidDonorCertificatePrivateKeyAsCertificate";
const migrationOpts = {
@@ -89,10 +102,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid donor private key, no header and trailer");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidDonorPrivateKeyNoHeaderAndTrailer";
const migrationOpts = {
@@ -111,10 +126,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid donor private key, use certificate as private key");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidDonorPrivateKeyCertificateAsPrivateKey";
const migrationOpts = {
@@ -133,10 +150,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid donor certificate and private key pair");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidDonorCertificatePrivateKeyPair";
const migrationOpts = {
@@ -155,10 +174,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test expired donor certificate and key");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "expiredDonorCertificate";
const migrationOpts = {
@@ -175,10 +196,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid recipient certificate, no header and trailer");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidRecipientCertificateNoHeaderAndTrailer";
const migrationOpts = {
@@ -197,10 +220,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid recipient certificate, use private key as certificate");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidRecipientCertificatePrivateKeyAsCertificate";
const migrationOpts = {
@@ -219,10 +244,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid recipient private key, no header and trailer");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidRecipientPrivateKeyNoHeaderAndTrailer";
const migrationOpts = {
@@ -241,10 +268,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid recipient private key, use certificate as private key");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidRecipientPrivateKeyCertificateAsPrivateKey";
const migrationOpts = {
@@ -263,10 +292,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test expired recipient certificate and key");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "expiredRecipientCertificate";
const migrationOpts = {
@@ -283,10 +314,12 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
jsTest.log("Test invalid recipient certificate and private key pair");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "invalidRecipientCertificatePrivateKeyPair";
const migrationOpts = {
@@ -305,16 +338,17 @@ const kRecipientCertificateAndPrivateKey =
ErrorCodes.InvalidSSLConfiguration);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
if (!TestData.auth) {
jsTestLog("Skipping testing authorization since auth is not enabled");
- tenantMigrationTest.stop();
return;
}
(() => {
jsTest.log("Test donor certificate without the required privileges");
+ const {tenantMigrationTest, teardown} = setup();
const migrationId = UUID();
const tenantId = "donorCertificateInsufficientPrivileges";
const migrationOpts = {
@@ -331,9 +365,11 @@ if (!TestData.auth) {
ErrorCodes.Unauthorized);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
(() => {
+ const {tenantMigrationTest, teardown} = setup();
jsTest.log("Test recipient certificate without the required privileges");
const migrationId = UUID();
const tenantId = "recipientCertificateInsufficientPrivileges";
@@ -351,7 +387,6 @@ if (!TestData.auth) {
ErrorCodes.Unauthorized);
tenantMigrationTest.verifyRecipientDB(
tenantId, dbName, collName, false /* migrationCommitted */);
+ teardown();
})();
-
-tenantMigrationTest.stop();
})();
diff --git a/jstests/replsets/tenant_migrations_noop_writes.js b/jstests/replsets/tenant_migrations_noop_writes.js
index abae2350f4c..f4dd9f647e9 100644
--- a/jstests/replsets/tenant_migrations_noop_writes.js
+++ b/jstests/replsets/tenant_migrations_noop_writes.js
@@ -70,40 +70,52 @@ function runAfterClusterTimeRead(dbName, collName, operationTime, clusterTime, e
}
}
-const donorRst = new ReplSetTest({
- nodes: 3,
- name: "donor",
- settings: {chainingAllowed: false},
- nodeOptions: Object.assign(migrationX509Options.donor, {
- setParameter: {
- // To allow after test hooks to run without errors.
- "failpoint.tenantMigrationDonorAllowsNonTimestampedReads": tojson({mode: "alwaysOn"}),
- }
- })
-});
-donorRst.startSet();
-donorRst.initiate();
-
-const recipientRst = new ReplSetTest({
- nodes: 3,
- name: "recipient",
- settings: {chainingAllowed: false},
- nodeOptions: migrationX509Options.recipient
-});
-recipientRst.startSet();
-recipientRst.initiate();
-
-const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
-
-const donorPrimary = tmt.getDonorPrimary();
-const recipientPrimary = tmt.getRecipientPrimary();
+function setup() {
+ const donorRst = new ReplSetTest({
+ nodes: 3,
+ name: "donor",
+ settings: {chainingAllowed: false},
+ nodeOptions: Object.assign(migrationX509Options.donor, {
+ setParameter: {
+ // To allow after test hooks to run without errors.
+ "failpoint.tenantMigrationDonorAllowsNonTimestampedReads":
+ tojson({mode: "alwaysOn"}),
+ }
+ })
+ });
+ donorRst.startSet();
+ donorRst.initiate();
+
+ const recipientRst = new ReplSetTest({
+ nodes: 3,
+ name: "recipient",
+ settings: {chainingAllowed: false},
+ nodeOptions: migrationX509Options.recipient
+ });
+ recipientRst.startSet();
+ recipientRst.initiate();
+
+ const tmt = new TenantMigrationTest({name: jsTestName(), donorRst, recipientRst});
+
+ return {
+ tmt,
+ teardown: function() {
+ donorRst.stopSet();
+ recipientRst.stopSet();
+ tmt.stop();
+ },
+ };
+}
{
jsTestLog("Testing noops on the recipient");
+ const {tmt, teardown} = setup();
+
const [tenantId, migrationId, migrationOpts, tenantDbName] = makeTestParams();
const laggedRecipientSecondary = tmt.getRecipientRst().getSecondary();
- const fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState");
+ const fp =
+ configureFailPoint(tmt.getDonorPrimary(), "pauseTenantMigrationBeforeLeavingBlockingState");
//
// Run a migration, pausing after selecting a block timestamp to advance cluster time beyond it
@@ -120,14 +132,15 @@ const recipientPrimary = tmt.getRecipientPrimary();
// already been replicated by this point.
stopServerReplication(laggedRecipientSecondary);
- advanceClusterTime(donorPrimary, kUnrelatedDbName, collName);
+ advanceClusterTime(tmt.getDonorPrimary(), kUnrelatedDbName, collName);
- const donorRes =
- assert.commandWorked(donorPrimary.getDB(tenantDbName).runCommand({find: collName}));
+ const donorRes = assert.commandWorked(
+ tmt.getDonorPrimary().getDB(tenantDbName).runCommand({find: collName}));
assert(donorRes.operationTime, tojson(donorRes));
- assert.eq(timestampCmp(donorRes.operationTime, getBlockTimestamp(donorPrimary, tenantId)),
- 1,
- tojson(donorRes));
+ assert.eq(
+ timestampCmp(donorRes.operationTime, getBlockTimestamp(tmt.getDonorPrimary(), tenantId)),
+ 1,
+ tojson(donorRes));
fp.off();
TenantMigrationTest.assertCommitted(tmt.waitForMigrationToComplete(
@@ -140,7 +153,7 @@ const recipientPrimary = tmt.getRecipientPrimary();
// profiled so we use a fail point to detect it.
//
- const hangInNoopFp = configureFailPoint(recipientPrimary, "hangInAppendOplogNote");
+ const hangInNoopFp = configureFailPoint(tmt.getRecipientPrimary(), "hangInAppendOplogNote");
const awaitReadOnRecipient = startParallelShell(funWithArgs(runAfterClusterTimeRead,
tenantDbName,
collName,
@@ -153,14 +166,18 @@ const recipientPrimary = tmt.getRecipientPrimary();
restartServerReplication(laggedRecipientSecondary);
awaitReadOnRecipient();
+ teardown();
}
{
jsTestLog("Testing noops on the donor");
+ const {tmt, teardown} = setup();
+
const [tenantId, migrationId, migrationOpts, tenantDbName] = makeTestParams();
const laggedDonorSecondary = tmt.getDonorRst().getSecondary();
- const fp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState");
+ const fp =
+ configureFailPoint(tmt.getDonorPrimary(), "pauseTenantMigrationBeforeLeavingBlockingState");
//
// Commit a normal migration, but disable replication on a donor secondary before the commit so
@@ -182,12 +199,13 @@ const recipientPrimary = tmt.getRecipientPrimary();
// Advance cluster time on the recipient beyond the block timestamp.
//
- advanceClusterTime(recipientPrimary, kUnrelatedDbName, collName);
+ advanceClusterTime(tmt.getRecipientPrimary(), kUnrelatedDbName, collName);
- const recipientRes =
- assert.commandWorked(recipientPrimary.getDB(tenantDbName).runCommand({find: collName}));
+ const recipientRes = assert.commandWorked(
+ tmt.getRecipientPrimary().getDB(tenantDbName).runCommand({find: collName}));
assert(recipientRes.operationTime, tojson(recipientRes));
- assert.eq(timestampCmp(recipientRes.operationTime, getBlockTimestamp(donorPrimary, tenantId)),
+ assert.eq(timestampCmp(recipientRes.operationTime,
+ getBlockTimestamp(tmt.getDonorPrimary(), tenantId)),
1,
tojson(recipientRes));
@@ -199,7 +217,7 @@ const recipientPrimary = tmt.getRecipientPrimary();
// necessary to unblock tenant operations waiting for a cluster time > the block timestamp.
//
- const hangInNoopFp = configureFailPoint(donorPrimary, "hangInAppendOplogNote");
+ const hangInNoopFp = configureFailPoint(tmt.getDonorPrimary(), "hangInAppendOplogNote");
const awaitReadOnDonor = startParallelShell(funWithArgs(runAfterClusterTimeRead,
tenantDbName,
collName,
@@ -213,9 +231,6 @@ const recipientPrimary = tmt.getRecipientPrimary();
restartServerReplication(laggedDonorSecondary);
awaitReadOnDonor();
+ teardown();
}
-
-donorRst.stopSet();
-recipientRst.stopSet();
-tmt.stop();
})();
diff --git a/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp b/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp
index ebd71c3b423..9952d301d47 100644
--- a/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp
+++ b/src/mongo/db/commands/tenant_migration_recipient_cmds.cpp
@@ -117,24 +117,13 @@ public:
auto returnAfterReachingDonorTs = cmd.getReturnAfterReachingDonorTimestamp();
- try {
- if (!returnAfterReachingDonorTs) {
- return Response(
- recipientInstance->waitUntilMigrationReachesConsistentState(opCtx));
- }
-
- return Response(
- recipientInstance->waitUntilMigrationReachesReturnAfterReachingTimestamp(
- opCtx, *returnAfterReachingDonorTs));
- } catch (ExceptionFor<ErrorCodes::ConflictingOperationInProgress>& ex) {
- // A conflict may arise when inserting the recipientInstance's state document.
- // Since the conflict occurred at the insert stage, that means this instance's
- // tenantId conflicts with an existing instance's tenantId. Therefore, remove the
- // instance that was just created.
- // The status from this exception will be passed to the instance interrupt() method.
- recipientService->releaseInstance(stateDocBson["_id"].wrap(), ex.toStatus());
- throw;
+ if (!returnAfterReachingDonorTs) {
+ return Response(recipientInstance->waitUntilMigrationReachesConsistentState(opCtx));
}
+
+ return Response(
+ recipientInstance->waitUntilMigrationReachesReturnAfterReachingTimestamp(
+ opCtx, *returnAfterReachingDonorTs));
}
private:
diff --git a/src/mongo/db/repl/tenant_migration_donor_service.cpp b/src/mongo/db/repl/tenant_migration_donor_service.cpp
index 510e24e8772..70fc5c1384f 100644
--- a/src/mongo/db/repl/tenant_migration_donor_service.cpp
+++ b/src/mongo/db/repl/tenant_migration_donor_service.cpp
@@ -34,6 +34,7 @@
#include "mongo/client/connection_string.h"
#include "mongo/client/replica_set_monitor.h"
#include "mongo/config.h"
+#include "mongo/db/commands/tenant_migration_donor_cmds_gen.h"
#include "mongo/db/commands/tenant_migration_recipient_cmds_gen.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
@@ -148,20 +149,32 @@ void TenantMigrationDonorService::checkIfConflictsWithOtherInstances(
OperationContext* opCtx,
BSONObj initialState,
const std::vector<const repl::PrimaryOnlyService::Instance*>& existingInstances) {
- auto tenantId = initialState["tenantId"].valueStringData();
- // Any existing migration for this tenant must be aborted and garbage-collectable.
- for (auto& instance : existingInstances) {
- auto typedInstance = checked_cast<const TenantMigrationDonorService::Instance*>(instance);
- auto durableState = typedInstance->getDurableState();
+ auto stateDoc = tenant_migration_access_blocker::parseDonorStateDocument(initialState);
+ auto isNewShardMerge = stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge;
- if (typedInstance->getTenantId() == tenantId) {
- // If a migration with the same tenantId exists, it needs to be already in kAborted
- // state
+ for (auto& instance : existingInstances) {
+ auto existingTypedInstance =
+ checked_cast<const TenantMigrationDonorService::Instance*>(instance);
+ auto existingState = existingTypedInstance->getDurableState();
+ auto existingIsAborted = existingState &&
+ existingState->state == TenantMigrationDonorStateEnum::kAborted &&
+ existingState->expireAt;
+
+ uassert(ErrorCodes::ConflictingOperationInProgress,
+ str::stream() << "Cannot start a shard merge with existing migrations in progress",
+ !isNewShardMerge || existingIsAborted);
+
+ uassert(
+ ErrorCodes::ConflictingOperationInProgress,
+ str::stream() << "Cannot start a migration with an existing shard merge in progress",
+ existingTypedInstance->getProtocol() != MigrationProtocolEnum::kShardMerge ||
+ existingIsAborted);
+
+ // Any existing migration for this tenant must be aborted and garbage-collectable.
+ if (existingTypedInstance->getTenantId() == stateDoc.getTenantId()) {
uassert(ErrorCodes::ConflictingOperationInProgress,
- str::stream() << "tenant " << tenantId << " is already migrating",
- durableState &&
- durableState->state == TenantMigrationDonorStateEnum::kAborted &&
- durableState->expireAt);
+ str::stream() << "tenant " << stateDoc.getTenantId() << " is already migrating",
+ existingIsAborted);
}
}
}
diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp
index 25312c71eb4..9b154211be1 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp
+++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp
@@ -40,6 +40,7 @@
#include "mongo/client/replica_set_monitor_manager.h"
#include "mongo/config.h"
#include "mongo/db/client.h"
+#include "mongo/db/commands/tenant_migration_donor_cmds_gen.h"
#include "mongo/db/commands/test_commands_enabled.h"
#include "mongo/db/concurrency/write_conflict_exception.h"
#include "mongo/db/db_raii.h"
@@ -315,7 +316,25 @@ void TenantMigrationRecipientService::checkIfConflictsWithOtherInstances(
OperationContext* opCtx,
BSONObj initialStateDoc,
const std::vector<const PrimaryOnlyService::Instance*>& existingInstances) {
- // TODO (SERVER-59786): check for conflicts here, not RecipientSyncDataCmd::typedRun.
+ auto tenantId = initialStateDoc["tenantId"].valueStringData();
+
+ for (auto& instance : existingInstances) {
+ auto existingTypedInstance =
+ checked_cast<const TenantMigrationRecipientService::Instance*>(instance);
+ auto existingState = existingTypedInstance->getState();
+ auto isDone = existingState.getState() == TenantMigrationRecipientStateEnum::kDone &&
+ existingState.getExpireAt();
+
+ uassert(ErrorCodes::ConflictingOperationInProgress,
+ "an existing shard merge is in progress",
+ isDone ||
+ (existingTypedInstance->getProtocol() != MigrationProtocolEnum::kShardMerge &&
+ existingState.getProtocol() != MigrationProtocolEnum::kShardMerge));
+
+ uassert(ErrorCodes::ConflictingOperationInProgress,
+ str::stream() << "tenant " << tenantId << " is already migrating",
+ isDone || existingTypedInstance->getTenantId() != tenantId);
+ }
}
std::shared_ptr<PrimaryOnlyService::Instance> TenantMigrationRecipientService::constructInstance(
@@ -2670,5 +2689,10 @@ const MigrationProtocolEnum& TenantMigrationRecipientService::Instance::getProto
return _protocol;
}
+const TenantMigrationRecipientDocument TenantMigrationRecipientService::Instance::getState() const {
+ stdx::lock_guard lk(_mutex);
+ return _stateDoc;
+}
+
} // namespace repl
} // namespace mongo
diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h
index 2de82942985..f35f1a93f19 100644
--- a/src/mongo/db/repl/tenant_migration_recipient_service.h
+++ b/src/mongo/db/repl/tenant_migration_recipient_service.h
@@ -148,6 +148,11 @@ public:
*/
const MigrationProtocolEnum& getProtocol() const;
+ /*
+ * Returns the recipient document state
+ */
+ const TenantMigrationRecipientDocument getState() const;
+
/**
* To be called on the instance returned by PrimaryOnlyService::getOrCreate(). Returns an
* error if the options this Instance was created with are incompatible with the options