diff options
author | Cheahuychou Mao <mao.cheahuychou@gmail.com> | 2021-03-23 19:12:32 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-03-25 18:23:45 +0000 |
commit | 2ff9b318f8ededdea173e7133cd016bd9b9bf48f (patch) | |
tree | 8c10fcc7a9ab9008dd151e1144eeecd4fa60ab35 | |
parent | 520afcb8c5270cb50c93ccfdadb513bf4cd0f7e6 (diff) | |
download | mongo-2ff9b318f8ededdea173e7133cd016bd9b9bf48f.tar.gz |
SERVER-55408 Make TenantMigration{Donor|Recipient}Service's checkIfOptionsConflict check migration certificates
11 files changed, 336 insertions, 109 deletions
diff --git a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js index 7133f6a1435..20ebaca89a9 100644 --- a/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js +++ b/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js @@ -31,15 +31,22 @@ function getTenantMigrationDonorCurrentOpEntries(donorPrimary, query) { return assert.commandWorked(donorPrimary.adminCommand(cmdObj)).inprog; } -const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; -let charIndex = 0; +/** + * Asserts that the string does not contain certificate or private pem string. + */ +function assertNoCertificateOrPrivateKey(string) { + assert(!string.includes("CERTIFICATE"), "found certificate"); + assert(!string.includes("PRIVATE KEY"), "found private key"); +} + +const kTenantIdPrefix = "testTenantId"; +let tenantCounter = 0; /** * Returns a tenantId that will not match any existing prefix. */ function generateUniqueTenantId() { - assert.lt(charIndex, chars.length); - return chars[charIndex++]; + return kTenantIdPrefix + tenantCounter++; } const donorRst = new ReplSetTest( @@ -109,8 +116,9 @@ function testStartingConflictingMigrationAfterInitialMigrationCommitted( tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) { tenantMigrationTest0.runMigration( migrationOpts0, false /* retryOnRetryableErrors */, false /* automaticForgetMigration */); - assert.commandFailedWithCode(tenantMigrationTest1.runMigration(migrationOpts1), - ErrorCodes.ConflictingOperationInProgress); + const res1 = assert.commandFailedWithCode(tenantMigrationTest1.runMigration(migrationOpts1), + ErrorCodes.ConflictingOperationInProgress); + assertNoCertificateOrPrivateKey(res1.errmsg); // If the second donorStartMigration had started a duplicate migration, there would be two donor // state docs and TenantMigrationDonorService instances. @@ -150,6 +158,7 @@ function testConcurrentConflictingMigrations( if (res0.ok) { assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress); + assertNoCertificateOrPrivateKey(res1.errmsg); assert.eq(1, configDonorsColl.count({_id: UUID(migrationOpts0.migrationIdString)})); assert.eq(1, getTenantMigrationDonorCurrentOpEntries(donorPrimary, { "instanceID.uuid": UUID(migrationOpts0.migrationIdString) @@ -170,6 +179,7 @@ function testConcurrentConflictingMigrations( tenantMigrationTest0.forgetMigration(migrationOpts0.migrationIdString)); } else { assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress); + assertNoCertificateOrPrivateKey(res0.errmsg); assert.eq(1, configDonorsColl.count({_id: UUID(migrationOpts1.migrationIdString)})); assert.eq(1, getTenantMigrationDonorCurrentOpEntries(donorPrimary, { "instanceID.uuid": UUID(migrationOpts1.migrationIdString) @@ -262,6 +272,51 @@ function testConcurrentConflictingMigrations( testConcurrentConflictingMigrations(...makeTestParams()); })(); +const kDonorCertificateAndPrivateKey = + TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_donor.pem"); +const kExpiredDonorCertificateAndPrivateKey = TenantMigrationUtil.getCertificateAndPrivateKey( + "jstests/libs/tenant_migration_donor_expired.pem"); +const kRecipientCertificateAndPrivateKey = + TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_recipient.pem"); +const kExpiredRecipientCertificateAndPrivateKey = TenantMigrationUtil.getCertificateAndPrivateKey( + "jstests/libs/tenant_migration_recipient_expired.pem"); + +// Test different donor certificates. +(() => { + let makeTestParams = () => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffDonorCertificate", + donorCertificateForRecipient: kDonorCertificateAndPrivateKey, + recipientCertificateForDonor: kRecipientCertificateAndPrivateKey + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.donorCertificateForRecipient = kExpiredDonorCertificateAndPrivateKey; + return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + }; + + testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); + testConcurrentConflictingMigrations(...makeTestParams()); +})(); + +// Test different recipient certificates. +(() => { + let makeTestParams = () => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffRecipientCertificate", + donorCertificateForRecipient: kDonorCertificateAndPrivateKey, + recipientCertificateForDonor: kRecipientCertificateAndPrivateKey + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.recipientCertificateForDonor = kExpiredRecipientCertificateAndPrivateKey; + return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1]; + }; + + testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams()); + testConcurrentConflictingMigrations(...makeTestParams()); +})(); + tenantMigrationTest0.stop(); donorRst.stopSet(); })(); diff --git a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js index d22ce1bf551..72d45983f6d 100644 --- a/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js +++ b/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js @@ -8,7 +8,8 @@ "use strict"; load("jstests/libs/fail_point_util.js"); -load("jstests/libs/parallel_shell_helpers.js"); +load("jstests/libs/parallelTester.js"); +load("jstests/libs/uuid_util.js"); load("jstests/replsets/libs/tenant_migration_util.js"); var rst = @@ -23,20 +24,46 @@ if (!TenantMigrationUtil.isFeatureFlagEnabled(rst.getPrimary())) { const primary = rst.getPrimary(); const configDB = primary.getDB("config"); -const tenantMigrationRecipientStateColl = configDB["tenantMigrationRecipients"]; +const configRecipientsColl = configDB["tenantMigrationRecipients"]; -const tenantId = "test"; -const connectionString = "foo/bar:12345"; -const readPreference = { - mode: 'primary' +const kDonorConnectionString0 = "foo/bar:12345"; +const kDonorConnectionString1 = "foo/bar:56789"; +const kPrimaryReadPreference = { + mode: "primary" }; +const kSecondaryReadPreference = { + mode: "secondary" +}; +const kRecipientCertificateForDonor = + TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_recipient.pem"); +const kExpiredRecipientCertificateForDonor = TenantMigrationUtil.getCertificateAndPrivateKey( + "jstests/libs/tenant_migration_recipient_expired.pem"); TestData.stopFailPointErrorCode = 4880402; -function checkTenantMigrationRecipientStateCollCount(expectedCount) { - let res = tenantMigrationRecipientStateColl.find().toArray(); - assert.eq(expectedCount, - res.length, - "'config.tenantMigrationRecipients' collection count mismatch: " + tojson(res)); + +/** + * Runs recipientSyncData on the given host and returns the response. + */ +function runRecipientSyncDataCmd(primaryHost, { + migrationIdString, + tenantId, + donorConnectionString, + readPreference, + recipientCertificateForDonor +}) { + jsTestLog("Starting a recipientSyncDataCmd for migrationId: " + migrationIdString + + " tenantId: '" + tenantId + "'"); + const primary = new Mongo(primaryHost); + const res = primary.adminCommand({ + recipientSyncData: 1, + migrationId: UUID(migrationIdString), + donorConnectionString: donorConnectionString, + tenantId: tenantId, + readPreference: readPreference, + startMigrationDonorTimestamp: Timestamp(1, 1), + recipientCertificateForDonor: recipientCertificateForDonor + }); + return res; } /** @@ -48,23 +75,22 @@ function getTenantMigrationRecipientCurrentOpEntries(recipientPrimary, query) { return assert.commandWorked(recipientPrimary.adminCommand(cmdObj)).inprog; } -function startRecipientSyncDataCmd(migrationUuid, tenantId, connectionString, readPreference) { - load("jstests/replsets/libs/tenant_migration_util.js"); +/** + * Asserts that the string does not contain certificate or private pem string. + */ +function assertNoCertificateOrPrivateKey(string) { + assert(!string.includes("CERTIFICATE"), "found certificate"); + assert(!string.includes("PRIVATE KEY"), "found private key"); +} - jsTestLog("Starting a recipientSyncDataCmd for migrationUuid: " + migrationUuid + - " tenantId: '" + tenantId + "'"); - assert.commandFailedWithCode( - db.adminCommand({ - recipientSyncData: 1, - migrationId: migrationUuid, - donorConnectionString: connectionString, - tenantId: tenantId, - readPreference: readPreference, - startMigrationDonorTimestamp: Timestamp(1, 1), - recipientCertificateForDonor: - TenantMigrationUtil.makeMigrationCertificatesForTest().recipientCertificateForDonor - }), - [TestData.stopFailPointErrorCode, ErrorCodes.ConflictingOperationInProgress]); +const kTenantIdPrefix = "testTenantId"; +let tenantCounter = 0; + +/** + * Returns a tenantId that will not match any existing prefix. + */ +function generateUniqueTenantId() { + return kTenantIdPrefix + tenantCounter++; } // Enable the failpoint to stop the tenant migration after persisting the state doc. @@ -74,20 +100,29 @@ assert.commandWorked(primary.adminCommand({ data: {action: "stop", stopErrorCode: NumberInt(TestData.stopFailPointErrorCode)} })); -{ +// Test migrations with different migrationIds but identical settings. +(() => { + const tenantId = generateUniqueTenantId() + "DiffMigrationId"; // Enable failPoint to pause the migration just as it starts. const fpPauseBeforeRunTenantMigrationRecipientInstance = configureFailPoint(primary, "pauseBeforeRunTenantMigrationRecipientInstance"); - // Sanity check : 'config.tenantMigrationRecipients' collection count should be empty. - checkTenantMigrationRecipientStateCollCount(0); - // Start the conflicting recipientSyncData cmds. - const recipientSyncDataCmd1 = startParallelShell( - funWithArgs(startRecipientSyncDataCmd, UUID(), tenantId, connectionString, readPreference), - primary.port); - const recipientSyncDataCmd2 = startParallelShell( - funWithArgs(startRecipientSyncDataCmd, UUID(), tenantId, connectionString, readPreference), - primary.port); + // Start the conflicting recipientSyncData cmds. + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: tenantId, + donorConnectionString: kDonorConnectionString0, + readPreference: kPrimaryReadPreference, + recipientCertificateForDonor: kRecipientCertificateForDonor + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.migrationIdString = extractUUIDFromObject(UUID()); + const recipientSyncDataThread0 = + new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts0); + const recipientSyncDataThread1 = + new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts1); + recipientSyncDataThread0.start(); + recipientSyncDataThread1.start(); jsTestLog("Waiting until both conflicting instances get started and hit the failPoint."); assert.commandWorked(primary.adminCommand({ @@ -108,8 +143,18 @@ assert.commandWorked(primary.adminCommand({ // Wait for both the conflicting instances to complete. Although both will "complete", one will // return with ErrorCodes.ConflictingOperationInProgress, and the other with a // TestData.stopFailPointErrorCode (a failpoint indicating that we have persisted the document). - recipientSyncDataCmd1(); - recipientSyncDataCmd2(); + const res0 = assert.commandFailed(recipientSyncDataThread0.returnData()); + const res1 = assert.commandFailed(recipientSyncDataThread1.returnData()); + + if (res0.code == TestData.stopFailPointErrorCode) { + assert.commandFailedWithCode(res0, TestData.stopFailPointErrorCode); + assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress); + assertNoCertificateOrPrivateKey(res1.errmsg); + } else { + assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress); + assert.commandFailedWithCode(res1, TestData.stopFailPointErrorCode); + assertNoCertificateOrPrivateKey(res0.errmsg); + } // One of the two instances should have been cleaned up, and therefore only one will remain. const currentOpEntriesAfterInsert = getTenantMigrationRecipientCurrentOpEntries( @@ -118,22 +163,137 @@ assert.commandWorked(primary.adminCommand({ // Only one instance should have succeeded in persisting the state doc, other should have failed // with ErrorCodes.ConflictingOperationInProgress. - checkTenantMigrationRecipientStateCollCount(1); -} + assert.eq(1, configRecipientsColl.count({tenantId: tenantId})); -{ - // Now, again call recipientSyncData cmd to run on the same tenant "test'. Since, our previous - // instance for tenant "test' wasn't garbage collected, the migration status for that tenant is - // considered as active. So, this command should fail with + // Run another recipientSyncData cmd for the tenant. Since the previous migration hasn't been + // garbage collected, the migration is considered as active. So this command should fail with // ErrorCodes.ConflictingOperationInProgress. - const recipientSyncDataCmd3 = startParallelShell( - funWithArgs(startRecipientSyncDataCmd, UUID(), tenantId, connectionString, readPreference), - primary.port); - recipientSyncDataCmd3(); + const migrationOpts2 = Object.extend({}, migrationOpts0, true); + migrationOpts2.migrationIdString = extractUUIDFromObject(UUID()); + const recipientSyncDataCmd2 = new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts2); + recipientSyncDataCmd2.start(); + const res2 = recipientSyncDataCmd2.returnData(); + assert.commandFailedWithCode(res2, ErrorCodes.ConflictingOperationInProgress); // Collection count should remain the same. - checkTenantMigrationRecipientStateCollCount(1); + assert.eq(1, configRecipientsColl.count({tenantId: tenantId})); + fpPauseBeforeRunTenantMigrationRecipientInstance.off(); +})(); + +/** + * Tests that if the client runs multiple recipientSyncData commands that would start conflicting + * migrations, only one of the migrations will start and succeed. + */ +function testConcurrentConflictingMigration(migrationOpts0, migrationOpts1) { + // Start the conflicting recipientSyncData cmds. + const recipientSyncDataThread0 = + new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts0); + const recipientSyncDataThread1 = + new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts1); + recipientSyncDataThread0.start(); + recipientSyncDataThread1.start(); + + const res0 = assert.commandFailed(recipientSyncDataThread0.returnData()); + const res1 = assert.commandFailed(recipientSyncDataThread1.returnData()); + + if (res0.code == TestData.stopFailPointErrorCode) { + assert.commandFailedWithCode(res0, TestData.stopFailPointErrorCode); + assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress); + assertNoCertificateOrPrivateKey(res1.errmsg); + assert.eq(1, configRecipientsColl.count({_id: UUID(migrationOpts0.migrationIdString)})); + assert.eq(1, getTenantMigrationRecipientCurrentOpEntries(primary, { + "instanceID.uuid": UUID(migrationOpts0.migrationIdString) + }).length); + if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) { + assert.eq(0, configRecipientsColl.count({_id: UUID(migrationOpts1.migrationIdString)})); + assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, { + "instanceID.uuid": UUID(migrationOpts1.migrationIdString) + }).length); + } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) { + assert.eq(0, configRecipientsColl.count({tenantId: migrationOpts1.tenantId})); + assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, { + tenantId: migrationOpts1.tenantId + }).length); + } + } else { + assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress); + assert.commandFailedWithCode(res1, TestData.stopFailPointErrorCode); + assertNoCertificateOrPrivateKey(res0.errmsg); + assert.eq(1, configRecipientsColl.count({_id: UUID(migrationOpts1.migrationIdString)})); + assert.eq(1, getTenantMigrationRecipientCurrentOpEntries(primary, { + "instanceID.uuid": UUID(migrationOpts1.migrationIdString) + }).length); + if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) { + assert.eq(0, configRecipientsColl.count({_id: UUID(migrationOpts0.migrationIdString)})); + assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, { + "instanceID.uuid": UUID(migrationOpts0.migrationIdString) + }).length); + } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) { + assert.eq(0, configRecipientsColl.count({tenantId: migrationOpts0.tenantId})); + assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, { + tenantId: migrationOpts0.tenantId + }).length); + } + } } +// Test reusing a migrationId with different migration settings. + +// Test different tenantIds. +(() => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffTenantId", + donorConnectionString: kDonorConnectionString0, + readPreference: kPrimaryReadPreference, + recipientCertificateForDonor: kRecipientCertificateForDonor + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.tenantId = generateUniqueTenantId() + "DiffTenantId"; + testConcurrentConflictingMigration(migrationOpts0, migrationOpts1); +})(); + +// Test different donor connection strings. +(() => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffDonorConnString", + donorConnectionString: kDonorConnectionString0, + readPreference: kPrimaryReadPreference, + recipientCertificateForDonor: kRecipientCertificateForDonor + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.donorConnectionString = kDonorConnectionString1; + testConcurrentConflictingMigration(migrationOpts0, migrationOpts1); +})(); + +// Test different read preference. +(() => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffReadPreference", + donorConnectionString: kDonorConnectionString0, + readPreference: kPrimaryReadPreference, + recipientCertificateForDonor: kRecipientCertificateForDonor + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.readPreference = kSecondaryReadPreference; + testConcurrentConflictingMigration(migrationOpts0, migrationOpts1); +})(); + +// Test different certificates. +(() => { + const migrationOpts0 = { + migrationIdString: extractUUIDFromObject(UUID()), + tenantId: generateUniqueTenantId() + "DiffCertificate", + donorConnectionString: kDonorConnectionString0, + readPreference: kPrimaryReadPreference, + recipientCertificateForDonor: kRecipientCertificateForDonor + }; + const migrationOpts1 = Object.extend({}, migrationOpts0, true); + migrationOpts1.recipientCertificateForDonor = kExpiredRecipientCertificateForDonor; + testConcurrentConflictingMigration(migrationOpts0, migrationOpts1); +})(); + rst.stopSet(); })(); diff --git a/jstests/replsets/tenant_migration_logs.js b/jstests/replsets/tenant_migration_logs.js index 6dfb68fdf3f..826229925fb 100644 --- a/jstests/replsets/tenant_migration_logs.js +++ b/jstests/replsets/tenant_migration_logs.js @@ -20,26 +20,6 @@ function assertNoCertificateOrPrivateKeyLogsForCmd(conn, cmdName) { "found private key in the logs"); } -function assertNoCertificateOrPrivateKeyFields(doc) { - for (let k of Object.keys(doc)) { - let v = doc[k]; - if (typeof v === "string") { - assert(!v.match(/BEGIN CERTIFICATE(.*\n.*)*END CERTIFICATE/m), - `found certificate field`); - assert(!v.match(/BEGIN PRIVATE KEY(.*\n.*)*END PRIVATE KEY/m), - `found private key field`); - } else if (Array.isArray(v)) { - v.forEach((item) => { - if (typeof item === "object" && item !== null) { - assertNoCertificateOrPrivateKeyFields(v); - } - }); - } else if (typeof v === "object" && v !== null) { - assertNoCertificateOrPrivateKeyFields(v); - } - } -} - const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); if (!tenantMigrationTest.isFeatureFlagEnabled()) { jsTestLog("Skipping test because the tenant migrations feature flag is disabled"); diff --git a/src/mongo/db/repl/tenant_migration_donor_service.cpp b/src/mongo/db/repl/tenant_migration_donor_service.cpp index 53472fdd68e..32859ce826e 100644 --- a/src/mongo/db/repl/tenant_migration_donor_service.cpp +++ b/src/mongo/db/repl/tenant_migration_donor_service.cpp @@ -362,18 +362,20 @@ boost::optional<BSONObj> TenantMigrationDonorService::Instance::reportForCurrent Status TenantMigrationDonorService::Instance::checkIfOptionsConflict( const TenantMigrationDonorDocument& stateDoc) { stdx::lock_guard<Latch> lg(_mutex); - - if (stateDoc.getId() != _migrationUuid || stateDoc.getTenantId() != _tenantId || - stateDoc.getRecipientConnectionString() != _recipientConnectionString || - SimpleBSONObjComparator::kInstance.compare(stateDoc.getReadPreference().toInnerBSON(), - _readPreference.toInnerBSON()) != 0) { - return Status(ErrorCodes::ConflictingOperationInProgress, - str::stream() - << "Found active migration for tenantId \"" << stateDoc.getTenantId() - << "\" with different options " << _stateDoc.toBSON()); + invariant(stateDoc.getId() == _migrationUuid); + + if (stateDoc.getTenantId() == _tenantId && + stateDoc.getRecipientConnectionString() == _recipientConnectionString && + stateDoc.getReadPreference().equals(_readPreference) && + stateDoc.getDonorCertificateForRecipient() == _donorCertificateForRecipient && + stateDoc.getRecipientCertificateForDonor() == _recipientCertificateForDonor) { + return Status::OK(); } - return Status::OK(); + return Status(ErrorCodes::ConflictingOperationInProgress, + str::stream() << "Found active migration for migrationId \"" + << _migrationUuid.toBSON() << "\" with different options " + << tenant_migration_util::redactStateDoc(_stateDoc.toBSON())); } TenantMigrationDonorService::Instance::DurableState diff --git a/src/mongo/db/repl/tenant_migration_donor_service.h b/src/mongo/db/repl/tenant_migration_donor_service.h index b7bbd9dbd82..2fd903f7fdf 100644 --- a/src/mongo/db/repl/tenant_migration_donor_service.h +++ b/src/mongo/db/repl/tenant_migration_donor_service.h @@ -96,8 +96,8 @@ public: /** * To be called on the instance returned by PrimaryOnlyService::getOrCreate. Returns an - * error if the options this Instance was created with are incompatible with a request for - * an instance with the options given in 'stateDoc'. + * error if the options this Instance was created with are incompatible with the options + * given in 'stateDoc'. */ Status checkIfOptionsConflict(const TenantMigrationDonorDocument& stateDoc); diff --git a/src/mongo/db/repl/tenant_migration_pem_payload.idl b/src/mongo/db/repl/tenant_migration_pem_payload.idl index 6886c4be34a..13ec011929d 100644 --- a/src/mongo/db/repl/tenant_migration_pem_payload.idl +++ b/src/mongo/db/repl/tenant_migration_pem_payload.idl @@ -37,6 +37,7 @@ structs: TenantMigrationPEMPayload: description: "Contains SSL certificate and private key PEM blobs for a replica set." strict: true + generate_comparison_operators: true fields: certificate: type: string diff --git a/src/mongo/db/repl/tenant_migration_recipient_entry_helpers.cpp b/src/mongo/db/repl/tenant_migration_recipient_entry_helpers.cpp index b7d7628444a..c3e17f2e27f 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_entry_helpers.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_entry_helpers.cpp @@ -80,8 +80,9 @@ Status insertStateDoc(OperationContext* opCtx, const TenantMigrationRecipientDoc invariant(!updateResult.numDocsModified); if (updateResult.upsertedId.isEmpty()) { return {ErrorCodes::ConflictingOperationInProgress, - str::stream() << "Failed to insert the state doc: " << stateDoc.toBSON() - << "; Active tenant migration found for tenantId: " + str::stream() << "Failed to insert the state doc: " + << tenant_migration_util::redactStateDoc(stateDoc.toBSON()) + << "; Found active tenant migration for tenantId: " << stateDoc.getTenantId()}; } return Status::OK(); diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp index d9b06537fc7..09768066123 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp @@ -284,6 +284,7 @@ TenantMigrationRecipientService::Instance::Instance( _donorConnectionString(_stateDoc.getDonorConnectionString().toString()), _donorUri(uassertStatusOK(MongoURI::parse(_stateDoc.getDonorConnectionString().toString()))), _readPreference(_stateDoc.getReadPreference()), + _recipientCertificateForDonor(_stateDoc.getRecipientCertificateForDonor()), _transientSSLParams([&]() -> boost::optional<TransientSSLParams> { if (auto recipientCertificate = _stateDoc.getRecipientCertificateForDonor()) { invariant(!repl::tenantMigrationDisableX509Auth); @@ -381,22 +382,21 @@ boost::optional<BSONObj> TenantMigrationRecipientService::Instance::reportForCur } Status TenantMigrationRecipientService::Instance::checkIfOptionsConflict( - const TenantMigrationRecipientDocument& requestedStateDoc) const { - invariant(requestedStateDoc.getId() == _migrationUuid); - - if (requestedStateDoc.getTenantId() == _tenantId && - requestedStateDoc.getDonorConnectionString() == _donorConnectionString && - requestedStateDoc.getReadPreference().equals(_readPreference)) { + const TenantMigrationRecipientDocument& stateDoc) const { + stdx::lock_guard<Latch> lg(_mutex); + invariant(stateDoc.getId() == _migrationUuid); + + if (stateDoc.getTenantId() == _tenantId && + stateDoc.getDonorConnectionString() == _donorConnectionString && + stateDoc.getReadPreference().equals(_readPreference) && + stateDoc.getRecipientCertificateForDonor() == _recipientCertificateForDonor) { return Status::OK(); } return Status(ErrorCodes::ConflictingOperationInProgress, - str::stream() << "Requested options for tenant migration doesn't match" - << " the active migration options, migrationId: " << _migrationUuid - << ", tenantId: " << _tenantId - << ", connectionString: " << _donorConnectionString - << ", readPreference: " << _readPreference.toString() - << ", requested options:" << requestedStateDoc.toBSON()); + str::stream() << "Found active migration for migrationId \"" + << _migrationUuid.toBSON() << "\" with different options " + << tenant_migration_util::redactStateDoc(_stateDoc.toBSON())); } OpTime TenantMigrationRecipientService::Instance::waitUntilMigrationReachesConsistentState( diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h index e68d0844f55..58e65811412 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.h +++ b/src/mongo/db/repl/tenant_migration_recipient_service.h @@ -131,8 +131,8 @@ public: /** * To be called on the instance returned by PrimaryOnlyService::getOrCreate(). Returns an - * error if the options this Instance was created with are incompatible with a request for - * an instance with the options given in 'stateDoc'. + * error if the options this Instance was created with are incompatible with the options + * given in 'stateDoc'. */ Status checkIfOptionsConflict(const TenantMigrationRecipientDocument& stateDoc) const; @@ -501,11 +501,12 @@ public: // This data is provided in the initial state doc and never changes. We keep copies to // avoid having to obtain the mutex to access them. - const std::string _tenantId; // (R) - const UUID _migrationUuid; // (R) - const std::string _donorConnectionString; // (R) - const MongoURI _donorUri; // (R) - const ReadPreferenceSetting _readPreference; // (R) + const std::string _tenantId; // (R) + const UUID _migrationUuid; // (R) + const std::string _donorConnectionString; // (R) + const MongoURI _donorUri; // (R) + const ReadPreferenceSetting _readPreference; // (R) + const boost::optional<TenantMigrationPEMPayload> _recipientCertificateForDonor; // (R) // TODO (SERVER-54085): Remove server parameter tenantMigrationDisableX509Auth. // Transient SSL params created based on the state doc if the server parameter // 'tenantMigrationDisableX509Auth' is false. diff --git a/src/mongo/db/repl/tenant_migration_util.cpp b/src/mongo/db/repl/tenant_migration_util.cpp index 7be5314a922..36bba920ad1 100644 --- a/src/mongo/db/repl/tenant_migration_util.cpp +++ b/src/mongo/db/repl/tenant_migration_util.cpp @@ -30,6 +30,8 @@ #include "mongo/db/repl/tenant_migration_util.h" #include "mongo/bson/json.h" +#include "mongo/bson/mutable/algorithm.h" +#include "mongo/bson/mutable/document.h" #include "mongo/db/concurrency/write_conflict_exception.h" #include "mongo/db/db_raii.h" #include "mongo/db/dbdirectclient.h" @@ -53,8 +55,15 @@ namespace mongo { namespace tenant_migration_util { +namespace { + +const std::set<std::string> kSensitiveFieldNames{"donorCertificateForRecipient", + "recipientCertificateForDonor"}; + MONGO_FAIL_POINT_DEFINE(pauseTenantMigrationBeforeMarkingExternalKeysGarbageCollectable); +} // namespace + const Backoff kExponentialBackoff(Seconds(1), Milliseconds::max()); ExternalKeysCollectionDocument makeExternalClusterTimeKeyDoc(UUID migrationId, BSONObj keyDoc) { @@ -391,6 +400,19 @@ ExecutorFuture<void> markExternalKeysAsGarbageCollectable( .on(parentExecutor, CancellationToken::uncancelable()); } +BSONObj redactStateDoc(BSONObj stateDoc) { + mutablebson::Document stateDocToLog(stateDoc, mutablebson::Document::kInPlaceDisabled); + for (auto& sensitiveField : kSensitiveFieldNames) { + for (mutablebson::Element element = + mutablebson::findFirstChildNamed(stateDocToLog.root(), sensitiveField); + element.ok(); + element = mutablebson::findElementNamed(element.rightSibling(), sensitiveField)) { + uassertStatusOK(element.setValueString("xxx")); + } + } + return stateDocToLog.getObject(); +} + } // namespace tenant_migration_util } // namespace mongo diff --git a/src/mongo/db/repl/tenant_migration_util.h b/src/mongo/db/repl/tenant_migration_util.h index 04cf2245048..09952929ed5 100644 --- a/src/mongo/db/repl/tenant_migration_util.h +++ b/src/mongo/db/repl/tenant_migration_util.h @@ -181,6 +181,11 @@ createRetryableWritesOplogFetchingPipelineForTenantMigrations( const Timestamp& startFetchingTimestamp, const std::string& tenantId); +/** + * Returns a new BSONObj created from 'stateDoc' with sensitive fields redacted. + */ +BSONObj redactStateDoc(BSONObj stateDoc); + } // namespace tenant_migration_util } // namespace mongo |