path: root/jstests/replsets/tenant_migration_conflicting_recipient_sync_data_cmds.js

/**
 * Test that tenant migration recipient rejects conflicting recipientSyncData commands.
 *
 * @tags: [
 *   incompatible_with_eft,
 *   incompatible_with_macos,
 *   requires_fcv_52,
 *   incompatible_with_windows_tls,
 *   requires_majority_read_concern,
 *   requires_persistence,
 * ]
 */
(function() {

"use strict";
load("jstests/libs/fail_point_util.js");
load("jstests/libs/parallelTester.js");
load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_util.js");

var rst =
    new ReplSetTest({nodes: 1, nodeOptions: TenantMigrationUtil.makeX509OptionsForTest().donor});
rst.startSet();
rst.initiate();
const primary = rst.getPrimary();
const configDB = primary.getDB("config");
const configRecipientsColl = configDB["tenantMigrationRecipients"];

const kDonorConnectionString0 = "foo/bar:12345";
const kDonorConnectionString1 = "foo/bar:56789";
const kPrimaryReadPreference = {
    mode: "primary"
};
const kSecondaryReadPreference = {
    mode: "secondary"
};
const kRecipientCertificateForDonor =
    TenantMigrationUtil.getCertificateAndPrivateKey("jstests/libs/tenant_migration_recipient.pem");
const kExpiredRecipientCertificateForDonor = TenantMigrationUtil.getCertificateAndPrivateKey(
    "jstests/libs/tenant_migration_recipient_expired.pem");

TestData.stopFailPointErrorCode = 4880402;

/**
 * Runs recipientSyncData on the given host and returns the response.
 */
function runRecipientSyncDataCmd(primaryHost, {
    migrationIdString,
    tenantId,
    donorConnectionString,
    readPreference,
    recipientCertificateForDonor
}) {
    jsTestLog("Starting a recipientSyncDataCmd for migrationId: " + migrationIdString +
              " tenantId: '" + tenantId + "'");
    const primary = new Mongo(primaryHost);
    const res = primary.adminCommand({
        recipientSyncData: 1,
        migrationId: UUID(migrationIdString),
        donorConnectionString: donorConnectionString,
        tenantId: tenantId,
        readPreference: readPreference,
        startMigrationDonorTimestamp: Timestamp(1, 1),
        recipientCertificateForDonor: recipientCertificateForDonor
    });
    return res;
}

/**
 * Returns an array of currentOp entries for the TenantMigrationRecipientService instances that
 * match the given query.
 */
function getTenantMigrationRecipientCurrentOpEntries(recipientPrimary, query) {
    const cmdObj = Object.assign({currentOp: true, desc: "tenant recipient migration"}, query);
    return assert.commandWorked(recipientPrimary.adminCommand(cmdObj)).inprog;
}

/**
 * Asserts that the string does not contain certificate or private pem string.
 */
function assertNoCertificateOrPrivateKey(string) {
    assert(!string.includes("CERTIFICATE"), "found certificate");
    assert(!string.includes("PRIVATE KEY"), "found private key");
}

const kTenantIdPrefix = "testTenantId";
let tenantCounter = 0;

/**
 * Returns a tenantId that will not match any existing prefix.
 */
function generateUniqueTenantId() {
    return kTenantIdPrefix + tenantCounter++;
}

// Enable the failpoint to stop the tenant migration after persisting the state doc.
assert.commandWorked(primary.adminCommand({
    configureFailPoint: "fpAfterPersistingTenantMigrationRecipientInstanceStateDoc",
    mode: "alwaysOn",
    data: {action: "stop", stopErrorCode: NumberInt(TestData.stopFailPointErrorCode)}
}));

// Test migrations with different migrationIds but identical settings.
(() => {
    const tenantId = generateUniqueTenantId() + "DiffMigrationId";
    // Enable failPoint to pause the migration just as it starts.
    const fpPauseBeforeRunTenantMigrationRecipientInstance =
        configureFailPoint(primary, "pauseBeforeRunTenantMigrationRecipientInstance");

    // Start the conflicting recipientSyncData cmds.
    const migrationOpts0 = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId: tenantId,
        donorConnectionString: kDonorConnectionString0,
        readPreference: kPrimaryReadPreference,
        recipientCertificateForDonor: kRecipientCertificateForDonor
    };
    const migrationOpts1 = Object.extend({}, migrationOpts0, true);
    migrationOpts1.migrationIdString = extractUUIDFromObject(UUID());
    const recipientSyncDataThread0 =
        new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts0);
    const recipientSyncDataThread1 =
        new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts1);
    recipientSyncDataThread0.start();
    recipientSyncDataThread1.start();

    jsTestLog("Waiting until one gets started and hits the failPoint.");
    assert.commandWorked(primary.adminCommand({
        waitForFailPoint: "pauseBeforeRunTenantMigrationRecipientInstance",
        timesEntered: fpPauseBeforeRunTenantMigrationRecipientInstance.timesEntered + 1,
        maxTimeMS: kDefaultWaitForFailPointTimeout
    }));

    // One instance is expected as the tenantId conflict is still unresolved.
    jsTestLog("Fetching current operations before conflict is resolved.");
    const currentOpEntriesBeforeInsert = getTenantMigrationRecipientCurrentOpEntries(
        primary, {desc: "tenant recipient migration", tenantId});
    assert.eq(1, currentOpEntriesBeforeInsert.length, tojson(currentOpEntriesBeforeInsert));

    jsTestLog("Unblocking the tenant migration instance from persisting the state doc.");
    fpPauseBeforeRunTenantMigrationRecipientInstance.off();

    // Check responses for both commands. One will return with
    // ErrorCodes.ConflictingOperationInProgress, and the other with a
    // TestData.stopFailPointErrorCode (a failpoint indicating that we have persisted the document).
    const res0 = assert.commandFailed(recipientSyncDataThread0.returnData());
    const res1 = assert.commandFailed(recipientSyncDataThread1.returnData());

    if (res0.code == TestData.stopFailPointErrorCode) {
        assert.commandFailedWithCode(res0, TestData.stopFailPointErrorCode);
        assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress);
        assertNoCertificateOrPrivateKey(res1.errmsg);
    } else {
        assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress);
        assert.commandFailedWithCode(res1, TestData.stopFailPointErrorCode);
        assertNoCertificateOrPrivateKey(res0.errmsg);
    }

    // One of the two instances should have been cleaned up, and therefore only one will remain.
    const currentOpEntriesAfterInsert = getTenantMigrationRecipientCurrentOpEntries(
        primary, {desc: "tenant recipient migration", tenantId});
    assert.eq(1, currentOpEntriesAfterInsert.length, tojson(currentOpEntriesAfterInsert));

    // Only one instance should have succeeded in persisting the state doc, other should have failed
    // with ErrorCodes.ConflictingOperationInProgress.
    assert.eq(1, configRecipientsColl.count({tenantId: tenantId}));

    // Run another recipientSyncData cmd for the tenant. Since the previous migration hasn't been
    // garbage collected, the migration is considered as active. So this command should fail with
    // ErrorCodes.ConflictingOperationInProgress.
    const migrationOpts2 = Object.extend({}, migrationOpts0, true);
    migrationOpts2.migrationIdString = extractUUIDFromObject(UUID());
    const recipientSyncDataCmd2 = new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts2);
    recipientSyncDataCmd2.start();
    const res2 = recipientSyncDataCmd2.returnData();
    assert.commandFailedWithCode(res2, ErrorCodes.ConflictingOperationInProgress);

    // Collection count should remain the same.
    assert.eq(1, configRecipientsColl.count({tenantId: tenantId}));
    fpPauseBeforeRunTenantMigrationRecipientInstance.off();
})();

/**
 * Tests that if the client runs multiple recipientSyncData commands that would start conflicting
 * migrations, only one of the migrations will start and succeed.
 */
function testConcurrentConflictingMigration(migrationOpts0, migrationOpts1) {
    // Start the conflicting recipientSyncData cmds.
    const recipientSyncDataThread0 =
        new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts0);
    const recipientSyncDataThread1 =
        new Thread(runRecipientSyncDataCmd, primary.host, migrationOpts1);
    recipientSyncDataThread0.start();
    recipientSyncDataThread1.start();

    const res0 = assert.commandFailed(recipientSyncDataThread0.returnData());
    const res1 = assert.commandFailed(recipientSyncDataThread1.returnData());

    if (res0.code == TestData.stopFailPointErrorCode) {
        assert.commandFailedWithCode(res0, TestData.stopFailPointErrorCode);
        assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress);
        assertNoCertificateOrPrivateKey(res1.errmsg);
        assert.eq(1, configRecipientsColl.count({_id: UUID(migrationOpts0.migrationIdString)}));
        assert.eq(1, getTenantMigrationRecipientCurrentOpEntries(primary, {
                         "instanceID": UUID(migrationOpts0.migrationIdString)
                     }).length);
        if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) {
            assert.eq(0, configRecipientsColl.count({_id: UUID(migrationOpts1.migrationIdString)}));
            assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, {
                             "instanceID": UUID(migrationOpts1.migrationIdString)
                         }).length);
        } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) {
            assert.eq(0, configRecipientsColl.count({tenantId: migrationOpts1.tenantId}));
            assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, {
                             tenantId: migrationOpts1.tenantId
                         }).length);
        }
    } else {
        assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress);
        assert.commandFailedWithCode(res1, TestData.stopFailPointErrorCode);
        assertNoCertificateOrPrivateKey(res0.errmsg);
        assert.eq(1, configRecipientsColl.count({_id: UUID(migrationOpts1.migrationIdString)}));
        assert.eq(1, getTenantMigrationRecipientCurrentOpEntries(primary, {
                         "instanceID": UUID(migrationOpts1.migrationIdString)
                     }).length);
        if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) {
            assert.eq(0, configRecipientsColl.count({_id: UUID(migrationOpts0.migrationIdString)}));
            assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, {
                             "instanceID": UUID(migrationOpts0.migrationIdString)
                         }).length);
        } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) {
            assert.eq(0, configRecipientsColl.count({tenantId: migrationOpts0.tenantId}));
            assert.eq(0, getTenantMigrationRecipientCurrentOpEntries(primary, {
                             tenantId: migrationOpts0.tenantId
                         }).length);
        }
    }
}

// Test reusing a migrationId with different migration settings.

// Test different tenantIds.
(() => {
    const migrationOpts0 = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId: generateUniqueTenantId() + "DiffTenantId",
        donorConnectionString: kDonorConnectionString0,
        readPreference: kPrimaryReadPreference,
        recipientCertificateForDonor: kRecipientCertificateForDonor
    };
    const migrationOpts1 = Object.extend({}, migrationOpts0, true);
    migrationOpts1.tenantId = generateUniqueTenantId() + "DiffTenantId";
    testConcurrentConflictingMigration(migrationOpts0, migrationOpts1);
})();

// Test different donor connection strings.
(() => {
    const migrationOpts0 = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId: generateUniqueTenantId() + "DiffDonorConnString",
        donorConnectionString: kDonorConnectionString0,
        readPreference: kPrimaryReadPreference,
        recipientCertificateForDonor: kRecipientCertificateForDonor
    };
    const migrationOpts1 = Object.extend({}, migrationOpts0, true);
    migrationOpts1.donorConnectionString = kDonorConnectionString1;
    testConcurrentConflictingMigration(migrationOpts0, migrationOpts1);
})();

// Test different read preference.
(() => {
    const migrationOpts0 = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId: generateUniqueTenantId() + "DiffReadPreference",
        donorConnectionString: kDonorConnectionString0,
        readPreference: kPrimaryReadPreference,
        recipientCertificateForDonor: kRecipientCertificateForDonor
    };
    const migrationOpts1 = Object.extend({}, migrationOpts0, true);
    migrationOpts1.readPreference = kSecondaryReadPreference;
    testConcurrentConflictingMigration(migrationOpts0, migrationOpts1);
})();

// Test different certificates.
(() => {
    const migrationOpts0 = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId: generateUniqueTenantId() + "DiffCertificate",
        donorConnectionString: kDonorConnectionString0,
        readPreference: kPrimaryReadPreference,
        recipientCertificateForDonor: kRecipientCertificateForDonor
    };
    const migrationOpts1 = Object.extend({}, migrationOpts0, true);
    migrationOpts1.recipientCertificateForDonor = kExpiredRecipientCertificateForDonor;
    testConcurrentConflictingMigration(migrationOpts0, migrationOpts1);
})();

rst.stopSet();
})();