path: root/jstests/replsets/tenant_migration_conflicting_donor_start_migration_cmds.js

/**
 * Test that tenant migration donors correctly join retried donorStartMigration commands and reject
 * conflicting donorStartMigration commands.
 *
 * @tags: [requires_fcv_47, requires_majority_read_concern, incompatible_with_eft]
 */
(function() {
'use strict';

load("jstests/libs/uuid_util.js");
load("jstests/replsets/libs/tenant_migration_test.js");

/**
 * Asserts that the number of recipientDataSync commands executed on the given recipient primary is
 * equal to the given number.
 */
function checkNumRecipientSyncDataCmdExecuted(recipientPrimary, expectedNumExecuted) {
    const recipientSyncDataMetrics =
        recipientPrimary.adminCommand({serverStatus: 1}).metrics.commands.recipientSyncData;
    assert.eq(0, recipientSyncDataMetrics.failed);
    assert.eq(expectedNumExecuted, recipientSyncDataMetrics.total);
}

/**
 * Returns an array of currentOp entries for the TenantMigrationDonorService instances that match
 * the given query.
 */
function getTenantMigrationDonorCurrentOpEntries(donorPrimary, query) {
    const cmdObj = Object.assign({currentOp: true, desc: "tenant donor migration"}, query);
    return assert.commandWorked(donorPrimary.adminCommand(cmdObj)).inprog;
}

const chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let charIndex = 0;

/**
 * Returns a tenantId that will not match any existing prefix.
 */
function generateUniqueTenantId() {
    assert.lt(charIndex, chars.length);
    return chars[charIndex++];
}

const donorRst = new ReplSetTest(
    {nodes: 1, name: 'donorRst', nodeOptions: TenantMigrationUtil.makeX509OptionsForTest().donor});

donorRst.startSet();
donorRst.initiate();

const tenantMigrationTest0 = new TenantMigrationTest({name: jsTestName(), donorRst});
if (!tenantMigrationTest0.isFeatureFlagEnabled()) {
    jsTestLog("Skipping test because the tenant migrations feature flag is disabled");
    donorRst.stopSet();
    return;
}

const donorPrimary = donorRst.getPrimary();
const recipientPrimary = tenantMigrationTest0.getRecipientPrimary();

let numRecipientSyncDataCmdSent = 0;

// Test that a retry of a donorStartMigration command joins the existing migration that has
// completed but has not been garbage-collected.
(() => {
    const tenantId = `${generateUniqueTenantId()}RetryAfterMigrationCompletes`;
    const migrationOpts = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId,
    };

    assert.commandWorked(tenantMigrationTest0.runMigration(migrationOpts));
    assert.commandWorked(tenantMigrationTest0.runMigration(migrationOpts));

    // If the second donorStartMigration had started a duplicate migration, the recipient would have
    // received four recipientSyncData commands instead of two.
    numRecipientSyncDataCmdSent += 2;
    checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent);
})();

// Test that a retry of a donorStartMigration command joins the ongoing migration.
(() => {
    const tenantId = `${generateUniqueTenantId()}RetryBeforeMigrationCompletes`;
    const migrationOpts = {
        migrationIdString: extractUUIDFromObject(UUID()),
        tenantId,
    };

    assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts));
    assert.commandWorked(tenantMigrationTest0.startMigration(migrationOpts));

    assert.commandWorked(tenantMigrationTest0.waitForMigrationToComplete(migrationOpts));
    assert.commandWorked(tenantMigrationTest0.waitForMigrationToComplete(migrationOpts));

    // If the second donorStartMigration had started a duplicate migration, the recipient would have
    // received four recipientSyncData commands instead of two.
    numRecipientSyncDataCmdSent += 2;
    checkNumRecipientSyncDataCmdExecuted(recipientPrimary, numRecipientSyncDataCmdSent);

    assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts.migrationIdString));
})();

/**
 * Tests that the donor throws a ConflictingOperationInProgress error if the client runs a
 * donorStartMigration command to start a migration that conflicts with an existing migration that
 * has committed but not garbage-collected (i.e. the donor has not received donorForgetMigration).
 */
function testStartingConflictingMigrationAfterInitialMigrationCommitted(
    tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) {
    tenantMigrationTest0.runMigration(
        migrationOpts0, false /* retryOnRetryableErrors */, false /* automaticForgetMigration */);
    assert.commandFailedWithCode(tenantMigrationTest1.runMigration(migrationOpts1),
                                 ErrorCodes.ConflictingOperationInProgress);

    // If the second donorStartMigration had started a duplicate migration, there would be two donor
    // state docs and TenantMigrationDonorService instances.
    let configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);
    assert.eq(1, configDonorsColl.count({_id: UUID(migrationOpts0.migrationIdString)}));
    assert.eq(1, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                     "instanceID.uuid": UUID(migrationOpts0.migrationIdString)
                 }).length);
    if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) {
        assert.eq(0, configDonorsColl.count({_id: UUID(migrationOpts1.migrationIdString)}));
        assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                         "instanceID.uuid": UUID(migrationOpts1.migrationIdString)
                     }).length);
    } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) {
        assert.eq(0, configDonorsColl.count({tenantId: migrationOpts1.tenantId}));
        assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                         tenantId: migrationOpts1.tenantId
                     }).length);
    }
    assert.commandWorked(tenantMigrationTest0.forgetMigration(migrationOpts0.migrationIdString));
}

/**
 * Tests that if the client runs multiple donorStartMigration commands that would start conflicting
 * migrations, only one of the migrations will start and succeed.
 */
function testConcurrentConflictingMigrations(
    tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1) {
    const res0 = tenantMigrationTest0.startMigration(migrationOpts0);
    const res1 = tenantMigrationTest1.startMigration(migrationOpts1);

    let configDonorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS);

    // Verify that exactly one migration succeeded.
    assert(res0.ok || res1.ok);
    assert(!res0.ok || !res1.ok);

    if (res0.ok) {
        assert.commandFailedWithCode(res1, ErrorCodes.ConflictingOperationInProgress);
        assert.eq(1, configDonorsColl.count({_id: UUID(migrationOpts0.migrationIdString)}));
        assert.eq(1, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                         "instanceID.uuid": UUID(migrationOpts0.migrationIdString)
                     }).length);
        if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) {
            assert.eq(0, configDonorsColl.count({_id: UUID(migrationOpts1.migrationIdString)}));
            assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                             "instanceID.uuid": UUID(migrationOpts1.migrationIdString)
                         }).length);
        } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) {
            assert.eq(0, configDonorsColl.count({tenantId: migrationOpts1.tenantId}));
            assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                             tenantId: migrationOpts1.tenantId
                         }).length);
        }
        assert.commandWorked(tenantMigrationTest0.waitForMigrationToComplete(migrationOpts0));
        assert.commandWorked(
            tenantMigrationTest0.forgetMigration(migrationOpts0.migrationIdString));
    } else {
        assert.commandFailedWithCode(res0, ErrorCodes.ConflictingOperationInProgress);
        assert.eq(1, configDonorsColl.count({_id: UUID(migrationOpts1.migrationIdString)}));
        assert.eq(1, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                         "instanceID.uuid": UUID(migrationOpts1.migrationIdString)
                     }).length);
        if (migrationOpts0.migrationIdString != migrationOpts1.migrationIdString) {
            assert.eq(0, configDonorsColl.count({_id: UUID(migrationOpts0.migrationIdString)}));
            assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                             "instanceID.uuid": UUID(migrationOpts0.migrationIdString)
                         }).length);
        } else if (migrationOpts0.tenantId != migrationOpts1.tenantId) {
            assert.eq(0, configDonorsColl.count({tenantId: migrationOpts0.tenantId}));
            assert.eq(0, getTenantMigrationDonorCurrentOpEntries(donorPrimary, {
                             tenantId: migrationOpts0.tenantId
                         }).length);
        }
        assert.commandWorked(tenantMigrationTest1.waitForMigrationToComplete(migrationOpts1));
        assert.commandWorked(
            tenantMigrationTest1.forgetMigration(migrationOpts1.migrationIdString));
    }
}

// Test migrations with different migrationIds but identical settings.
(() => {
    let makeTestParams = () => {
        const migrationOpts0 = {
            migrationIdString: extractUUIDFromObject(UUID()),
            tenantId: generateUniqueTenantId() + "DiffMigrationId",
        };
        const migrationOpts1 = Object.extend({}, migrationOpts0, true);
        migrationOpts1.migrationIdString = extractUUIDFromObject(UUID());
        return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
    };

    testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
    testConcurrentConflictingMigrations(...makeTestParams());
})();

// Test reusing a migrationId for different migration settings.

// Test different tenantIds.
(() => {
    let makeTestParams = () => {
        const migrationOpts0 = {
            migrationIdString: extractUUIDFromObject(UUID()),
            tenantId: generateUniqueTenantId() + "DiffTenantId",
        };
        const migrationOpts1 = Object.extend({}, migrationOpts0, true);
        migrationOpts1.tenantId = generateUniqueTenantId() + "DiffTenantId";
        return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
    };

    testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
    testConcurrentConflictingMigrations(...makeTestParams());
})();

// Test different recipient connection strings.
(() => {
    const tenantMigrationTest1 = new TenantMigrationTest({name: `${jsTestName()}1`, donorRst});

    let makeTestParams = () => {
        const migrationOpts0 = {
            migrationIdString: extractUUIDFromObject(UUID()),
            tenantId: generateUniqueTenantId() + "DiffRecipientConnString",
        };
        // The recipient connection string will be populated by the TenantMigrationTest fixture, so
        // no need to set it here.
        const migrationOpts1 = Object.extend({}, migrationOpts0, true);
        return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest1, migrationOpts1];
    };

    testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
    testConcurrentConflictingMigrations(...makeTestParams());

    tenantMigrationTest1.stop();
})();

// Test different cloning read preference.
(() => {
    let makeTestParams = () => {
        const migrationOpts0 = {
            migrationIdString: extractUUIDFromObject(UUID()),
            tenantId: generateUniqueTenantId() + "DiffReadPref",
        };
        const migrationOpts1 = Object.extend({}, migrationOpts0, true);
        migrationOpts1.readPreference = {mode: "secondary"};
        return [tenantMigrationTest0, migrationOpts0, tenantMigrationTest0, migrationOpts1];
    };

    testStartingConflictingMigrationAfterInitialMigrationCommitted(...makeTestParams());
    testConcurrentConflictingMigrations(...makeTestParams());
})();

tenantMigrationTest0.stop();
donorRst.stopSet();
})();