From ddf7d937d4228b16d03935296480c345e4bdd171 Mon Sep 17 00:00:00 2001 From: Christopher Caplinger Date: Wed, 6 Apr 2022 15:39:11 +0000 Subject: SERVER-63122: Remove logical cloning procedure for shard merge protocol --- etc/backports_required_for_multiversion_tests.yml | 20 +++ jstests/replsets/libs/tenant_migration_test.js | 6 +- .../tenant_migration_buildindex_shard_merge.js | 9 +- ...ncy_commit_optime_before_last_cloning_optime.js | 5 + jstests/replsets/tenant_migration_cloner_stats.js | 4 + ...migration_cloning_uses_read_concern_majority.js | 21 ++- .../replsets/tenant_migration_collection_rename.js | 5 + .../replsets/tenant_migration_collection_ttl.js | 23 +-- .../tenant_migration_commit_transaction_retry.js | 3 +- ...nant_migration_concurrent_reads_on_recipient.js | 8 +- ...ant_migration_concurrent_writes_on_recipient.js | 8 +- ...donor_unblock_reads_and_writes_on_completion.js | 68 +++++---- .../replsets/tenant_migration_drop_collection.js | 4 + ...enant_migration_fetch_committed_transactions.js | 3 +- ...migration_fetch_committed_transactions_retry.js | 24 ++-- jstests/replsets/tenant_migration_multi_writes.js | 4 + .../replsets/tenant_migration_multikey_index.js | 3 +- .../tenant_migration_network_error_via_rollback.js | 2 +- jstests/replsets/tenant_migration_no_failover.js | 3 +- .../tenant_migration_recipient_current_op.js | 155 +++++++++++---------- ..._does_not_change_sync_source_after_step_down.js | 4 + ...pient_fetches_retryable_writes_oplog_entries.js | 21 +-- ...ches_synthetic_find_and_modify_oplog_entries.js | 6 +- .../tenant_migration_recipient_has_tenant_data.js | 40 +++--- ...nt_migration_recipient_initial_sync_recovery.js | 2 +- .../tenant_migration_recipient_startup_recovery.js | 2 +- .../tenant_migration_retry_session_migration.js | 20 +-- .../tenant_migration_retryable_write_retry.js | 4 + ...migration_retryable_write_retry_on_recipient.js | 8 +- ...ard_merge_recipient_fetches_retryable_writes.js | 71 ++++++++++ ...nt_fetches_synthetic_find_and_modify_entries.js | 121 ++++++++++++++++ ...imeseries_retryable_write_retry_on_recipient.js | 11 +- src/mongo/db/repl/tenant_file_importer_service.cpp | 15 +- .../db/repl/tenant_migration_donor_service.cpp | 3 +- .../tenant_migration_recipient_op_observer.cpp | 2 +- .../db/repl/tenant_migration_recipient_service.cpp | 114 ++++++++------- .../db/repl/tenant_migration_recipient_service.h | 3 +- .../tenant_migration_recipient_service_test.cpp | 12 +- .../db/repl/tenant_migration_shard_merge_util.cpp | 2 +- 39 files changed, 572 insertions(+), 267 deletions(-) create mode 100644 jstests/replsets/tenant_migration_shard_merge_recipient_fetches_retryable_writes.js create mode 100644 jstests/replsets/tenant_migration_shard_merge_recipient_fetches_synthetic_find_and_modify_entries.js diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index cd6e212338f..56dc76ebbad 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -152,6 +152,26 @@ last-continuous: test_file: jstests/core/exhaust.js - ticket: SERVER-63141 test_file: jstests/aggregation/lookup_let_optimization.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_collection_ttl.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_current_op.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retry_session_migration.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js + - ticket: SERVER-63122 + test_file: jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js - ticket: SERVER-63129 test_file: jstests/replsets/tenant_migration_resume_collection_cloner_after_recipient_failover_with_dropped_views.js - ticket: SERVER-61864 diff --git a/jstests/replsets/libs/tenant_migration_test.js b/jstests/replsets/libs/tenant_migration_test.js index 7671112d677..b7b81e3f01d 100644 --- a/jstests/replsets/libs/tenant_migration_test.js +++ b/jstests/replsets/libs/tenant_migration_test.js @@ -477,8 +477,10 @@ function TenantMigrationTest({ */ this.verifyRecipientDB = function( tenantId, dbName, collName, migrationCommitted = true, data = loadDummyData()) { - const shouldMigrate = - migrationCommitted && TenantMigrationUtil.isNamespaceForTenant(tenantId, dbName); + // We should migrate all data regardless of tenant id for shard merge. + const shouldMigrate = migrationCommitted && + (TenantMigrationUtil.isShardMergeEnabled(this.getRecipientPrimary().getDB("admin")) || + TenantMigrationUtil.isNamespaceForTenant(tenantId, dbName)); jsTestLog(`Verifying that data in collection ${collName} of DB ${dbName} was ${ (shouldMigrate ? "" : "not")} migrated to the recipient`); diff --git a/jstests/replsets/tenant_migration_buildindex_shard_merge.js b/jstests/replsets/tenant_migration_buildindex_shard_merge.js index 16536d37a2b..dc4285d10ac 100644 --- a/jstests/replsets/tenant_migration_buildindex_shard_merge.js +++ b/jstests/replsets/tenant_migration_buildindex_shard_merge.js @@ -27,6 +27,14 @@ load("jstests/replsets/libs/tenant_migration_util.js"); // Index builds should be blocked by the tenant access blocker, not maxNumActiveUserIndexBuilds. const tenantMigrationTest = new TenantMigrationTest( {name: jsTestName(), sharedOptions: {setParameter: {maxNumActiveUserIndexBuilds: 100}}}); + +if (TenantMigrationUtil.isShardMergeEnabled(tenantMigrationTest.getDonorPrimary().getDB("admin"))) { + // TODO (SERVER-65084): Re-enable this test. + jsTestLog("Skip: Temporarily skipping test, see SERVER-65084."); + tenantMigrationTest.stop(); + return; +} + const donorPrimary = tenantMigrationTest.getDonorPrimary(); const kTenant1Id = "testTenantId1"; const kTenant2Id = "testTenantId2"; @@ -146,7 +154,6 @@ TenantMigrationTest.assertCommitted(migrationThread.returnData()); // The index creation threads should be done. racyIndexThread1.join(); -// TODO: remove, search everywhere racyIndexThread2.join(); tenant1IndexThread.join(); tenant2IndexThread.join(); diff --git a/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js b/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js index bbb3640c806..242566c2bac 100644 --- a/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js +++ b/jstests/replsets/tenant_migration_causal_consistency_commit_optime_before_last_cloning_optime.js @@ -2,9 +2,14 @@ * Verify that causal consistency is respected if a tenant migration commits with an earlier optime * timestamp than the latest optime associated with cloning on the recipient. * + * TODO (SERVER-61231): This test currently relies on a TenantCollectionCloner failpoint, which is + * not used by shard merge, but the behavior we are testing here is likely still relevant. Adapt + * for shard merge. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, diff --git a/jstests/replsets/tenant_migration_cloner_stats.js b/jstests/replsets/tenant_migration_cloner_stats.js index 790436f8d10..48fcfb1d56b 100644 --- a/jstests/replsets/tenant_migration_cloner_stats.js +++ b/jstests/replsets/tenant_migration_cloner_stats.js @@ -2,10 +2,14 @@ * Tests tenant migration cloner stats such as 'approxTotalDataSize', 'approxTotalBytesCopied' * across multiple databases and collections in the absence of failovers. * + * TODO SERVER-63517: incompatible_with_shard_merge because this specifically tests logical + * cloning behavior. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, * incompatible_with_windows_tls, + * incompatible_with_shard_merge, * requires_majority_read_concern, * requires_persistence, * serverless, diff --git a/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js b/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js index 731eb2a5ebe..dcb0ee7ac5d 100644 --- a/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js +++ b/jstests/replsets/tenant_migration_cloning_uses_read_concern_majority.js @@ -1,9 +1,14 @@ /** * Tests that in a tenant migration, the recipient primary will use majority read concern when * cloning documents from the donor. + * + * TODO (SERVER-63517): Remove this test, it requires failover, and the ability to write to the + * donor during migration, and it tests a cloning method superseded by Shard Merge. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, @@ -32,14 +37,6 @@ const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); const donorRst = tenantMigrationTest.getDonorRst(); const donorTestColl = donorPrimary.getDB(dbName).getCollection(collName); -// TODO (SERVER-63517): Remove this test, it requires failover, and the ability to write to the -// donor during migration, and it tests a cloning method superseded by Shard Merge. -if (TenantMigrationUtil.isShardMergeEnabled(donorRst.getPrimary().getDB("adminDB"))) { - jsTestLog("Skip: featureFlagShardMerge enabled, but shard merge does not survive failover"); - tenantMigrationTest.stop(); - return; -} - // The default WC is majority and stopReplicationOnSecondaries will prevent satisfying any majority assert.commandWorked(recipientPrimary.adminCommand( {setDefaultRWConcern: 1, defaultWriteConcern: {w: 1}, writeConcern: {w: "majority"}})); @@ -84,13 +81,13 @@ assert.eq(4, donorTestColl.find().itcount()); assert.eq(2, donorTestColl.find().readConcern("majority").itcount()); // Let the cloner finish. -const waitAfterCloning = - configureFailPoint(recipientDb, "fpAfterCollectionClonerDone", {action: "hang"}); +const waitBeforeFetchingTransactions = + configureFailPoint(recipientDb, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); waitBeforeCloning.off(); // Wait for the cloning phase to finish. Check that the recipient has only cloned documents that are // majority committed on the donor replica set. -waitAfterCloning.wait(); +waitBeforeFetchingTransactions.wait(); // Tenant migration recipient rejects all reads until it has applied data past the blockTimestamp // (returnAfterReachingTimestamp). Use this failpoint to allow the find command below to succeed. const notRejectReadsFp = @@ -101,7 +98,7 @@ notRejectReadsFp.off(); // Restart secondary replication in the donor replica set and complete the migration. restartReplicationOnSecondaries(donorRst); -waitAfterCloning.off(); +waitBeforeFetchingTransactions.off(); TenantMigrationTest.assertCommitted(migrationThread.returnData()); tenantMigrationTest.stop(); })(); diff --git a/jstests/replsets/tenant_migration_collection_rename.js b/jstests/replsets/tenant_migration_collection_rename.js index 0f5f8b55d3b..759eed60ff1 100644 --- a/jstests/replsets/tenant_migration_collection_rename.js +++ b/jstests/replsets/tenant_migration_collection_rename.js @@ -1,9 +1,14 @@ /** * Tests that tenant migrations aborts without crashing when a donor collection is renamed. * + * TODO SERVER-61231: shard merge does not use collection cloner, so we need another way + * to pause the migration at the correct time. What should shard merge behavior be for + * renaming a collection while a migration is underway? adapt this test + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_fcv_52, * requires_majority_read_concern, diff --git a/jstests/replsets/tenant_migration_collection_ttl.js b/jstests/replsets/tenant_migration_collection_ttl.js index a73bc44f00a..1f4515f568e 100644 --- a/jstests/replsets/tenant_migration_collection_ttl.js +++ b/jstests/replsets/tenant_migration_collection_ttl.js @@ -103,6 +103,12 @@ function assertTTLDeleteExpiredDocs(dbName, node) { // 1. At the recipient, the TTL deletions are suspended during the cloning phase. // 2. At the donor, TTL deletions are not suspended before blocking state. (() => { + if (TenantMigrationUtil.isShardMergeEnabled(donorPrimary.getDB("admin"))) { + jsTestLog( + "Skip: featureFlagShardMerge enabled, but shard merge does not use logical cloning"); + return; + } + jsTest.log("Test that the TTL does not delete documents on recipient during cloning"); const tenantId = "testTenantId-duringCloning"; @@ -122,11 +128,11 @@ function assertTTLDeleteExpiredDocs(dbName, node) { prepareDb(dbName, 3); const recipientDb = recipientPrimary.getDB(dbName); - const hangAfterCollectionClone = - configureFailPoint(recipientDb, "fpAfterCollectionClonerDone", {action: "hang"}); + const hangBeforeFetchingCommittedTransactions = + configureFailPoint(recipientDb, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); - hangAfterCollectionClone.wait(); + hangBeforeFetchingCommittedTransactions.wait(); // On a very slow machine, there is a chance that a TTL cycle happened at the donor before the // recipient cloned the documents. Therefore, these checks are only valid when we are sure the @@ -140,7 +146,7 @@ function assertTTLDeleteExpiredDocs(dbName, node) { assertTTLNotDeleteExpiredDocs(dbName, recipientPrimary); } - hangAfterCollectionClone.off(); + hangBeforeFetchingCommittedTransactions.off(); TenantMigrationTest.assertCommitted( tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); @@ -155,12 +161,12 @@ function assertTTLDeleteExpiredDocs(dbName, node) { })(); // Tests that: -// 1. At the recipient, the TTL deletions are suspended after the cloning phase until migration is -// forgotten. +// 1. At the recipient, the TTL deletions are suspended until migration is forgotten. // 2. At the donor, TTL deletions are suspended during blocking state. This verifies that // the TTL mechanism respects the same MTAB mechanism as normal updates. (() => { - jsTest.log("Test that the TTL does not delete documents on recipient after cloning"); + jsTest.log( + "Test that the TTL does not delete documents on recipient before migration is forgotten"); const tenantId = "testTenantId-afterCloning"; const dbName = tenantMigrationTest.tenantDB(tenantId, "testDB"); @@ -182,7 +188,8 @@ function assertTTLDeleteExpiredDocs(dbName, node) { let blockFp = configureFailPoint(donorPrimary, "pauseTenantMigrationBeforeLeavingBlockingState"); - assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); + assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); blockFp.wait(); // At a very slow machine, there is a chance that a TTL cycle happened at the donor diff --git a/jstests/replsets/tenant_migration_commit_transaction_retry.js b/jstests/replsets/tenant_migration_commit_transaction_retry.js index 992bb10584f..2d2eedb4bfe 100644 --- a/jstests/replsets/tenant_migration_commit_transaction_retry.js +++ b/jstests/replsets/tenant_migration_commit_transaction_retry.js @@ -122,7 +122,8 @@ const migrationOpts2 = { migrationIdString: extractUUIDFromObject(migrationId2), tenantId: kTenantId, }; -TenantMigrationTest.assertCommitted(tenantMigrationTest2.runMigration(migrationOpts2)); +TenantMigrationTest.assertCommitted( + tenantMigrationTest2.runMigration(migrationOpts2, {enableDonorStartMigrationFsync: true})); const recipientPrimary2 = tenantMigrationTest2.getRecipientPrimary(); const recipientTxnEntries2 = recipientPrimary2.getDB("config")["transactions"].find().toArray(); jsTestLog(`Recipient2 config.transactions: ${tojson(recipientTxnEntries2)}`); diff --git a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js index 07a59dbc2a1..761e5967446 100644 --- a/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js +++ b/jstests/replsets/tenant_migration_concurrent_reads_on_recipient.js @@ -67,14 +67,14 @@ function testRejectAllReadsAfterCloningDone({testCase, dbName, collName, tenantM const recipientRst = tenantMigrationTest.getRecipientRst(); const recipientPrimary = recipientRst.getPrimary(); - let clonerDoneFp = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + let beforeFetchingTransactionsFp = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); const donorRstArgs = TenantMigrationUtil.createRstArgs(donorRst); const runMigrationThread = new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs); runMigrationThread.start(); - clonerDoneFp.wait(); + beforeFetchingTransactionsFp.wait(); // Wait for the write to mark cloning as done to be replicated to all nodes. recipientRst.awaitReplication(); @@ -88,7 +88,7 @@ function testRejectAllReadsAfterCloningDone({testCase, dbName, collName, tenantM runCommand(db, command, ErrorCodes.SnapshotTooOld); }); - clonerDoneFp.off(); + beforeFetchingTransactionsFp.off(); TenantMigrationTest.assertCommitted(runMigrationThread.returnData()); assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); } diff --git a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js index de768ff147b..35923b68ec4 100644 --- a/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js +++ b/jstests/replsets/tenant_migration_concurrent_writes_on_recipient.js @@ -49,8 +49,8 @@ const kTenantId = "testTenantId"; configureFailPoint(recipientPrimary, "fpAfterStartingOplogFetcherMigrationRecipientInstance", {action: "hang"}); - let clonerDoneFp = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + let beforeFetchingTransactionsFp = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); let waitForRejectReadsBeforeTsFp = configureFailPoint( recipientPrimary, "fpAfterWaitForRejectReadsBeforeTimestamp", {action: "hang"}); @@ -67,12 +67,12 @@ const kTenantId = "testTenantId"; } startOplogFetcherFp.off(); - clonerDoneFp.wait(); + beforeFetchingTransactionsFp.wait(); // Write after cloning is done should fail with SnapshotTooOld since no read is allowed. assert.commandFailedWithCode(tenantCollOnRecipient.remove({_id: 1}), ErrorCodes.SnapshotTooOld); - clonerDoneFp.off(); + beforeFetchingTransactionsFp.off(); waitForRejectReadsBeforeTsFp.wait(); // Write after the recipient applied data past the rejectReadsBeforeTimestamp. diff --git a/jstests/replsets/tenant_migration_donor_unblock_reads_and_writes_on_completion.js b/jstests/replsets/tenant_migration_donor_unblock_reads_and_writes_on_completion.js index e9f56b09935..164cc6d5555 100644 --- a/jstests/replsets/tenant_migration_donor_unblock_reads_and_writes_on_completion.js +++ b/jstests/replsets/tenant_migration_donor_unblock_reads_and_writes_on_completion.js @@ -46,28 +46,36 @@ function startWriteThread(node, dbName, collName) { return writeThread; } -const donorRst = new ReplSetTest({ - nodes: 3, - name: "donorRst", - nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, { - setParameter: { - tenantMigrationGarbageCollectionDelayMS: 1, - ttlMonitorSleepSecs: 1, +function setup() { + const donorRst = new ReplSetTest({ + nodes: 3, + name: "donorRst", + nodeOptions: Object.assign(TenantMigrationUtil.makeX509OptionsForTest().donor, { + setParameter: { + tenantMigrationGarbageCollectionDelayMS: 1, + ttlMonitorSleepSecs: 1, + } + }), + // Disallow chaining to force both secondaries to sync from the primary. One of the test + // cases below disables replication on one of the secondaries, with chaining it would + // effectively disable replication on both secondaries, causing the migration to hang since + // majority write concern is unsatsifiable. + settings: {chainingAllowed: false} + }); + donorRst.startSet(); + donorRst.initiate(); + + const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst}); + + return { + donorRst, + tenantMigrationTest, + teardown: () => { + donorRst.stopSet(); + tenantMigrationTest.stop(); } - }), - // Disallow chaining to force both secondaries to sync from the primary. One of the test cases - // below disables replication on one of the secondaries, with chaining it would effectively - // disable replication on both secondaries, causing the migration to hang since majority - // write concern is unsatsifiable. - settings: {chainingAllowed: false} -}); -donorRst.startSet(); -donorRst.initiate(); - -const tenantMigrationTest = new TenantMigrationTest({name: jsTestName(), donorRst}); - -const donorPrimary = tenantMigrationTest.getDonorPrimary(); -const donorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); + }; +} const kTenantIdPrefix = "testTenantId"; const kDbName = "testDb"; @@ -76,6 +84,9 @@ const kCollName = "testColl"; (() => { jsTest.log( "Test that a lagged donor secondary correctly unblocks blocked reads after the migration aborts"); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + const donorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); const tenantId = kTenantIdPrefix + "LaggedSecondaryMigrationAborted"; const dbName = tenantId + "_" + kDbName; assert.commandWorked( @@ -116,11 +127,16 @@ const kCollName = "testColl"; assert.commandWorked(readThread.returnData()); abortFp.off(); snapshotFp.off(); + + teardown(); })(); (() => { jsTest.log( "Test that a lagged donor secondary correctly unblocks blocked reads after the migration commits"); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + const donorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); const tenantId = kTenantIdPrefix + "LaggedSecondaryMigrationCommitted"; const dbName = tenantId + "_" + kDbName; assert.commandWorked( @@ -158,11 +174,16 @@ const kCollName = "testColl"; assert.commandFailedWithCode(readThread.returnData(), ErrorCodes.TenantMigrationCommitted); snapshotFp.off(); + + teardown(); })(); (() => { jsTest.log( "Test that blocked writes and reads are interrupted when the donor's state doc collection is dropped"); + const {tenantMigrationTest, donorRst, teardown} = setup(); + const donorPrimary = tenantMigrationTest.getDonorPrimary(); + const donorsColl = donorPrimary.getCollection(TenantMigrationTest.kConfigDonorsNS); const tenantId = kTenantIdPrefix + "DropStateDocCollection"; const dbName = tenantId + "_" + kDbName; assert.commandWorked( @@ -201,8 +222,7 @@ const kCollName = "testColl"; assert.commandFailedWithCode(readThread.returnData(), ErrorCodes.Interrupted); assert.commandFailedWithCode(writeThread.returnData(), ErrorCodes.Interrupted); blockingFp.off(); -})(); -donorRst.stopSet(); -tenantMigrationTest.stop(); + teardown(); +})(); })(); diff --git a/jstests/replsets/tenant_migration_drop_collection.js b/jstests/replsets/tenant_migration_drop_collection.js index b040d39738e..c3b13f5c992 100644 --- a/jstests/replsets/tenant_migration_drop_collection.js +++ b/jstests/replsets/tenant_migration_drop_collection.js @@ -2,9 +2,13 @@ * Tests that TenantCollectionCloner completes without error when a collection is dropped during * cloning as part of a tenant migration. * + * TODO SERVER-61231: relies on various failpoints and such in TenantCollectionCloner, which is + * not used for by Shard Merge, but we should likely test similar behavior, adapt for Shard Merge + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, diff --git a/jstests/replsets/tenant_migration_fetch_committed_transactions.js b/jstests/replsets/tenant_migration_fetch_committed_transactions.js index 22b9d37e519..4d4c3bd5f7b 100644 --- a/jstests/replsets/tenant_migration_fetch_committed_transactions.js +++ b/jstests/replsets/tenant_migration_fetch_committed_transactions.js @@ -124,7 +124,8 @@ const migrationOpts = { const pauseAfterRetrievingLastTxnMigrationRecipientInstance = configureFailPoint(recipientPrimary, "pauseAfterRetrievingLastTxnMigrationRecipientInstance"); -assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); +assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); pauseAfterRetrievingLastTxnMigrationRecipientInstance.wait(); diff --git a/jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js b/jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js index 3c33072711d..d1bb969a219 100644 --- a/jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js +++ b/jstests/replsets/tenant_migration_fetch_committed_transactions_retry.js @@ -103,10 +103,10 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { donorRst.restart(donorPrimary); // Let the migration restart and hang before it tries to re-fetch committed transactions. - const fpAfterCollectionClonerDone = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + const fpBeforeFetchingTransactions = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); fpAfterFetchingCommittedTransactions.off(); - fpAfterCollectionClonerDone.wait(); + fpBeforeFetchingTransactions.wait(); // The recipient should indicate that the migration has restarted. let recipientDoc; @@ -117,7 +117,7 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { // The state doc should indicate that the migration has already updated 'config.transaction' // entries. assert.eq(true, recipientDoc[0].completedUpdatingTransactionsBeforeStartOpTime); - fpAfterCollectionClonerDone.off(); + fpBeforeFetchingTransactions.off(); // Verify that the migration completes successfully. TenantMigrationTest.assertCommitted( @@ -171,10 +171,10 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { donorRst.restart(donorPrimary); // Let the migration restart and hang before it tries to re-fetch committed transactions. - const fpAfterCollectionClonerDone = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + const fpBeforeFetchingTransactions = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); hangAfterUpdatingTransactionEntry.off(); - fpAfterCollectionClonerDone.wait(); + fpBeforeFetchingTransactions.wait(); // The recipient should indicate that the migration has restarted. let recipientDoc; @@ -185,7 +185,7 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { // Verify that the 'completedUpdatingTransactionsBeforeStartOpTime' flag is false since the // migration was forced to restart before it fully completed fetching. assert.eq(false, recipientDoc[0].completedUpdatingTransactionsBeforeStartOpTime); - fpAfterCollectionClonerDone.off(); + fpBeforeFetchingTransactions.off(); // Verify that the migration completes successfully. TenantMigrationTest.assertCommitted( @@ -250,10 +250,10 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { donorRst.restart(donorPrimary); // Let the migration restart and hang before it tries to re-fetch committed transactions. - const fpAfterCollectionClonerDone = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + const fpBeforeFetchingTransactions = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); hangAfterUpdatingTransactionEntry.off(); - fpAfterCollectionClonerDone.wait(); + fpBeforeFetchingTransactions.wait(); // The recipient should indicate that the migration has restarted. let recipientDoc; @@ -264,7 +264,7 @@ const assertTransactionEntries = (donorTxnEntries, recipientTxnEntries) => { // Verify that the 'completedUpdatingTransactionsBeforeStartOpTime' flag is false since the // migration was forced to restart before it fully completed fetching. assert.eq(false, recipientDoc[0].completedUpdatingTransactionsBeforeStartOpTime); - fpAfterCollectionClonerDone.off(); + fpBeforeFetchingTransactions.off(); // Verify that the migration completes successfully. TenantMigrationTest.assertCommitted( diff --git a/jstests/replsets/tenant_migration_multi_writes.js b/jstests/replsets/tenant_migration_multi_writes.js index aa05e251432..4901684df43 100644 --- a/jstests/replsets/tenant_migration_multi_writes.js +++ b/jstests/replsets/tenant_migration_multi_writes.js @@ -3,9 +3,13 @@ * were not retried on migration abort, which would create duplicate updates. Partially * updated collection where each update is applied no more than once is still an expected result. * + * TODO SERVER-61231: aborts migration after sending recipientSyncData and starting + * cloning on recipient, adapt this test to handle file cleanup on recipient. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, diff --git a/jstests/replsets/tenant_migration_multikey_index.js b/jstests/replsets/tenant_migration_multikey_index.js index 196a546ee5f..ab1b654951d 100644 --- a/jstests/replsets/tenant_migration_multikey_index.js +++ b/jstests/replsets/tenant_migration_multikey_index.js @@ -91,7 +91,8 @@ const fpBeforeFulfillingDataConsistentPromise = configureFailPoint( recipientPrimary, "fpBeforeFulfillingDataConsistentPromise", {action: "hang"}); jsTestLog("Starting the tenant migration"); -assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); +assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); fpBeforeFulfillingDataConsistentPromise.wait(); diff --git a/jstests/replsets/tenant_migration_network_error_via_rollback.js b/jstests/replsets/tenant_migration_network_error_via_rollback.js index ea115802f20..cdb09f63577 100644 --- a/jstests/replsets/tenant_migration_network_error_via_rollback.js +++ b/jstests/replsets/tenant_migration_network_error_via_rollback.js @@ -284,7 +284,7 @@ switch (caseNum) { case 11: jsTestLog("[11] Testing rollback after finishing cloning."); runTest({ - failPointName: "fpAfterCollectionClonerDone", + failPointName: "fpBeforeFetchingCommittedTransactions", failPointData: { action: "hang", }, diff --git a/jstests/replsets/tenant_migration_no_failover.js b/jstests/replsets/tenant_migration_no_failover.js index a1084fc30b9..f0d52e0d304 100644 --- a/jstests/replsets/tenant_migration_no_failover.js +++ b/jstests/replsets/tenant_migration_no_failover.js @@ -38,7 +38,8 @@ const migrationOpts = { tenantId, }; -TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); +TenantMigrationTest.assertCommitted( + tenantMigrationTest.runMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); for (const db of [...tenantDBs, ...nonTenantDBs]) { for (const coll of collNames) { diff --git a/jstests/replsets/tenant_migration_recipient_current_op.js b/jstests/replsets/tenant_migration_recipient_current_op.js index 526cec91ddc..c5d88734a00 100644 --- a/jstests/replsets/tenant_migration_recipient_current_op.js +++ b/jstests/replsets/tenant_migration_recipient_current_op.js @@ -78,21 +78,24 @@ function checkPostConsistentFieldsOK(res) { assert(currOp.hasOwnProperty("dataConsistentStopDonorOpTime") && checkOptime(currOp.dataConsistentStopDonorOpTime), res); - assert(currOp.hasOwnProperty("cloneFinishedRecipientOpTime") && - checkOptime(currOp.cloneFinishedRecipientOpTime), - res); - assert(currOp.hasOwnProperty("approxTotalDataSize") && - currOp.approxTotalDataSize instanceof NumberLong, - res); - assert(currOp.hasOwnProperty("approxTotalBytesCopied") && - currOp.approxTotalBytesCopied instanceof NumberLong, - res); - assert(currOp.hasOwnProperty("totalReceiveElapsedMillis") && - currOp.totalReceiveElapsedMillis instanceof NumberLong, - res); - assert(currOp.hasOwnProperty("remainingReceiveEstimatedMillis") && - currOp.remainingReceiveEstimatedMillis instanceof NumberLong, - res); + + if (!shardMergeIsEnabled) { + assert(currOp.hasOwnProperty("cloneFinishedRecipientOpTime") && + checkOptime(currOp.cloneFinishedRecipientOpTime), + res); + assert(currOp.hasOwnProperty("approxTotalDataSize") && + currOp.approxTotalDataSize instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("approxTotalBytesCopied") && + currOp.approxTotalBytesCopied instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("totalReceiveElapsedMillis") && + currOp.totalReceiveElapsedMillis instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("remainingReceiveEstimatedMillis") && + currOp.remainingReceiveEstimatedMillis instanceof NumberLong, + res); + } } // Validates the fields of an optime object. @@ -112,8 +115,8 @@ const fpAfterPersistingStateDoc = {action: "hang"}); const fpAfterRetrievingStartOpTime = configureFailPoint( recipientPrimary, "fpAfterRetrievingStartOpTimesMigrationRecipientInstance", {action: "hang"}); -const fpAfterCollectionCloner = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); +const fpBeforeFetchingTransactions = + configureFailPoint(recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); const fpAfterDataConsistent = configureFailPoint( recipientPrimary, "fpAfterDataConsistentMigrationRecipientInstance", {action: "hang"}); const fpAfterForgetMigration = configureFailPoint( @@ -121,7 +124,8 @@ const fpAfterForgetMigration = configureFailPoint( jsTestLog("Starting tenant migration with migrationId: " + kMigrationId + ", tenantId: " + kTenantId); -assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); +assert.commandWorked( + tenantMigrationTest.startMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); // Wait until a current operation corresponding to "tenant recipient migration" with state kStarted // is visible on the recipientPrimary. @@ -137,13 +141,15 @@ assert.eq(currOp.dataSyncCompleted, false, res); assert(!currOp.hasOwnProperty("startFetchingDonorOpTime"), res); assert(!currOp.hasOwnProperty("startApplyingDonorOpTime"), res); assert(!currOp.hasOwnProperty("dataConsistentStopDonorOpTime"), res); -assert(!currOp.hasOwnProperty("cloneFinishedRecipientOpTime"), res); assert(!currOp.hasOwnProperty("expireAt"), res); assert(!currOp.hasOwnProperty("donorSyncSource"), res); -assert(!currOp.hasOwnProperty("approxTotalDataSize"), res); -assert(!currOp.hasOwnProperty("approxTotalBytesCopied"), res); -assert(!currOp.hasOwnProperty("totalReceiveElapsedMillis"), res); -assert(!currOp.hasOwnProperty("remainingReceiveEstimatedMillis"), res); +if (!shardMergeIsEnabled) { + assert(!currOp.hasOwnProperty("cloneFinishedRecipientOpTime"), res); + assert(!currOp.hasOwnProperty("approxTotalDataSize"), res); + assert(!currOp.hasOwnProperty("approxTotalBytesCopied"), res); + assert(!currOp.hasOwnProperty("totalReceiveElapsedMillis"), res); + assert(!currOp.hasOwnProperty("remainingReceiveEstimatedMillis"), res); +} fpAfterPersistingStateDoc.off(); // Allow the migration to move to the point where the startFetchingDonorOpTime has been obtained. @@ -158,13 +164,14 @@ assert.eq(currOp.state, TenantMigrationTest.RecipientStateEnum.kStarted, res); assert.eq(currOp.migrationCompleted, false, res); assert.eq(currOp.dataSyncCompleted, false, res); assert(!currOp.hasOwnProperty("dataConsistentStopDonorOpTime"), res); -assert(!currOp.hasOwnProperty("cloneFinishedRecipientOpTime"), res); assert(!currOp.hasOwnProperty("expireAt"), res); -assert(!currOp.hasOwnProperty("approxTotalDataSize"), res); -assert(!currOp.hasOwnProperty("approxTotalBytesCopied"), res); -assert(!currOp.hasOwnProperty("totalReceiveElapsedMillis"), res); -assert(!currOp.hasOwnProperty("remainingReceiveEstimatedMillis"), res); -// Must exist now. +if (!shardMergeIsEnabled) { + assert(!currOp.hasOwnProperty("cloneFinishedRecipientOpTime"), res); + assert(!currOp.hasOwnProperty("approxTotalDataSize"), res); + assert(!currOp.hasOwnProperty("approxTotalBytesCopied"), res); + assert(!currOp.hasOwnProperty("totalReceiveElapsedMillis"), res); + assert(!currOp.hasOwnProperty("remainingReceiveEstimatedMillis"), res); +} assert(currOp.hasOwnProperty("startFetchingDonorOpTime") && checkOptime(currOp.startFetchingDonorOpTime), res); @@ -174,10 +181,8 @@ assert(currOp.hasOwnProperty("startApplyingDonorOpTime") && assert(currOp.hasOwnProperty("donorSyncSource") && typeof currOp.donorSyncSource === 'string', res); fpAfterRetrievingStartOpTime.off(); -// Wait until collection cloning is done, and cloneFinishedRecipientOpTime -// and dataConsistentStopDonorOpTime are visible. -jsTestLog("Waiting for collection cloning to complete."); -fpAfterCollectionCloner.wait(); +jsTestLog("Waiting until we are ready to fetch committed transactions."); +fpBeforeFetchingTransactions.wait(); res = recipientPrimary.adminCommand({currentOp: true, desc: "tenant recipient migration"}); checkStandardFieldsOK(res); @@ -203,22 +208,24 @@ assert(currOp.hasOwnProperty("donorSyncSource") && typeof currOp.donorSyncSource assert(currOp.hasOwnProperty("dataConsistentStopDonorOpTime") && checkOptime(currOp.dataConsistentStopDonorOpTime), res); -assert(currOp.hasOwnProperty("cloneFinishedRecipientOpTime") && - checkOptime(currOp.cloneFinishedRecipientOpTime), - res); -assert(currOp.hasOwnProperty("approxTotalDataSize") && - currOp.approxTotalDataSize instanceof NumberLong, - res); -assert(currOp.hasOwnProperty("approxTotalBytesCopied") && - currOp.approxTotalBytesCopied instanceof NumberLong, - res); -assert(currOp.hasOwnProperty("totalReceiveElapsedMillis") && - currOp.totalReceiveElapsedMillis instanceof NumberLong, - res); -assert(currOp.hasOwnProperty("remainingReceiveEstimatedMillis") && - currOp.remainingReceiveEstimatedMillis instanceof NumberLong, - res); -fpAfterCollectionCloner.off(); +if (!shardMergeIsEnabled) { + assert(currOp.hasOwnProperty("cloneFinishedRecipientOpTime") && + checkOptime(currOp.cloneFinishedRecipientOpTime), + res); + assert(currOp.hasOwnProperty("approxTotalDataSize") && + currOp.approxTotalDataSize instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("approxTotalBytesCopied") && + currOp.approxTotalBytesCopied instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("totalReceiveElapsedMillis") && + currOp.totalReceiveElapsedMillis instanceof NumberLong, + res); + assert(currOp.hasOwnProperty("remainingReceiveEstimatedMillis") && + currOp.remainingReceiveEstimatedMillis instanceof NumberLong, + res); +} +fpBeforeFetchingTransactions.off(); // Wait for the "kConsistent" state to be reached. jsTestLog("Waiting for the kConsistent state to be reached."); @@ -290,30 +297,32 @@ assert.eq(currOp.dataSyncCompleted, true, res); assert.eq(currOp.state, TenantMigrationTest.RecipientStateEnum.kDone, res); assert.eq(currOp.migrationCompleted, true, res); assert(currOp.hasOwnProperty("expireAt") && currOp.expireAt instanceof Date, res); -assert(currOp.hasOwnProperty("databases")); -assert.eq(0, currOp.databases.databasesClonedBeforeFailover, tojson(res)); -assert.eq(dbsToClone.length, currOp.databases.databasesToClone, tojson(res)); -assert.eq(dbsToClone.length, currOp.databases.databasesCloned, tojson(res)); -for (const db of dbsToClone) { - const tenantDB = tenantMigrationTest.tenantDB(kTenantId, db); - assert(currOp.databases.hasOwnProperty(tenantDB), tojson(res)); - const dbStats = currOp.databases[tenantDB]; - assert.eq(0, dbStats.clonedCollectionsBeforeFailover, tojson(res)); - assert.eq(collsToClone.length, dbStats.collections, tojson(res)); - assert.eq(collsToClone.length, dbStats.clonedCollections, tojson(res)); - assert(dbStats.hasOwnProperty("start"), tojson(res)); - assert(dbStats.hasOwnProperty("end"), tojson(res)); - assert.neq(0, dbStats.elapsedMillis, tojson(res)); - for (const coll of collsToClone) { - assert(dbStats.hasOwnProperty(`${tenantDB}.${coll}`), tojson(res)); - const collStats = dbStats[`${tenantDB}.${coll}`]; - assert.eq(docs.length, collStats.documentsToCopy, tojson(res)); - assert.eq(docs.length, collStats.documentsCopied, tojson(res)); - assert.eq(1, collStats.indexes, tojson(res)); - assert.eq(collStats.insertedBatches, collStats.receivedBatches, tojson(res)); - assert(collStats.hasOwnProperty("start"), tojson(res)); - assert(collStats.hasOwnProperty("end"), tojson(res)); - assert.neq(0, collStats.elapsedMillis, tojson(res)); +if (!shardMergeIsEnabled) { + assert(currOp.hasOwnProperty("databases")); + assert.eq(0, currOp.databases.databasesClonedBeforeFailover, tojson(res)); + assert.eq(dbsToClone.length, currOp.databases.databasesToClone, tojson(res)); + assert.eq(dbsToClone.length, currOp.databases.databasesCloned, tojson(res)); + for (const db of dbsToClone) { + const tenantDB = tenantMigrationTest.tenantDB(kTenantId, db); + assert(currOp.databases.hasOwnProperty(tenantDB), tojson(res)); + const dbStats = currOp.databases[tenantDB]; + assert.eq(0, dbStats.clonedCollectionsBeforeFailover, tojson(res)); + assert.eq(collsToClone.length, dbStats.collections, tojson(res)); + assert.eq(collsToClone.length, dbStats.clonedCollections, tojson(res)); + assert(dbStats.hasOwnProperty("start"), tojson(res)); + assert(dbStats.hasOwnProperty("end"), tojson(res)); + assert.neq(0, dbStats.elapsedMillis, tojson(res)); + for (const coll of collsToClone) { + assert(dbStats.hasOwnProperty(`${tenantDB}.${coll}`), tojson(res)); + const collStats = dbStats[`${tenantDB}.${coll}`]; + assert.eq(docs.length, collStats.documentsToCopy, tojson(res)); + assert.eq(docs.length, collStats.documentsCopied, tojson(res)); + assert.eq(1, collStats.indexes, tojson(res)); + assert.eq(collStats.insertedBatches, collStats.receivedBatches, tojson(res)); + assert(collStats.hasOwnProperty("start"), tojson(res)); + assert(collStats.hasOwnProperty("end"), tojson(res)); + assert.neq(0, collStats.elapsedMillis, tojson(res)); + } } } diff --git a/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js b/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js index 8378912b83e..62c899fa87c 100644 --- a/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js +++ b/jstests/replsets/tenant_migration_recipient_does_not_change_sync_source_after_step_down.js @@ -2,9 +2,13 @@ * Test that in tenant migration, the recipient does not change sync source * even after its current sync source steps down as primary. * + * TODO SERVER-63517: incompatible_with_shard_merge because this relies on + * logical cloning behavior. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, diff --git a/jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js b/jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js index a13329f4d50..4b2bc334f0f 100644 --- a/jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js +++ b/jstests/replsets/tenant_migration_recipient_fetches_retryable_writes_oplog_entries.js @@ -2,10 +2,15 @@ * Tests that the tenant migration recipient correctly fetches retryable writes oplog entries * and adds them to its oplog buffer. * + * TODO SERVER-63517: incompatible_with_shard_merge, this tests specific implementation + * details related to MT Migrations. Retryable write behavior is tested in various other + * tests for shard merge. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, * incompatible_with_windows_tls, + * incompatible_with_shard_merge, * requires_majority_read_concern, * requires_persistence, * serverless, @@ -39,7 +44,7 @@ const kParams = { internalInsertMaxBatchSize: kMaxBatchSize, }; -function runTest(storeImagesInSideCollection) { +function runTest({storeFindAndModifyImagesInSideCollection = false}) { const tenantMigrationTest = new TenantMigrationTest( {name: jsTestName(), sharedOptions: {nodes: 1, setParameter: kParams}}); @@ -53,7 +58,7 @@ function runTest(storeImagesInSideCollection) { const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); const setParam = { setParameter: 1, - storeFindAndModifyImagesInSideCollection: storeImagesInSideCollection + storeFindAndModifyImagesInSideCollection, }; donorPrimary.adminCommand(setParam); recipientPrimary.adminCommand(setParam); @@ -99,10 +104,10 @@ function runTest(storeImagesInSideCollection) { // `startFetchingDonorOpTime`. const writeFp = configureFailPoint(donorPrimary, "hangDuringBatchInsert", {}, {skip: 1}); - var batchInsertWorker = new Thread((host, dbName, collName, numToInsert) => { + const batchInsertWorker = new Thread((host, dbName, collName, numToInsert) => { // Insert elements [{_id: bulkRetryableWrite0}, {_id: bulkRetryableWrite1}]. const docsToInsert = - [...Array(numToInsert).keys()].map(i => ({_id: "bulkRetryableWrite" + i})); + [...Array(numToInsert).keys()].map(i => ({_id: `bulkRetryableWrite${i}`})); donorConn = new Mongo(host); const tenantSession4 = donorConn.startSession({retryWrites: true}); @@ -164,7 +169,7 @@ function runTest(storeImagesInSideCollection) { fpAfterRetrievingStartOpTime.off(); fpAfterRetrievingRetryableWrites.wait(); - const kOplogBufferNS = "repl.migration.oplog_" + migrationOpts.migrationIdString; + const kOplogBufferNS = `repl.migration.oplog_${migrationOpts.migrationIdString}`; const recipientOplogBuffer = recipientPrimary.getDB("config")[kOplogBufferNS]; jsTestLog({"oplog buffer ns": kOplogBufferNS}); @@ -195,8 +200,6 @@ function runTest(storeImagesInSideCollection) { tenantMigrationTest.stop(); } -// Run test with `storeFindAndModifyImagesInSideCollection`=false. -runTest(false); -// Run test with `storeFindAndModifyImagesInSideCollection`=true. -runTest(true); +runTest({storeFindAndModifyImagesInSideCollection: false}); +runTest({storeFindAndModifyImagesInSideCollection: true}); })(); diff --git a/jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js b/jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js index 05708a93e7e..bc0848a10d7 100644 --- a/jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js +++ b/jstests/replsets/tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js @@ -6,6 +6,7 @@ * incompatible_with_eft, * incompatible_with_macos, * incompatible_with_windows_tls, + * incompatible_with_shard_merge, * requires_majority_read_concern, * requires_persistence, * serverless, @@ -28,8 +29,7 @@ if (!RetryableWritesUtil.storageEngineSupportsRetryableWrites(jsTest.options().s const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); const kTenantId = "testTenantId"; -const kDbName = kTenantId + "_" + - "testDb"; +const kDbName = `${kTenantId}_testDb`; const kCollName = "testColl"; const donorPrimary = tenantMigrationTest.getDonorPrimary(); @@ -75,7 +75,7 @@ const fpAfterPreFetchingRetryableWrites = configureFailPoint( fpBeforeRetrievingStartOpTime.off(); fpAfterPreFetchingRetryableWrites.wait(); -const kOplogBufferNS = "repl.migration.oplog_" + migrationOpts.migrationIdString; +const kOplogBufferNS = `repl.migration.oplog_${migrationOpts.migrationIdString}`; const recipientOplogBuffer = recipientPrimary.getDB("config")[kOplogBufferNS]; jsTestLog({"oplog buffer ns": kOplogBufferNS}); let res = recipientOplogBuffer.find({"entry.o._id": "retryableWrite"}).toArray(); diff --git a/jstests/replsets/tenant_migration_recipient_has_tenant_data.js b/jstests/replsets/tenant_migration_recipient_has_tenant_data.js index 6f607cc9f77..2bcb99a60c0 100644 --- a/jstests/replsets/tenant_migration_recipient_has_tenant_data.js +++ b/jstests/replsets/tenant_migration_recipient_has_tenant_data.js @@ -6,6 +6,7 @@ * incompatible_with_eft, * incompatible_with_macos, * incompatible_with_windows_tls, + * incompatible_with_shard_merge, * requires_majority_read_concern, * requires_persistence, * serverless, @@ -38,8 +39,12 @@ const donorRst = new ReplSetTest({ donorRst.startSet(); donorRst.initiate(); -const tenantMigrationTest = new TenantMigrationTest( - {name: jsTestName(), donorRst, sharedOptions: {setParameter: kGarbageCollectionParams}}); +const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + donorRst, + quickGarbageCollection: true, + sharedOptions: {setParameter: kGarbageCollectionParams} +}); const kTenantId = "testTenantId"; const kNs = kTenantId + "_testDb.testColl"; @@ -48,32 +53,25 @@ assert.commandWorked(tenantMigrationTest.getDonorPrimary().getCollection(kNs).in jsTest.log("Start a tenant migration and verify that it commits successfully"); -(() => { - const migrationId = UUID(); - const migrationOpts = { - migrationIdString: extractUUIDFromObject(migrationId), - tenantId: kTenantId, - }; +let migrationId = UUID(); +const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, +}; - TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); - assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); - tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, kTenantId); -})(); +TenantMigrationTest.assertCommitted( + tenantMigrationTest.runMigration(migrationOpts, {enableDonorStartMigrationFsync: true})); +tenantMigrationTest.waitForMigrationGarbageCollection(migrationId, kTenantId); jsTest.log( "Retry the migration without dropping tenant database on the recipient and verify that " + "the migration aborted with NamespaceExists as the abort reason"); -(() => { - const migrationId = UUID(); - const migrationOpts = { - migrationIdString: extractUUIDFromObject(migrationId), - tenantId: kTenantId, - }; +migrationId = UUID(); +migrationOpts.migrationIdString = extractUUIDFromObject(migrationId); - TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration(migrationOpts), - ErrorCodes.NamespaceExists); -})(); +TenantMigrationTest.assertAborted(tenantMigrationTest.runMigration(migrationOpts), + ErrorCodes.NamespaceExists); donorRst.stopSet(); tenantMigrationTest.stop(); diff --git a/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js b/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js index e243a9cdbdf..d79224aef6e 100644 --- a/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js +++ b/jstests/replsets/tenant_migration_recipient_initial_sync_recovery.js @@ -33,7 +33,7 @@ let recipientPrimary = tenantMigrationTest.getRecipientPrimary(); // Force the migration to pause after entering a randomly selected state. Random.setRandomSeed(); const kMigrationFpNames = [ - "fpAfterCollectionClonerDone", + "fpBeforeFetchingCommittedTransactions", "fpAfterWaitForRejectReadsBeforeTimestamp", ]; const index = Random.randInt(kMigrationFpNames.length + 1); diff --git a/jstests/replsets/tenant_migration_recipient_startup_recovery.js b/jstests/replsets/tenant_migration_recipient_startup_recovery.js index 1f39895994f..26e76979df0 100644 --- a/jstests/replsets/tenant_migration_recipient_startup_recovery.js +++ b/jstests/replsets/tenant_migration_recipient_startup_recovery.js @@ -44,7 +44,7 @@ let recipientPrimary = tenantMigrationTest.getRecipientPrimary(); // Force the migration to pause after entering a randomly selected state. Random.setRandomSeed(); const kMigrationFpNames = [ - "fpAfterCollectionClonerDone", + "fpBeforeFetchingCommittedTransactions", "fpAfterWaitForRejectReadsBeforeTimestamp", ]; const index = Random.randInt(kMigrationFpNames.length + 1); diff --git a/jstests/replsets/tenant_migration_retry_session_migration.js b/jstests/replsets/tenant_migration_retry_session_migration.js index 0b3d4783fc6..35b8961bed6 100644 --- a/jstests/replsets/tenant_migration_retry_session_migration.js +++ b/jstests/replsets/tenant_migration_retry_session_migration.js @@ -2,9 +2,13 @@ * Tests that retrying a failed tenant migration works even if the config.transactions on the * recipient is not cleaned up after the failed migration. * + * TODO SERVER-61231: aborts migration after sending recipientSyncData and starting + * cloning on recipient, adapt this test to handle file cleanup on recipient. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, + * incompatible_with_shard_merge, * incompatible_with_windows_tls, * requires_majority_read_concern, * requires_persistence, @@ -32,8 +36,8 @@ const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); tenantMigrationTest.insertDonorDB(kDbName, kCollName, [{_id: 1}, {_id: 2}]); -let waitAfterCloning = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); +let waitBeforeFetchingTransactions = + configureFailPoint(recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); const migrationId = UUID(); const migrationOpts = { @@ -42,7 +46,7 @@ const migrationOpts = { }; tenantMigrationTest.startMigration(migrationOpts); -waitAfterCloning.wait(); +waitBeforeFetchingTransactions.wait(); // Run transactions against the donor while the migration is running. const session1 = donorPrimary.startSession(); @@ -77,7 +81,7 @@ for (const lsid of [lsid1, lsid2]) { // Abort the first migration. const abortFp = configureFailPoint(donorPrimary, "abortTenantMigrationBeforeLeavingBlockingState"); -waitAfterCloning.off(); +waitBeforeFetchingTransactions.off(); TenantMigrationTest.assertAborted(tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString); @@ -88,13 +92,13 @@ abortFp.off(); // Clean up tenant data after a failed migration. assert.commandWorked(recipientPrimary.getDB(kDbName).dropDatabase()); -waitAfterCloning = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); +waitBeforeFetchingTransactions = + configureFailPoint(recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); // Retry the migration. tenantMigrationTest.startMigration(migrationOpts); -waitAfterCloning.wait(); +waitBeforeFetchingTransactions.wait(); // Run a newer transaction on session2 during the migration. session2.startTransaction({writeConcern: {w: "majority"}}); @@ -113,7 +117,7 @@ assert.commandWorked(donorPrimary.getDB(kDbName).runCommand({ lsid: lsid2 })); -waitAfterCloning.off(); +waitBeforeFetchingTransactions.off(); TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); diff --git a/jstests/replsets/tenant_migration_retryable_write_retry.js b/jstests/replsets/tenant_migration_retryable_write_retry.js index b445dea7342..ee34d613642 100644 --- a/jstests/replsets/tenant_migration_retryable_write_retry.js +++ b/jstests/replsets/tenant_migration_retryable_write_retry.js @@ -2,10 +2,14 @@ * Tests aggregation pipeline for cloning oplog chains for retryable writes on the tenant migration * donor that committed before a certain donor Timestamp. * + * Relies on MT Migrations implementation details, overall end-to-end behavior of migrating + * retryable writes is tested elsewhere. + * * @tags: [ * incompatible_with_eft, * incompatible_with_macos, * incompatible_with_windows_tls, + * incompatible_with_shard_merge, * requires_majority_read_concern, * requires_persistence, * serverless, diff --git a/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js b/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js index 35751c03771..7b7292f85a3 100644 --- a/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js +++ b/jstests/replsets/tenant_migration_retryable_write_retry_on_recipient.js @@ -44,8 +44,8 @@ if (TenantMigrationUtil.isShardMergeEnabled(donorPrimary.getDB("adminDB"))) { } jsTestLog("Run a migration to the end of cloning"); -const waitAfterCloning = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); +const waitBeforeFetchingTransactions = + configureFailPoint(recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); const migrationId = UUID(); const migrationOpts = { @@ -169,7 +169,7 @@ const donorRstArgs = TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDo const migrationThread = new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs); migrationThread.start(); -waitAfterCloning.wait(); +waitBeforeFetchingTransactions.wait(); jsTestLog("Run retryable writes during the migration"); @@ -184,7 +184,7 @@ assert.commandWorked( // Wait for the migration to complete. jsTest.log("Waiting for migration to complete"); -waitAfterCloning.off(); +waitBeforeFetchingTransactions.off(); TenantMigrationTest.assertCommitted(migrationThread.returnData()); // Print the no-op oplog entries for debugging purposes. diff --git a/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_retryable_writes.js b/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_retryable_writes.js new file mode 100644 index 00000000000..e37d579d742 --- /dev/null +++ b/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_retryable_writes.js @@ -0,0 +1,71 @@ +/** + * Tests that the shard merge recipient correctly fetches retryable writes. + * + * @tags: [ + * incompatible_with_eft, + * incompatible_with_macos, + * incompatible_with_windows_tls, + * featureFlagShardMerge, + * requires_majority_read_concern, + * requires_persistence, + * serverless, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/retryable_writes_util.js"); +load("jstests/replsets/libs/tenant_migration_test.js"); +load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). + +if (!RetryableWritesUtil.storageEngineSupportsRetryableWrites(jsTest.options().storageEngine)) { + jsTestLog("Retryable writes are not supported, skipping test"); + return; +} + +const kParams = { + ttlMonitorSleepSecs: 1, +}; + +const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + sharedOptions: {nodes: 1, setParameter: kParams}, + quickGarbageCollection: true +}); + +const kTenantId = "testTenantId"; + +const donorRst = tenantMigrationTest.getDonorRst(); +const donorPrimary = tenantMigrationTest.getDonorPrimary(); +const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + +jsTestLog("Run retryable write prior to the migration"); + +const lsid = UUID(); +const cmd = { + insert: "collection", + documents: [{_id: 1}, {_id: 2}], + ordered: false, + lsid: {id: lsid}, + txnNumber: NumberLong(123), +}; + +assert.commandWorked(donorPrimary.getDB("database").runCommand(cmd)); +assert.eq(2, donorPrimary.getDB("database").collection.find().itcount()); + +const migrationId = UUID(); +const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, +}; + +jsTestLog(`Starting migration: ${tojson(migrationOpts)}`); +TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration(migrationOpts)); + +const {ok, n} = assert.commandWorked(recipientPrimary.getDB("database").runCommand(cmd)); +assert.eq(1, ok); +assert.eq(2, n); +assert.eq(2, recipientPrimary.getDB("database").collection.find().itcount()); + +tenantMigrationTest.stop(); +})(); diff --git a/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_synthetic_find_and_modify_entries.js b/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_synthetic_find_and_modify_entries.js new file mode 100644 index 00000000000..6159cc5c307 --- /dev/null +++ b/jstests/replsets/tenant_migration_shard_merge_recipient_fetches_synthetic_find_and_modify_entries.js @@ -0,0 +1,121 @@ +/** + * Tests that the shard merge recipient correctly prefetches synthetic findAndModify oplog + * entries with timestamp less than the 'startFetchingDonorTimestamp'. Note, this test is + * based off of tenant_migration_recipient_fetches_synthetic_find_and_modify_oplog_entries.js + * but avoids testing implementation details that are not relevant to shard merge. + * + * @tags: [ + * incompatible_with_eft, + * incompatible_with_macos, + * incompatible_with_windows_tls, + * featureFlagShardMerge, + * requires_majority_read_concern, + * requires_persistence, + * serverless, + * ] + */ +(function() { +"use strict"; + +load("jstests/libs/retryable_writes_util.js"); +load("jstests/replsets/libs/tenant_migration_test.js"); +load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). +load("jstests/libs/fail_point_util.js"); // For configureFailPoint(). +load("jstests/libs/parallelTester.js"); // For Thread. + +if (!RetryableWritesUtil.storageEngineSupportsRetryableWrites(jsTest.options().storageEngine)) { + jsTestLog("Retryable writes are not supported, skipping test"); + return; +} + +const tenantMigrationTest = new TenantMigrationTest({name: jsTestName()}); + +const kTenantId = "testTenantId"; +const kDbName = `${kTenantId}_testDb`; +const kCollName = "testColl"; + +const donorPrimary = tenantMigrationTest.getDonorPrimary(); +const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + +const tenantCollection = donorPrimary.getDB(kDbName)[kCollName]; + +jsTestLog("Run retryable findAndModify prior to the migration startFetchingDonorOpTime"); + +const migrationId = UUID(); +const migrationOpts = { + migrationIdString: extractUUIDFromObject(migrationId), + tenantId: kTenantId, +}; + +// Hang before we get the 'startFetchingDonorOpTime'. +const fpBeforeRetrievingStartOpTime = + configureFailPoint(recipientPrimary, "fpAfterComparingRecipientAndDonorFCV", {action: "hang"}); + +jsTestLog(`Starting migration: ${tojson(migrationOpts)}`); +assert.commandWorked(tenantMigrationTest.startMigration(migrationOpts)); +fpBeforeRetrievingStartOpTime.wait(); + +const lsid1 = UUID(); +const lsid2 = UUID(); +const cmds = [ + { + // Retryable write with `postImageOpTime`. + findAndModify: kCollName, + query: {_id: "retryableWrite"}, + update: {$set: {x: 1}}, + new: true, + upsert: true, + lsid: {id: lsid1}, + txnNumber: NumberLong(2), + writeConcern: {w: "majority"}, + }, + { + findAndModify: kCollName, + query: {_id: "otherRetryableWrite"}, + update: {$inc: {count: 1}}, + new: false, + upsert: true, + lsid: {id: lsid2}, + txnNumber: NumberLong(2), + writeConcern: {w: "majority"}, + } +]; +assert.commandWorked(tenantCollection.insert({_id: "retryableWrite", count: 0})); +assert.commandWorked(tenantCollection.insert({_id: "otherRetryableWrite", count: 0})); +const [cmdResponse1, cmdResponse2] = + cmds.map(cmd => assert.commandWorked(donorPrimary.getDB(kDbName).runCommand(cmd))); + +// Release the previous failpoint to hang after fetching the retryable writes entries before the +// 'startFetchingDonorOpTime'. +const fpAfterPreFetchingRetryableWrites = configureFailPoint( + recipientPrimary, "fpAfterFetchingRetryableWritesEntriesBeforeStartOpTime", {action: "hang"}); +fpBeforeRetrievingStartOpTime.off(); +fpAfterPreFetchingRetryableWrites.wait(); + +// Resume the migration. +fpAfterPreFetchingRetryableWrites.off(); + +jsTestLog("Wait for migration to complete"); +TenantMigrationTest.assertCommitted(tenantMigrationTest.waitForMigrationToComplete(migrationOpts)); + +// Test that retrying the findAndModify commands on the recipient will give us the same results and +// pre or post image. +const [retryResponse1, retryResponse2] = + cmds.map(cmd => assert.commandWorked(recipientPrimary.getDB(kDbName).runCommand(cmd))); +[[cmdResponse1, retryResponse1], [cmdResponse2, retryResponse2]].forEach( + ([cmdResponse, retryResponse]) => { + // The retry response can contain a different 'clusterTime' and 'operationTime' from the + // initial response. + delete cmdResponse.$clusterTime; + delete retryResponse.$clusterTime; + delete cmdResponse.operationTime; + delete retryResponse.operationTime; + // The retry response contains the "retriedStmtId" field but the initial response does not. + delete retryResponse.retriedStmtId; + }); +assert.eq(0, bsonWoCompare(cmdResponse1, retryResponse1), retryResponse1); +assert.eq(0, bsonWoCompare(cmdResponse2, retryResponse2), retryResponse2); + +assert.commandWorked(tenantMigrationTest.forgetMigration(migrationOpts.migrationIdString)); +tenantMigrationTest.stop(); +})(); diff --git a/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js b/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js index e1be377bc20..5fc0eb8b9ee 100644 --- a/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js +++ b/jstests/replsets/tenant_migration_timeseries_retryable_write_retry_on_recipient.js @@ -41,9 +41,8 @@ function testRetryOnRecipient(ordered) { const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); const recipientDb = recipientPrimary.getDB(kDbName); - jsTestLog("Run a migration to the end of cloning"); - const waitAfterCloning = - configureFailPoint(recipientPrimary, "fpAfterCollectionClonerDone", {action: "hang"}); + const waitBeforeFetchingTransactions = configureFailPoint( + recipientPrimary, "fpBeforeFetchingCommittedTransactions", {action: "hang"}); const migrationId = UUID(); const migrationOpts = { @@ -96,14 +95,15 @@ function testRetryOnRecipient(ordered) { const migrationThread = new Thread(TenantMigrationUtil.runMigrationAsync, migrationOpts, donorRstArgs); migrationThread.start(); - waitAfterCloning.wait(); + + waitBeforeFetchingTransactions.wait(); jsTestLog("Run retryable writes during the migration"); assert.commandWorked(donorDb.runCommand(duringWrites.retryableInsertCommand)); // Wait for the migration to complete. jsTest.log("Waiting for migration to complete"); - waitAfterCloning.off(); + waitBeforeFetchingTransactions.off(); TenantMigrationTest.assertCommitted(migrationThread.returnData()); // Print the no-op oplog entries for debugging purposes. @@ -124,6 +124,7 @@ function testRetryOnRecipient(ordered) { jsTestLog("Run retryable write on primary after the migration"); testRecipientRetryableWrites(recipientDb, beforeWrites); testRecipientRetryableWrites(recipientDb, duringWrites); + jsTestLog("Step up secondary"); const recipientRst = tenantMigrationTest.getRecipientRst(); recipientRst.awaitReplication(); diff --git a/src/mongo/db/repl/tenant_file_importer_service.cpp b/src/mongo/db/repl/tenant_file_importer_service.cpp index 0c1e3f1e44b..b315b7f43d3 100644 --- a/src/mongo/db/repl/tenant_file_importer_service.cpp +++ b/src/mongo/db/repl/tenant_file_importer_service.cpp @@ -107,12 +107,15 @@ void importCopiedFiles(OperationContext* opCtx, donorConnectionString); } - // TODO SERVER-63122: Remove the try-catch block once logical cloning is removed for - // shard merge protocol. - try { - wiredTigerImportFromBackupCursor(opCtx, metadatas, tempWTDirectory.string()); - } catch (const ExceptionFor& ex) { - LOGV2_WARNING(6113314, "Temporarily ignoring the error", "error"_attr = ex.toStatus()); + wiredTigerImportFromBackupCursor(opCtx, metadatas, tempWTDirectory.string()); + + auto catalog = CollectionCatalog::get(opCtx); + for (auto&& m : metadatas) { + Lock::CollectionLock systemViewsLock( + opCtx, + NamespaceString(m.ns.db(), NamespaceString::kSystemDotViewsCollectionName), + MODE_X); + uassertStatusOK(catalog->reloadViews(opCtx, m.ns.db())); } } } // namespace diff --git a/src/mongo/db/repl/tenant_migration_donor_service.cpp b/src/mongo/db/repl/tenant_migration_donor_service.cpp index 820e4eb18c7..dbbd56c654f 100644 --- a/src/mongo/db/repl/tenant_migration_donor_service.cpp +++ b/src/mongo/db/repl/tenant_migration_donor_service.cpp @@ -1353,7 +1353,8 @@ ExecutorFuture TenantMigrationDonorService::Instance::_handleErrorOrEnterA LOGV2(6104912, "Entering 'aborted' state.", "migrationId"_attr = _migrationUuid, - "tenantId"_attr = _tenantId); + "tenantId"_attr = _tenantId, + "status"_attr = status); // Enter "abort" state. _abortReason.emplace(status); return _updateStateDoc(executor, TenantMigrationDonorStateEnum::kAborted, token) diff --git a/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp b/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp index 46b4d34355a..e52cdecc8b3 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp @@ -58,7 +58,7 @@ const auto migrationIdToDeleteDecoration = * Initializes the TenantMigrationRecipientAccessBlocker for the tenant migration denoted by the * given state doc. * - * TODO (SERVER-63122): Skip for protocol kShardMerge. + * TODO (SERVER-64616): Skip for protocol kShardMerge. */ void createAccessBlockerIfNeeded(OperationContext* opCtx, const TenantMigrationRecipientDocument& recipientStateDoc) { diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp index 5341f2e4fd7..3f32411c555 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp @@ -161,7 +161,8 @@ MONGO_FAIL_POINT_DEFINE(fpAfterStartingOplogFetcherMigrationRecipientInstance); MONGO_FAIL_POINT_DEFINE(setTenantMigrationRecipientInstanceHostTimeout); MONGO_FAIL_POINT_DEFINE(pauseAfterRetrievingLastTxnMigrationRecipientInstance); MONGO_FAIL_POINT_DEFINE(fpBeforeMarkingCollectionClonerDone); -MONGO_FAIL_POINT_DEFINE(fpAfterCollectionClonerDone); +MONGO_FAIL_POINT_DEFINE(fpBeforeFetchingCommittedTransactions); +MONGO_FAIL_POINT_DEFINE(fpAfterFetchingCommittedTransactions); MONGO_FAIL_POINT_DEFINE(fpAfterStartingOplogApplierMigrationRecipientInstance); MONGO_FAIL_POINT_DEFINE(fpBeforeFulfillingDataConsistentPromise); MONGO_FAIL_POINT_DEFINE(fpAfterDataConsistentMigrationRecipientInstance); @@ -173,7 +174,6 @@ MONGO_FAIL_POINT_DEFINE(hangAfterCreatingRSM); MONGO_FAIL_POINT_DEFINE(skipRetriesWhenConnectingToDonorHost); MONGO_FAIL_POINT_DEFINE(fpBeforeDroppingOplogBufferCollection); MONGO_FAIL_POINT_DEFINE(fpWaitUntilTimestampMajorityCommitted); -MONGO_FAIL_POINT_DEFINE(fpAfterFetchingCommittedTransactions); MONGO_FAIL_POINT_DEFINE(hangAfterUpdatingTransactionEntry); MONGO_FAIL_POINT_DEFINE(fpBeforeAdvancingStableTimestamp); MONGO_FAIL_POINT_DEFINE(hangMigrationBeforeRetryCheck); @@ -1113,14 +1113,12 @@ void TenantMigrationRecipientService::Instance::_getStartOpTimesFromDonor(WithLo return; } - // We only expect to already have start optimes populated if we are not - // resuming a migration and this is a multitenant migration. - // TODO (SERVER-61145) Eventually we'll skip _getStartopTimesFromDonor entirely - // for shard merge, but currently _getDonorFilenames will populate optimes for - // the shard merge case. We can just overwrite here since we aren't doing anything - // with the backup cursor results yet. auto isShardMerge = _stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge; - invariant(isShardMerge || !_stateDoc.getStartApplyingDonorOpTime().has_value()); + if (isShardMerge) { + invariant(_stateDoc.getStartApplyingDonorOpTime().has_value()); + } else { + invariant(!_stateDoc.getStartApplyingDonorOpTime().has_value()); + } invariant(isShardMerge || !_stateDoc.getStartFetchingDonorOpTime().has_value()); auto lastOplogEntry1OpTime = _getDonorMajorityOpTime(_client); @@ -1154,21 +1152,20 @@ void TenantMigrationRecipientService::Instance::_getStartOpTimesFromDonor(WithLo pauseAfterRetrievingLastTxnMigrationRecipientInstance.pauseWhileSet(); - // We need to fetch the last oplog entry both before and after getting the transaction - // table entry, as otherwise there is a potential race where we may try to apply - // a commit for which we have not fetched a previous transaction oplog entry. - auto lastOplogEntry2OpTime = _getDonorMajorityOpTime(_client); - LOGV2_DEBUG(4880604, - 2, - "Found last oplog entry at the read concern majority optime (after reading txn " - "table) on remote node", - "migrationId"_attr = getMigrationUUID(), - "tenantId"_attr = _stateDoc.getTenantId(), - "lastOplogEntry"_attr = lastOplogEntry2OpTime.toBSON()); - - // TODO (SERVER-61145) We'll want to skip this as well for shard merge, since this should - // be set in _getDonorFilenames. - _stateDoc.setStartApplyingDonorOpTime(lastOplogEntry2OpTime); + if (!isShardMerge) { + // We need to fetch the last oplog entry both before and after getting the transaction + // table entry, as otherwise there is a potential race where we may try to apply + // a commit for which we have not fetched a previous transaction oplog entry. + auto lastOplogEntry2OpTime = _getDonorMajorityOpTime(_client); + LOGV2_DEBUG(4880604, + 2, + "Found last oplog entry at the read concern majority optime (after reading txn " + "table) on remote node", + "migrationId"_attr = getMigrationUUID(), + "tenantId"_attr = _stateDoc.getTenantId(), + "lastOplogEntry"_attr = lastOplogEntry2OpTime.toBSON()); + _stateDoc.setStartApplyingDonorOpTime(lastOplogEntry2OpTime); + } OpTime startFetchingDonorOpTime = lastOplogEntry1OpTime; if (!earliestOpenTransactionBson.isEmpty()) { @@ -1178,7 +1175,11 @@ void TenantMigrationRecipientService::Instance::_getStartOpTimesFromDonor(WithLo startFetchingDonorOpTime = OpTime::parse(startOpTimeField.Obj()); } } - _stateDoc.setStartFetchingDonorOpTime(startFetchingDonorOpTime); + + // TODO SERVER-61145: This change can lead to missing oplog chain with timestamps + // between startFetchingDonorOpTime and startApplyingDonorOpTime. + _stateDoc.setStartFetchingDonorOpTime( + std::min(startFetchingDonorOpTime, *_stateDoc.getStartApplyingDonorOpTime())); } AggregateCommandRequest @@ -1775,12 +1776,13 @@ bool TenantMigrationRecipientService::Instance::_isCloneCompletedMarkerSet(WithL return _stateDoc.getCloneFinishedRecipientOpTime().has_value(); } -OpTime TenantMigrationRecipientService::Instance::_getOplogResumeApplyingDonorOptime( - const OpTime startApplyingDonorOpTime, const OpTime cloneFinishedRecipientOpTime) const { - { +OpTime TenantMigrationRecipientService::Instance::_getOplogResumeApplyingDonorOptime() const { + auto cloneFinishedRecipientOpTime = [this, self = shared_from_this()] { stdx::lock_guard lk(_mutex); - invariant(_stateDoc.getCloneFinishedRecipientOpTime().has_value()); - } + auto opt = _stateDoc.getCloneFinishedRecipientOpTime(); + invariant(opt.has_value()); + return *opt; + }(); auto opCtx = cc().makeOperationContext(); OplogInterfaceLocal oplog(opCtx.get()); auto oplogIter = oplog.makeIterator(); @@ -2148,7 +2150,6 @@ void TenantMigrationRecipientService::Instance::_interrupt(Status status, if (_taskState.isNotStarted()) { invariant(skipWaitingForForgetMigration); _stateDocPersistedPromise.setError(status); - _dataSyncStartedPromise.setError(status); _dataConsistentPromise.setError(status); _dataSyncCompletionPromise.setError(status); @@ -2219,7 +2220,6 @@ void TenantMigrationRecipientService::Instance::_cleanupOnDataSyncCompletion(Sta invariant(!status.isOK()); setPromiseErrorifNotReady(lk, _stateDocPersistedPromise, status); - setPromiseErrorifNotReady(lk, _dataSyncStartedPromise, status); setPromiseErrorifNotReady(lk, _dataConsistentPromise, status); setPromiseErrorifNotReady(lk, _dataSyncCompletionPromise, status); @@ -2616,18 +2616,11 @@ SemiFuture TenantMigrationRecipientService::Instance::run( OpTime beginApplyingAfterOpTime; Timestamp resumeBatchingTs; if (_isCloneCompletedMarkerSet(lk)) { - // We are retrying from failure. Find the point at which we should resume - // oplog batching and oplog application. - const auto startApplyingDonorOpTime = - *_stateDoc.getStartApplyingDonorOpTime(); - const auto cloneFinishedRecipientOptime = - *_stateDoc.getCloneFinishedRecipientOpTime(); lk.unlock(); // We avoid holding the mutex while scanning the local oplog which // acquires the RSTL in IX mode. This is to allow us to be interruptable // via a concurrent stepDown which acquires the RSTL in X mode. - const auto resumeOpTime = _getOplogResumeApplyingDonorOptime( - startApplyingDonorOpTime, cloneFinishedRecipientOptime); + const auto resumeOpTime = _getOplogResumeApplyingDonorOptime(); if (!resumeOpTime.isNull()) { // It's possible we've applied retryable writes no-op oplog entries // with donor opTimes earlier than 'startApplyingDonorOpTime'. In @@ -2635,15 +2628,19 @@ SemiFuture TenantMigrationRecipientService::Instance::run( // 'beginApplyingAfterOpTime'. resumeBatchingTs = resumeOpTime.getTimestamp(); } + + lk.lock(); + + // We are retrying from failure. Find the point at which we should resume + // oplog batching and oplog application. beginApplyingAfterOpTime = - std::max(resumeOpTime, startApplyingDonorOpTime); + std::max(resumeOpTime, *_stateDoc.getStartApplyingDonorOpTime()); LOGV2_DEBUG(5394601, 1, "Resuming oplog application from previous tenant " "migration attempt", "startApplyingDonorOpTime"_attr = beginApplyingAfterOpTime, "resumeBatchingOpTime"_attr = resumeOpTime); - lk.lock(); } else { beginApplyingAfterOpTime = *_stateDoc.getStartApplyingDonorOpTime(); } @@ -2673,23 +2670,28 @@ SemiFuture TenantMigrationRecipientService::Instance::run( **_scopedExecutor, _writerPool.get(), resumeBatchingTs); - - // Start the cloner. - auto clonerFuture = _startTenantAllDatabaseCloner(lk); - - // Signal that the data sync has started successfully. - if (!_dataSyncStartedPromise.getFuture().isReady()) { - _dataSyncStartedPromise.emplaceValue(); + }) + .then([this, self = shared_from_this()] { + if (_stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge) { + return Future::makeReady(); + } + stdx::lock_guard lk(_mutex); + return _startTenantAllDatabaseCloner(lk); + }) + .then([this, self = shared_from_this()] { + if (_stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge) { + return SemiFuture::makeReady(); } - return clonerFuture; + return _onCloneSuccess(); }) - .then([this, self = shared_from_this()] { return _onCloneSuccess(); }) .then([this, self = shared_from_this()] { if (_stateDoc.getProtocol() != MigrationProtocolEnum::kShardMerge) { return SemiFuture::makeReady(); } stdx::lock_guard lk(_mutex); + _stateDoc.setDataConsistentStopDonorOpTime( + _stateDoc.getStartApplyingDonorOpTime()); _stateDoc.setState(TenantMigrationRecipientStateEnum::kLearnedFilenames); return _updateStateDocForMajority(lk); }) @@ -2706,7 +2708,8 @@ SemiFuture TenantMigrationRecipientService::Instance::run( .then([this, self = shared_from_this()] { { auto opCtx = cc().makeOperationContext(); - _stopOrHangOnFailPoint(&fpAfterCollectionClonerDone, opCtx.get()); + _stopOrHangOnFailPoint(&fpBeforeFetchingCommittedTransactions, + opCtx.get()); } return _fetchCommittedTransactionsBeforeStartOpTime(); }) @@ -2719,8 +2722,13 @@ SemiFuture TenantMigrationRecipientService::Instance::run( "migrationId"_attr = getMigrationUUID()); { stdx::lock_guard lk(_mutex); + auto cloneFinishedRecipientOpTime = + _stateDoc.getProtocol() == MigrationProtocolEnum::kShardMerge + ? repl::ReplicationCoordinator::get(cc().getServiceContext()) + ->getMyLastAppliedOpTime() + : *_stateDoc.getCloneFinishedRecipientOpTime(); _tenantOplogApplier->setCloneFinishedRecipientOpTime( - *_stateDoc.getCloneFinishedRecipientOpTime()); + cloneFinishedRecipientOpTime); uassertStatusOK(_tenantOplogApplier->startup()); _isRestartingOplogApplier = false; _restartOplogApplierCondVar.notify_all(); diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h index 5ffe03b072d..6972379144f 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.h +++ b/src/mongo/db/repl/tenant_migration_recipient_service.h @@ -448,8 +448,7 @@ public: * should resume from. The oplog applier should resume applying entries that have a greater * optime than the returned value. */ - OpTime _getOplogResumeApplyingDonorOptime(OpTime startApplyingDonorOpTime, - OpTime cloneFinishedRecipientOpTime) const; + OpTime _getOplogResumeApplyingDonorOptime() const; /* * Starts the tenant cloner. diff --git a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp index 0f732622b9a..2763fe2bfd6 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service_test.cpp @@ -1582,7 +1582,7 @@ TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartOplogFe } TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientStartsCloner) { - stopFailPointEnableBlock fp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions"); auto taskFp = globalFailPointRegistry().find("hangBeforeTaskCompletion"); ScopeGuard taskFpGuard([&taskFp] { taskFp->setMode(FailPoint::off); }); @@ -2610,7 +2610,7 @@ TEST_F(TenantMigrationRecipientServiceTest, StoppingApplierAllowsCompletion) { } TEST_F(TenantMigrationRecipientServiceTest, TenantMigrationRecipientAddResumeTokenNoopsToBuffer) { - stopFailPointEnableBlock fp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3026,7 +3026,7 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientForgetMigration_AfterFail) auto autoForgetFp = globalFailPointRegistry().find("autoRecipientForgetMigration"); autoForgetFp->setMode(FailPoint::off); - stopFailPointEnableBlock fp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock fp("fpBeforeFetchingCommittedTransactions"); const UUID migrationUUID = UUID::gen(); const OpTime topOfOplogOpTime(Timestamp(5, 1), 1); @@ -3376,7 +3376,7 @@ TEST_F(TenantMigrationRecipientServiceTest, WaitUntilMigrationReachesReturnAfter } TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableFetcherError) { - stopFailPointEnableBlock stopFp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock stopFp("fpBeforeFetchingCommittedTransactions"); auto fp = globalFailPointRegistry().find("fpAfterStartingOplogFetcherMigrationRecipientInstance"); auto initialTimesEntered = fp->setMode(FailPoint::alwaysOn, @@ -3632,7 +3632,7 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientWillNotRetryOnReceivingForg } TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableClonerError) { - stopFailPointEnableBlock stopFp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock stopFp("fpBeforeFetchingCommittedTransactions"); auto fp = globalFailPointRegistry().find("fpAfterStartingOplogFetcherMigrationRecipientInstance"); auto initialTimesEntered = fp->setMode(FailPoint::alwaysOn, @@ -3705,7 +3705,7 @@ TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesRetriableClonerErro } TEST_F(TenantMigrationRecipientServiceTest, RecipientReceivesNonRetriableClonerError) { - stopFailPointEnableBlock stopFp("fpAfterCollectionClonerDone"); + stopFailPointEnableBlock stopFp("fpBeforeFetchingCommittedTransactions"); auto fp = globalFailPointRegistry().find("fpAfterStartingOplogFetcherMigrationRecipientInstance"); auto initialTimesEntered = fp->setMode(FailPoint::alwaysOn, diff --git a/src/mongo/db/repl/tenant_migration_shard_merge_util.cpp b/src/mongo/db/repl/tenant_migration_shard_merge_util.cpp index ae0e4b0bb5e..99f65df2274 100644 --- a/src/mongo/db/repl/tenant_migration_shard_merge_util.cpp +++ b/src/mongo/db/repl/tenant_migration_shard_merge_util.cpp @@ -63,7 +63,7 @@ using namespace fmt::literals; void moveFile(const std::string& src, const std::string& dst) { LOGV2_DEBUG(6114304, 1, "Moving file", "src"_attr = src, "dst"_attr = dst); - tassert(6114401, + uassert(6114401, "Destination file '{}' already exists"_format(dst), !boost::filesystem::exists(dst)); -- cgit v1.2.1