diff options
5 files changed, 173 insertions, 83 deletions
diff --git a/etc/backports_required_for_multiversion_tests.yml b/etc/backports_required_for_multiversion_tests.yml index daa724fc5c5..61c8c5ac3cf 100644 --- a/etc/backports_required_for_multiversion_tests.yml +++ b/etc/backports_required_for_multiversion_tests.yml @@ -264,6 +264,8 @@ last-continuous: ticket: SERVER-68728 - test_file: jstests/sharding/resharding_critical_section_metrics.js ticket: SERVER-68932 + - test_file: jstests/replsets/tenant_migration_recipient_forget_migration.js + ticket: SERVER-69229 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_cannot_vote_twice_same_term.js ticket: SERVER-69861 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_election_during_storage_change.js @@ -619,6 +621,8 @@ last-lts: ticket: SERVER-68628 - test_file: jstests/sharding/move_chunk_interrupt_postimage.js ticket: SERVER-68728 + - test_file: jstests/replsets/tenant_migration_recipient_forget_migration.js + ticket: SERVER-69229 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_cannot_vote_twice_same_term.js ticket: SERVER-69861 - test_file: src/mongo/db/modules/enterprise/jstests/fcbis/fcbis_election_during_storage_change.js diff --git a/jstests/replsets/tenant_migration_recipient_forget_migration.js b/jstests/replsets/tenant_migration_recipient_forget_migration.js new file mode 100644 index 00000000000..e3d5cd36a8a --- /dev/null +++ b/jstests/replsets/tenant_migration_recipient_forget_migration.js @@ -0,0 +1,145 @@ +/** + * Tests forgetMigration cleanup behavior. + * + * @tags: [ + * incompatible_with_macos, + * incompatible_with_windows_tls, + * requires_persistence, + * requires_replication, + * serverless, + * ] + */ + +(function() { + +"use strict"; +load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). +load("jstests/libs/fail_point_util.js"); // For configureFailPoint(). +load("jstests/libs/parallelTester.js"); // For Thread(), used for async forgetMigration. +load("jstests/replsets/libs/tenant_migration_test.js"); +load("jstests/replsets/libs/tenant_migration_util.js"); + +const tenantMigrationTest = new TenantMigrationTest({ + name: jsTestName(), + sharedOptions: {nodes: 2}, + quickGarbageCollection: true, +}); + +const kTenantId = "testTenantId"; +const kReadPreference = { + mode: "primary" +}; + +const isShardMergeEnabled = + TenantMigrationUtil.isShardMergeEnabled(tenantMigrationTest.getDonorPrimary().getDB("admin")); + +const oplogBufferCollectionName = (migrationIdString) => + `repl.migration.oplog_${migrationIdString}`; +const donatedFilesCollectionName = (migrationIdString) => `donatedFiles.${migrationIdString}`; + +const assertTempCollectionsExist = (conn, migrationIdString) => { + const collections = conn.getDB("config").getCollectionNames(); + assert(collections.includes(oplogBufferCollectionName(migrationIdString)), collections); + if (isShardMergeEnabled) { + assert(collections.includes(donatedFilesCollectionName(migrationIdString)), collections); + } +}; + +const assertTempCollectionsDoNotExist = (conn, migrationIdString) => { + const collections = conn.getDB("config").getCollectionNames(); + assert(!collections.includes(oplogBufferCollectionName(migrationIdString)), collections); + if (isShardMergeEnabled) { + assert(!collections.includes(donatedFilesCollectionName(migrationIdString)), collections); + } +}; + +(() => { + jsTestLog("Test that expected collections are cleaned up when forgetting a migration."); + const kMigrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(kMigrationId), + tenantId: kTenantId, + readPreference: kReadPreference + }; + + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration( + migrationOpts, {retryOnRetryableErrors: true, automaticForgetMigration: false})); + + const fpBeforeDroppingTempCollections = + configureFailPoint(tenantMigrationTest.getRecipientPrimary(), + "fpBeforeDroppingTempCollections", + {action: "hang"}); + + jsTestLog("Issuing a forget migration command."); + const forgetMigrationThread = + new Thread(TenantMigrationUtil.forgetMigrationAsync, + migrationOpts.migrationIdString, + TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()), + true /* retryOnRetryableErrors */); + forgetMigrationThread.start(); + + fpBeforeDroppingTempCollections.wait(); + + const recipientPrimary = tenantMigrationTest.getRecipientPrimary(); + + assertTempCollectionsExist(recipientPrimary, migrationOpts.migrationIdString); + + fpBeforeDroppingTempCollections.off(); + + jsTestLog("Waiting for forget migration to complete."); + assert.commandWorked(forgetMigrationThread.returnData()); + + assertTempCollectionsDoNotExist(recipientPrimary, migrationOpts.migrationIdString); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +})(); + +(() => { + jsTestLog( + "Tests whether the new recipient primary properly processes a forgetMigration when " + + "the original primary steps down after the migration is marked as garbage collectable."); + const kMigrationId = UUID(); + const migrationOpts = { + migrationIdString: extractUUIDFromObject(kMigrationId), + tenantId: kTenantId, + readPreference: kReadPreference + }; + + TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration( + migrationOpts, {retryOnRetryableErrors: true, automaticForgetMigration: false})); + + const fpBeforeDroppingTempCollections = + configureFailPoint(tenantMigrationTest.getRecipientPrimary(), + "fpBeforeDroppingTempCollections", + {action: "hang"}); + + jsTestLog("Issuing a forget migration command."); + const forgetMigrationThread = + new Thread(TenantMigrationUtil.forgetMigrationAsync, + migrationOpts.migrationIdString, + TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()), + true /* retryOnRetryableErrors */); + forgetMigrationThread.start(); + + fpBeforeDroppingTempCollections.wait(); + + assertTempCollectionsExist(tenantMigrationTest.getRecipientPrimary(), + migrationOpts.migrationIdString); + + jsTestLog("Stepping up a new recipient primary."); + tenantMigrationTest.getRecipientRst().stepUp( + tenantMigrationTest.getRecipientRst().getSecondaries()[0]); + + fpBeforeDroppingTempCollections.off(); + + jsTestLog("Waiting for forget migration to complete."); + assert.commandWorked(forgetMigrationThread.returnData()); + + assertTempCollectionsDoNotExist(tenantMigrationTest.getRecipientPrimary(), + migrationOpts.migrationIdString); + + tenantMigrationTest.waitForMigrationGarbageCollection(migrationOpts.migrationIdString); +})(); + +tenantMigrationTest.stop(); +})(); diff --git a/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js b/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js deleted file mode 100644 index 3e542cabaec..00000000000 --- a/jstests/replsets/tenant_migration_recipient_stepdown_after_forget.js +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Tests whether the new recipient primary properly processes a forgetMigration when the original - * primary is made to step down after marking as garbage collectable. The oplog buffer collection - * must be dropped. - * - * @tags: [ - * incompatible_with_macos, - * incompatible_with_windows_tls, - * requires_persistence, - * requires_replication, - * serverless, - * ] - */ - -(function() { - -"use strict"; -load("jstests/libs/uuid_util.js"); // For extractUUIDFromObject(). -load("jstests/libs/fail_point_util.js"); // For configureFailPoint(). -load("jstests/libs/parallelTester.js"); // For Thread(), used for async forgetMigration. -load("jstests/replsets/libs/tenant_migration_test.js"); -load("jstests/replsets/libs/tenant_migration_util.js"); - -const tenantMigrationTest = - new TenantMigrationTest({name: jsTestName(), sharedOptions: {nodes: 2}}); - -const kMigrationId = UUID(); -const kTenantId = 'testTenantId'; -const kReadPreference = { - mode: "primary" -}; -const migrationOpts = { - migrationIdString: extractUUIDFromObject(kMigrationId), - tenantId: kTenantId, - readPreference: kReadPreference -}; - -TenantMigrationTest.assertCommitted(tenantMigrationTest.runMigration( - migrationOpts, {retryOnRetryableErrors: true, automaticForgetMigration: false})); - -const fpBeforeDroppingOplogBufferCollection = - configureFailPoint(tenantMigrationTest.getRecipientPrimary(), - "fpBeforeDroppingOplogBufferCollection", - {action: "hang"}); - -jsTestLog("Issuing a forget migration command."); -const forgetMigrationThread = - new Thread(TenantMigrationUtil.forgetMigrationAsync, - migrationOpts.migrationIdString, - TenantMigrationUtil.createRstArgs(tenantMigrationTest.getDonorRst()), - true /* retryOnRetryableErrors */); -forgetMigrationThread.start(); - -fpBeforeDroppingOplogBufferCollection.wait(); - -jsTestLog("Step up a new recipient primary."); -tenantMigrationTest.getRecipientRst().stepUp( - tenantMigrationTest.getRecipientRst().getSecondaries()[0]); - -fpBeforeDroppingOplogBufferCollection.off(); - -jsTestLog("Waiting for forget migration to complete."); -assert.commandWorked(forgetMigrationThread.returnData()); - -const configDBCollections = - tenantMigrationTest.getRecipientPrimary().getDB('config').getCollectionNames(); -assert(!configDBCollections.includes('repl.migration.oplog_' + migrationOpts.migrationIdString), - configDBCollections); - -tenantMigrationTest.stop(); -})(); diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.cpp b/src/mongo/db/repl/tenant_migration_recipient_service.cpp index c6c4d4d2958..fc11edd65a4 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_service.cpp @@ -179,7 +179,7 @@ MONGO_FAIL_POINT_DEFINE(hangBeforeTaskCompletion); MONGO_FAIL_POINT_DEFINE(fpAfterReceivingRecipientForgetMigration); MONGO_FAIL_POINT_DEFINE(hangAfterCreatingRSM); MONGO_FAIL_POINT_DEFINE(skipRetriesWhenConnectingToDonorHost); -MONGO_FAIL_POINT_DEFINE(fpBeforeDroppingOplogBufferCollection); +MONGO_FAIL_POINT_DEFINE(fpBeforeDroppingTempCollections); MONGO_FAIL_POINT_DEFINE(fpWaitUntilTimestampMajorityCommitted); MONGO_FAIL_POINT_DEFINE(hangAfterUpdatingTransactionEntry); MONGO_FAIL_POINT_DEFINE(fpBeforeAdvancingStableTimestamp); @@ -2729,6 +2729,23 @@ TenantMigrationRecipientService::Instance::_migrateUsingShardMergeProtocol( .semi(); } +void TenantMigrationRecipientService::Instance::_dropTempCollections() { + _stopOrHangOnFailPoint(&fpBeforeDroppingTempCollections); + + auto opCtx = cc().makeOperationContext(); + auto storageInterface = StorageInterface::get(opCtx.get()); + + // The donated files and oplog buffer collections can be safely dropped at this + // point. In case either collection does not exist, dropping will be a no-op. + // It isn't necessary that a given drop is majority-committed. A new primary will + // attempt to drop the collection anyway. + uassertStatusOK(storageInterface->dropCollection( + opCtx.get(), shard_merge_utils::getDonatedFilesNs(getMigrationUUID()))); + + uassertStatusOK( + storageInterface->dropCollection(opCtx.get(), getOplogBufferNs(getMigrationUUID()))); +} + SemiFuture<void> TenantMigrationRecipientService::Instance::run( std::shared_ptr<executor::ScopedTaskExecutor> executor, const CancellationToken& token) noexcept { @@ -3060,17 +3077,7 @@ SemiFuture<void> TenantMigrationRecipientService::Instance::run( token); }) .then([this, self = shared_from_this()] { return _markStateDocAsGarbageCollectable(); }) - .then([this, self = shared_from_this()] { - _stopOrHangOnFailPoint(&fpBeforeDroppingOplogBufferCollection); - auto opCtx = cc().makeOperationContext(); - auto storageInterface = StorageInterface::get(opCtx.get()); - - // The oplog buffer collection can be safely dropped at this point. In case it - // doesn't exist, dropping will be a no-op. It isn't necessary that the drop is - // majority-committed. A new primary will attempt to drop the collection anyway. - return storageInterface->dropCollection(opCtx.get(), - getOplogBufferNs(getMigrationUUID())); - }) + .then([this, self = shared_from_this()] { _dropTempCollections(); }) .then([this, self = shared_from_this(), token] { { stdx::lock_guard lk(_mutex); diff --git a/src/mongo/db/repl/tenant_migration_recipient_service.h b/src/mongo/db/repl/tenant_migration_recipient_service.h index a71f29139d9..393c5c77448 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_service.h +++ b/src/mongo/db/repl/tenant_migration_recipient_service.h @@ -615,6 +615,11 @@ public: const CancellationToken& token); /* + * Drops ephemeral collections used for tenant migrations. + */ + void _dropTempCollections(); + + /* * Send the killBackupCursor command to the remote in order to close the backup cursor * connection on the donor. */ |