diff options
author | jannaerin <golden.janna@gmail.com> | 2021-04-26 20:23:44 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-05 18:13:49 +0000 |
commit | 946fb51b19e4943a069b8af1442ba5daeef61fa2 (patch) | |
tree | 2b428d27a1bd2beed1fc80ee8480424bfe2331ef | |
parent | 6701ac9e1cb8f42ae479d70f0fa6d1fa2b8bc995 (diff) | |
download | mongo-946fb51b19e4943a069b8af1442ba5daeef61fa2.tar.gz |
SERVER-54982 Test resumability of changes streams together with reshardCollection
32 files changed, 457 insertions, 30 deletions
diff --git a/jstests/sharding/libs/resharding_test_fixture.js b/jstests/sharding/libs/resharding_test_fixture.js index ba167f78fef..36bcfd52675 100644 --- a/jstests/sharding/libs/resharding_test_fixture.js +++ b/jstests/sharding/libs/resharding_test_fixture.js @@ -30,6 +30,8 @@ var ReshardingTest = class { minimumOperationDurationMS: minimumOperationDurationMS = undefined, criticalSectionTimeoutMS: criticalSectionTimeoutMS = 24 * 60 * 60 * 1000 /* 1 day */, commitImplicitly: commitImplicitly = true, + periodicNoopIntervalSecs: periodicNoopIntervalSecs = undefined, + writePeriodicNoops: writePeriodicNoops = undefined, } = {}) { // The @private JSDoc comments cause VS Code to not display the corresponding properties and // methods in its autocomplete list. This makes it simpler for test authors to know what the @@ -50,6 +52,10 @@ var ReshardingTest = class { this._criticalSectionTimeoutMS = criticalSectionTimeoutMS; /** @private */ this._commitImplicitly = commitImplicitly; + /** @private */ + this._periodicNoopIntervalSecs = periodicNoopIntervalSecs; + /** @private */ + this._writePeriodicNoops = writePeriodicNoops; // Properties set by setup(). /** @private */ @@ -97,6 +103,15 @@ var ReshardingTest = class { this._criticalSectionTimeoutMS; } + let rsConfig = {setParameter: {featureFlagResharding: true}}; + if (this._periodicNoopIntervalSecs !== undefined) { + rsConfig.setParameter.periodicNoopIntervalSecs = this._periodicNoopIntervalSecs; + } + + if (this._writePeriodicNoops !== undefined) { + rsConfig.setParameter.writePeriodicNoops = this._writePeriodicNoops; + } + this._st = new ShardingTest({ mongos: 1, mongosOptions: {setParameter: {featureFlagResharding: true}}, @@ -104,7 +119,7 @@ var ReshardingTest = class { configOptions: config, shards: this._numShards, rs: {nodes: 2}, - rsOptions: {setParameter: {featureFlagResharding: true}}, + rsOptions: rsConfig, manualAddShard: true, }); diff --git a/jstests/sharding/resharding_change_streams_resumability.js b/jstests/sharding/resharding_change_streams_resumability.js new file mode 100644 index 00000000000..577f4974109 --- /dev/null +++ b/jstests/sharding/resharding_change_streams_resumability.js @@ -0,0 +1,170 @@ +// Tests that change streams on a collection can be resumed during and after the given collection is +// resharded. +// +// @tags: [ +// requires_majority_read_concern, +// sbe_incompatible, +// uses_change_streams, +// requires_fcv_49, +// uses_atclustertime, +// ] +(function() { +"use strict"; + +load('jstests/libs/change_stream_util.js'); +load("jstests/libs/discover_topology.js"); +load("jstests/sharding/libs/resharding_test_fixture.js"); + +// Use a higher frequency for periodic noops to speed up the test. +const reshardingTest = new ReshardingTest({ + numDonors: 2, + numRecipients: 1, + reshardInPlace: false, + periodicNoopIntervalSecs: 1, + writePeriodicNoops: true +}); +reshardingTest.setup(); + +const kDbName = "reshardingDb"; +const collName = "coll"; +const ns = kDbName + "." + collName; + +const donorShardNames = reshardingTest.donorShardNames; +const sourceCollection = reshardingTest.createShardedCollection({ + ns: ns, + shardKeyPattern: {oldKey: 1}, + chunks: [ + {min: {oldKey: MinKey}, max: {oldKey: 0}, shard: donorShardNames[0]}, + {min: {oldKey: 0}, max: {oldKey: MaxKey}, shard: donorShardNames[1]} + ], + primaryShardName: donorShardNames[0] +}); + +const mongos = sourceCollection.getMongo(); +const reshardingDb = mongos.getDB(kDbName); + +const cst = new ChangeStreamTest(reshardingDb); + +// Open a change streams cursor on the collection that will be resharded. +let changeStreamsCursor = + cst.startWatchingChanges({pipeline: [{$changeStream: {}}], collection: collName}); +assert.eq([], changeStreamsCursor.firstBatch, "Expected cursor not to have changes, but it did"); + +const expectedChanges = [ + { + documentKey: {_id: 0, oldKey: 0}, + fullDocument: {_id: 0, oldKey: 0}, + ns: {db: kDbName, coll: collName}, + operationType: "insert", + }, + { + documentKey: {oldKey: 1, _id: 1}, + fullDocument: {_id: 1, oldKey: 1}, + ns: {db: kDbName, coll: collName}, + operationType: "insert", + }, + { + documentKey: {oldKey: 2, _id: 2}, + fullDocument: {_id: 2, oldKey: 2}, + ns: {db: kDbName, coll: collName}, + operationType: "insert", + }, + { + documentKey: {newKey: 3, _id: 3}, + fullDocument: {_id: 3, newKey: 3, oldKey: 3}, + ns: {db: kDbName, coll: collName}, + operationType: "insert", + }, + { + documentKey: {newKey: 4, _id: 4}, + fullDocument: {_id: 4, newKey: 4, oldKey: 4}, + ns: {db: kDbName, coll: collName}, + operationType: "insert", + } +]; +const preReshardCollectionChange = expectedChanges[0]; +const midReshardCollectionChanges = expectedChanges.slice(1, 3); +const postReshardCollectionChanges = expectedChanges.slice(3); + +// Verify that the cursor sees changes before the collection is resharded. +assert.commandWorked(sourceCollection.insert({_id: 0, oldKey: 0})); +const preReshardCollectionResumeToken = + cst.assertNextChangesEqual( + {cursor: changeStreamsCursor, expectedChanges: [preReshardCollectionChange]})[0] + ._id; + +const recipientShardNames = reshardingTest.recipientShardNames; +let midReshardCollectionResumeToken; +let changeStreamsCursor2; +reshardingTest.withReshardingInBackground( // + { + // If a donor is also a recipient, the donor state machine will run renameCollection with + // {dropTarget : true} rather than running drop and letting the recipient state machine run + // rename at the end of the resharding operation. So, we ensure that only one of the donor + // shards will also be a recipient shard order to verify that neither the rename with + // {dropTarget : true} nor the drop command are picked up by the change streams cursor. + newShardKeyPattern: {newKey: 1}, + newChunks: [ + {min: {newKey: MinKey}, max: {newKey: MaxKey}, shard: recipientShardNames[0]}, + ], + }, + () => { + // Wait until participants are aware of the resharding operation. + reshardingTest.awaitCloneTimestampChosen(); + + // Open another change streams cursor while the collection is being resharded. + changeStreamsCursor2 = + cst.startWatchingChanges({pipeline: [{$changeStream: {}}], collection: collName}); + + assert.commandWorked(sourceCollection.insert({_id: 1, oldKey: 1})); + assert.commandWorked(sourceCollection.insert({_id: 2, oldKey: 2})); + + // Assert that both the cursors see the two new inserts. + cst.assertNextChangesEqual( + {cursor: changeStreamsCursor, expectedChanges: midReshardCollectionChanges}); + cst.assertNextChangesEqual( + {cursor: changeStreamsCursor2, expectedChanges: midReshardCollectionChanges}); + + // Check that we can resume from the token returned before resharding began. + let resumedCursor = cst.startWatchingChanges({ + pipeline: [{$changeStream: {resumeAfter: preReshardCollectionResumeToken}}], + collection: collName + }); + midReshardCollectionResumeToken = + cst.assertNextChangesEqual( + {cursor: resumedCursor, expectedChanges: midReshardCollectionChanges})[1] + ._id; + }); + +assert.commandWorked(sourceCollection.insert({_id: 3, newKey: 3, oldKey: 3})); + +// Assert that both the cursor opened before resharding started and the one opened during +// resharding see the insert after resharding has finished. +cst.assertNextChangesEqual( + {cursor: changeStreamsCursor, expectedChanges: [postReshardCollectionChanges[0]]}); +cst.assertNextChangesEqual( + {cursor: changeStreamsCursor2, expectedChanges: [postReshardCollectionChanges[0]]}); + +// Check that we can resume from both the token returned before resharding began and the token +// returned during resharding. +let resumedCursorFromPreOperation = cst.startWatchingChanges({ + pipeline: [{$changeStream: {resumeAfter: preReshardCollectionResumeToken}}], + collection: collName +}); +let midAndPostReshardCollectionChanges = + midReshardCollectionChanges.concat(postReshardCollectionChanges); + +let resumedCursorFromMidOperation = cst.startWatchingChanges({ + pipeline: [{$changeStream: {resumeAfter: midReshardCollectionResumeToken}}], + collection: collName +}); + +assert.commandWorked(sourceCollection.insert({_id: 4, newKey: 4, oldKey: 4})); + +cst.assertNextChangesEqual( + {cursor: resumedCursorFromPreOperation, expectedChanges: midAndPostReshardCollectionChanges}); +cst.assertNextChangesEqual( + {cursor: resumedCursorFromMidOperation, expectedChanges: postReshardCollectionChanges}); + +reshardingTest.teardown(); +})(); diff --git a/src/mongo/db/auth/auth_op_observer.h b/src/mongo/db/auth/auth_op_observer.h index 635babfc83f..5948d210362 100644 --- a/src/mongo/db/auth/auth_op_observer.h +++ b/src/mongo/db/auth/auth_op_observer.h @@ -120,6 +120,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final; + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -132,6 +133,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) final; + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -149,6 +151,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) final; + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/catalog/database.h b/src/mongo/db/catalog/database.h index f7ba622493b..b81b7939919 100644 --- a/src/mongo/db/catalog/database.h +++ b/src/mongo/db/catalog/database.h @@ -116,7 +116,8 @@ public: repl::OpTime dropOpTime = {}) const = 0; virtual Status dropCollectionEvenIfSystem(OperationContext* const opCtx, NamespaceString nss, - repl::OpTime dropOpTime = {}) const = 0; + repl::OpTime dropOpTime = {}, + bool markFromMigrate = false) const = 0; virtual Status dropView(OperationContext* const opCtx, NamespaceString viewName) const = 0; diff --git a/src/mongo/db/catalog/database_impl.cpp b/src/mongo/db/catalog/database_impl.cpp index 3104171f859..5c1d10e527d 100644 --- a/src/mongo/db/catalog/database_impl.cpp +++ b/src/mongo/db/catalog/database_impl.cpp @@ -363,7 +363,8 @@ Status DatabaseImpl::dropCollection(OperationContext* opCtx, Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx, NamespaceString nss, - repl::OpTime dropOpTime) const { + repl::OpTime dropOpTime, + bool markFromMigrate) const { invariant(opCtx->lockState()->isCollectionLockedForMode(nss, MODE_X)); LOGV2_DEBUG(20313, 1, "dropCollection: {namespace}", "dropCollection", "namespace"_attr = nss); @@ -405,8 +406,12 @@ Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx, auto isOplogDisabledForNamespace = replCoord->isOplogDisabledFor(opCtx, nss); if (dropOpTime.isNull() && isOplogDisabledForNamespace) { _dropCollectionIndexes(opCtx, nss, collection.getWritableCollection()); - opObserver->onDropCollection( - opCtx, nss, uuid, numRecords, OpObserver::CollectionDropType::kOnePhase); + opObserver->onDropCollection(opCtx, + nss, + uuid, + numRecords, + OpObserver::CollectionDropType::kOnePhase, + markFromMigrate); return _finishDropCollection(opCtx, nss, collection.getWritableCollection()); } @@ -431,14 +436,22 @@ Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx, "commitTimestamp"_attr = commitTimestamp); if (dropOpTime.isNull()) { // Log oplog entry for collection drop and remove the UUID. - dropOpTime = opObserver->onDropCollection( - opCtx, nss, uuid, numRecords, OpObserver::CollectionDropType::kOnePhase); + dropOpTime = opObserver->onDropCollection(opCtx, + nss, + uuid, + numRecords, + OpObserver::CollectionDropType::kOnePhase, + markFromMigrate); invariant(!dropOpTime.isNull()); } else { // If we are provided with a valid 'dropOpTime', it means we are dropping this // collection in the context of applying an oplog entry on a secondary. - auto opTime = opObserver->onDropCollection( - opCtx, nss, uuid, numRecords, OpObserver::CollectionDropType::kOnePhase); + auto opTime = opObserver->onDropCollection(opCtx, + nss, + uuid, + numRecords, + OpObserver::CollectionDropType::kOnePhase, + markFromMigrate); // OpObserver::onDropCollection should not be writing to the oplog on the secondary. invariant(opTime.isNull(), str::stream() << "OpTime is not null. OpTime: " << opTime.toString()); @@ -452,14 +465,22 @@ Status DatabaseImpl::dropCollectionEvenIfSystem(OperationContext* opCtx, if (dropOpTime.isNull()) { // Log oplog entry for collection drop. - dropOpTime = opObserver->onDropCollection( - opCtx, nss, uuid, numRecords, OpObserver::CollectionDropType::kTwoPhase); + dropOpTime = opObserver->onDropCollection(opCtx, + nss, + uuid, + numRecords, + OpObserver::CollectionDropType::kTwoPhase, + markFromMigrate); invariant(!dropOpTime.isNull()); } else { // If we are provided with a valid 'dropOpTime', it means we are dropping this // collection in the context of applying an oplog entry on a secondary. - auto opTime = opObserver->onDropCollection( - opCtx, nss, uuid, numRecords, OpObserver::CollectionDropType::kTwoPhase); + auto opTime = opObserver->onDropCollection(opCtx, + nss, + uuid, + numRecords, + OpObserver::CollectionDropType::kTwoPhase, + markFromMigrate); // OpObserver::onDropCollection should not be writing to the oplog on the secondary. invariant(opTime.isNull()); } diff --git a/src/mongo/db/catalog/database_impl.h b/src/mongo/db/catalog/database_impl.h index 7abe7488dad..893d66588f6 100644 --- a/src/mongo/db/catalog/database_impl.h +++ b/src/mongo/db/catalog/database_impl.h @@ -66,7 +66,8 @@ public: repl::OpTime dropOpTime) const final; Status dropCollectionEvenIfSystem(OperationContext* opCtx, NamespaceString nss, - repl::OpTime dropOpTime) const final; + repl::OpTime dropOpTime, + bool markFromMigrate = false) const final; Status dropView(OperationContext* opCtx, NamespaceString viewName) const final; diff --git a/src/mongo/db/catalog/drop_database_test.cpp b/src/mongo/db/catalog/drop_database_test.cpp index a77d3a34e2b..a053f953cc6 100644 --- a/src/mongo/db/catalog/drop_database_test.cpp +++ b/src/mongo/db/catalog/drop_database_test.cpp @@ -67,6 +67,8 @@ using namespace mongo; class OpObserverMock : public OpObserverNoop { public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override; + + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, diff --git a/src/mongo/db/catalog/rename_collection.cpp b/src/mongo/db/catalog/rename_collection.cpp index dfbccc63f34..b419e735ca3 100644 --- a/src/mongo/db/catalog/rename_collection.cpp +++ b/src/mongo/db/catalog/rename_collection.cpp @@ -198,7 +198,8 @@ Status renameCollectionDirectly(OperationContext* opCtx, // avoid unintentionally removing a collection on a secondary with the same name as // the target. auto opObserver = opCtx->getServiceContext()->getOpObserver(); - opObserver->onRenameCollection(opCtx, source, target, uuid, {}, 0U, options.stayTemp); + opObserver->onRenameCollection( + opCtx, source, target, uuid, {}, 0U, options.stayTemp, options.markFromMigrate); wunit.commit(); return Status::OK(); @@ -231,8 +232,15 @@ Status renameCollectionAndDropTarget(OperationContext* opCtx, auto numRecords = targetColl->numRecords(opCtx); auto opObserver = opCtx->getServiceContext()->getOpObserver(); - auto renameOpTime = opObserver->preRenameCollection( - opCtx, source, target, uuid, targetColl->uuid(), numRecords, options.stayTemp); + + auto renameOpTime = opObserver->preRenameCollection(opCtx, + source, + target, + uuid, + targetColl->uuid(), + numRecords, + options.stayTemp, + options.markFromMigrate); if (!renameOpTimeFromApplyOps.isNull()) { // 'renameOpTime' must be null because a valid 'renameOpTimeFromApplyOps' implies diff --git a/src/mongo/db/catalog/rename_collection.h b/src/mongo/db/catalog/rename_collection.h index 7b406ef3392..b6e8b974dfa 100644 --- a/src/mongo/db/catalog/rename_collection.h +++ b/src/mongo/db/catalog/rename_collection.h @@ -50,6 +50,7 @@ class OpTime; struct RenameCollectionOptions { bool dropTarget = false; bool stayTemp = false; + bool markFromMigrate = false; }; void doLocalRenameIfOptionsAndIndexesHaveNotChanged(OperationContext* opCtx, diff --git a/src/mongo/db/catalog/rename_collection_test.cpp b/src/mongo/db/catalog/rename_collection_test.cpp index 7acd0df9b1e..35eacf51294 100644 --- a/src/mongo/db/catalog/rename_collection_test.cpp +++ b/src/mongo/db/catalog/rename_collection_test.cpp @@ -116,12 +116,14 @@ public: const BSONObj& idIndex, const OplogSlot& createOpTime) override; + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, std::uint64_t numRecords, CollectionDropType dropType) override; + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -130,6 +132,7 @@ public: std::uint64_t numRecords, bool stayTemp) override; + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/commands/mr_test.cpp b/src/mongo/db/commands/mr_test.cpp index da4dc97f507..eaada81ab68 100644 --- a/src/mongo/db/commands/mr_test.cpp +++ b/src/mongo/db/commands/mr_test.cpp @@ -273,6 +273,7 @@ public: const BSONObj& idIndex, const OplogSlot& createOpTime) override; + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, diff --git a/src/mongo/db/fcv_op_observer.h b/src/mongo/db/fcv_op_observer.h index 9e26f5c483c..0719263aa2d 100644 --- a/src/mongo/db/fcv_op_observer.h +++ b/src/mongo/db/fcv_op_observer.h @@ -123,6 +123,7 @@ public: const CollectionOptions& oldCollOptions, boost::optional<IndexCollModInfo> indexInfo) final {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -135,6 +136,7 @@ public: OptionalCollectionUUID uuid, const std::string& indexName, const BSONObj& idxDescriptor) final {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -150,6 +152,7 @@ public: const BSONObj& catalogEntry, const BSONObj& storageMetadata, bool isDryRun) final {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h index e6b7cc6a422..b2e73af1f41 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.h +++ b/src/mongo/db/free_mon/free_mon_op_observer.h @@ -120,6 +120,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -132,6 +133,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) final {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -149,6 +151,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) final {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h index e488f85724a..a226d631eff 100644 --- a/src/mongo/db/op_observer.h +++ b/src/mongo/db/op_observer.h @@ -246,6 +246,15 @@ public: OptionalCollectionUUID uuid, std::uint64_t numRecords, CollectionDropType dropType) = 0; + virtual repl::OpTime onDropCollection(OperationContext* opCtx, + const NamespaceString& collectionName, + OptionalCollectionUUID uuid, + std::uint64_t numRecords, + CollectionDropType dropType, + bool markFromMigrate) { + return onDropCollection(opCtx, collectionName, uuid, numRecords, dropType); + } + /** * This function logs an oplog entry when an index is dropped. The namespace of the index, @@ -277,6 +286,18 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) = 0; + + virtual repl::OpTime preRenameCollection(OperationContext* opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) { + return preRenameCollection( + opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); + } /** * This function performs all op observer handling for a 'renameCollection' command except for * logging the oplog entry. It should be used specifically in instances where the optime is @@ -301,6 +322,17 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) = 0; + virtual void onRenameCollection(OperationContext* opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) { + onRenameCollection( + opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); + } virtual void onImportCollection(OperationContext* opCtx, const UUID& importUUID, diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index e2715df74aa..fd7bd32e97c 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -859,13 +859,24 @@ repl::OpTime OpObserverImpl::onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, std::uint64_t numRecords, - const CollectionDropType dropType) { + CollectionDropType dropType) { + return onDropCollection( + opCtx, collectionName, uuid, numRecords, dropType, false /* markFromMigrate */); +} + +repl::OpTime OpObserverImpl::onDropCollection(OperationContext* opCtx, + const NamespaceString& collectionName, + OptionalCollectionUUID uuid, + std::uint64_t numRecords, + const CollectionDropType dropType, + bool markFromMigrate) { if (!collectionName.isSystemDotProfile()) { // Do not replicate system.profile modifications. MutableOplogEntry oplogEntry; oplogEntry.setOpType(repl::OpTypeEnum::kCommand); oplogEntry.setNss(collectionName.getCommandNS()); oplogEntry.setUuid(uuid); + oplogEntry.setFromMigrateIfTrue(markFromMigrate); oplogEntry.setObject(BSON("drop" << collectionName.coll())); oplogEntry.setObject2(makeObject2ForDropOrRename(numRecords)); logOperation(opCtx, &oplogEntry); @@ -918,7 +929,6 @@ void OpObserverImpl::onDropIndex(OperationContext* opCtx, logOperation(opCtx, &oplogEntry); } - repl::OpTime OpObserverImpl::preRenameCollection(OperationContext* const opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -926,6 +936,24 @@ repl::OpTime OpObserverImpl::preRenameCollection(OperationContext* const opCtx, OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) { + return preRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + false /* markFromMigrate */); +} + +repl::OpTime OpObserverImpl::preRenameCollection(OperationContext* const opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) { BSONObjBuilder builder; builder.append("renameCollection", fromCollection.ns()); builder.append("to", toCollection.ns()); @@ -938,6 +966,7 @@ repl::OpTime OpObserverImpl::preRenameCollection(OperationContext* const opCtx, oplogEntry.setOpType(repl::OpTypeEnum::kCommand); oplogEntry.setNss(fromCollection.getCommandNS()); oplogEntry.setUuid(uuid); + oplogEntry.setFromMigrateIfTrue(markFromMigrate); oplogEntry.setObject(builder.done()); if (dropTargetUUID) oplogEntry.setObject2(makeObject2ForDropOrRename(numRecords)); @@ -965,8 +994,32 @@ void OpObserverImpl::onRenameCollection(OperationContext* const opCtx, OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) { - preRenameCollection( - opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); + onRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + false /* markFromMigrate */); +} + +void OpObserverImpl::onRenameCollection(OperationContext* const opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) { + preRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + markFromMigrate); postRenameCollection(opCtx, fromCollection, toCollection, uuid, dropTargetUUID, stayTemp); } diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h index 750cad14d2d..0c04baef118 100644 --- a/src/mongo/db/op_observer_impl.h +++ b/src/mongo/db/op_observer_impl.h @@ -134,6 +134,12 @@ public: OptionalCollectionUUID uuid, std::uint64_t numRecords, CollectionDropType dropType) final; + repl::OpTime onDropCollection(OperationContext* opCtx, + const NamespaceString& collectionName, + OptionalCollectionUUID uuid, + std::uint64_t numRecords, + CollectionDropType dropType, + bool markFromMigrate) final; void onDropIndex(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, @@ -146,6 +152,14 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) final; + repl::OpTime preRenameCollection(OperationContext* opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) final; void postRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -159,6 +173,14 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) final; + void onRenameCollection(OperationContext* opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) final; void onImportCollection(OperationContext* opCtx, const UUID& importUUID, const NamespaceString& nss, diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index e6fb3ae1db9..70f4589eafe 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -103,6 +103,7 @@ public: const CollectionOptions& oldCollOptions, boost::optional<IndexCollModInfo> indexInfo) override {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -115,6 +116,7 @@ public: OptionalCollectionUUID uuid, const std::string& indexName, const BSONObj& idxDescriptor) override {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -130,6 +132,7 @@ public: const BSONObj& catalogEntry, const BSONObj& storageMetadata, bool isDryRun) override {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h index e83ca638c20..2b6b0db7e81 100644 --- a/src/mongo/db/op_observer_registry.h +++ b/src/mongo/db/op_observer_registry.h @@ -203,10 +203,20 @@ public: const OptionalCollectionUUID uuid, std::uint64_t numRecords, const CollectionDropType dropType) override { + return onDropCollection( + opCtx, collectionName, uuid, numRecords, dropType, false /* markFromMigrate*/); + } + + repl::OpTime onDropCollection(OperationContext* const opCtx, + const NamespaceString& collectionName, + const OptionalCollectionUUID uuid, + std::uint64_t numRecords, + const CollectionDropType dropType, + bool markFromMigrate) override { ReservedTimes times{opCtx}; for (auto& observer : this->_observers) { - auto time = - observer->onDropCollection(opCtx, collectionName, uuid, numRecords, dropType); + auto time = observer->onDropCollection( + opCtx, collectionName, uuid, numRecords, dropType, markFromMigrate); invariant(time.isNull()); } return _getOpTimeToReturn(times.get().reservedOpTimes); @@ -222,7 +232,6 @@ public: o->onDropIndex(opCtx, nss, uuid, indexName, idxDescriptor); } - void onRenameCollection(OperationContext* const opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -230,10 +239,34 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) override { + onRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + false /* markFromMigrate */); + } + + void onRenameCollection(OperationContext* const opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) override { ReservedTimes times{opCtx}; for (auto& o : _observers) - o->onRenameCollection( - opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); + o->onRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + markFromMigrate); } void onImportCollection(OperationContext* opCtx, @@ -263,10 +296,34 @@ public: OptionalCollectionUUID dropTargetUUID, std::uint64_t numRecords, bool stayTemp) override { + return preRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + false /* markFromMigrate */); + } + + repl::OpTime preRenameCollection(OperationContext* const opCtx, + const NamespaceString& fromCollection, + const NamespaceString& toCollection, + OptionalCollectionUUID uuid, + OptionalCollectionUUID dropTargetUUID, + std::uint64_t numRecords, + bool stayTemp, + bool markFromMigrate) override { ReservedTimes times{opCtx}; for (auto& observer : this->_observers) { - const auto time = observer->preRenameCollection( - opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); + const auto time = observer->preRenameCollection(opCtx, + fromCollection, + toCollection, + uuid, + dropTargetUUID, + numRecords, + stayTemp, + markFromMigrate); invariant(time.isNull()); } return _getOpTimeToReturn(times.get().reservedOpTimes); diff --git a/src/mongo/db/op_observer_registry_test.cpp b/src/mongo/db/op_observer_registry_test.cpp index 14be2cbfa8a..428a3d71e0d 100644 --- a/src/mongo/db/op_observer_registry_test.cpp +++ b/src/mongo/db/op_observer_registry_test.cpp @@ -53,6 +53,7 @@ struct TestObserver : public OpObserverNoop { void onDropDatabase(OperationContext* opCtx, const std::string& dbName) { drops++; } + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -62,6 +63,7 @@ struct TestObserver : public OpObserverNoop { OpObserver::Times::get(opCtx).reservedOpTimes.push_back(opTime); return {}; } + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -73,6 +75,8 @@ struct TestObserver : public OpObserverNoop { opCtx, fromCollection, toCollection, uuid, dropTargetUUID, numRecords, stayTemp); postRenameCollection(opCtx, fromCollection, toCollection, uuid, dropTargetUUID, stayTemp); } + + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/repl/primary_only_service_op_observer.h b/src/mongo/db/repl/primary_only_service_op_observer.h index c183c3cfbed..2dd563137a4 100644 --- a/src/mongo/db/repl/primary_only_service_op_observer.h +++ b/src/mongo/db/repl/primary_only_service_op_observer.h @@ -122,6 +122,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -134,6 +135,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) final {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -151,6 +153,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) final {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/repl/replication_recovery_test.cpp b/src/mongo/db/repl/replication_recovery_test.cpp index 7c75e9378fa..6350054b441 100644 --- a/src/mongo/db/repl/replication_recovery_test.cpp +++ b/src/mongo/db/repl/replication_recovery_test.cpp @@ -119,6 +119,7 @@ private: class ReplicationRecoveryTestObObserver : public OpObserverNoop { public: + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, diff --git a/src/mongo/db/repl/rollback_test_fixture.cpp b/src/mongo/db/repl/rollback_test_fixture.cpp index be74b09840f..4ae77c557ef 100644 --- a/src/mongo/db/repl/rollback_test_fixture.cpp +++ b/src/mongo/db/repl/rollback_test_fixture.cpp @@ -68,6 +68,7 @@ ReplSettings createReplSettings() { class RollbackTestOpObserver : public OpObserverNoop { public: + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, diff --git a/src/mongo/db/repl/tenant_migration_donor_op_observer.h b/src/mongo/db/repl/tenant_migration_donor_op_observer.h index 4cdd5d30c3f..3424ec154d6 100644 --- a/src/mongo/db/repl/tenant_migration_donor_op_observer.h +++ b/src/mongo/db/repl/tenant_migration_donor_op_observer.h @@ -119,7 +119,7 @@ public: boost::optional<IndexCollModInfo> indexInfo) final {} void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} - + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -132,6 +132,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) final {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -149,6 +150,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) final {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h index ccf3e6db38f..a4d3092037b 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h +++ b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h @@ -121,6 +121,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) final {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -133,6 +134,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) final {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -150,6 +152,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) final {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index c5b5e7c90a1..f7d2f65029c 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -122,6 +122,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -134,6 +135,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) override {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -149,6 +151,7 @@ public: const BSONObj& catalogEntry, const BSONObj& storageMetadata, bool isDryRun) override {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp index d3786c4f688..902be1c6eed 100644 --- a/src/mongo/db/s/resharding/resharding_data_copy_util.cpp +++ b/src/mongo/db/s/resharding/resharding_data_copy_util.cpp @@ -86,7 +86,8 @@ void ensureCollectionDropped(OperationContext* opCtx, } WriteUnitOfWork wuow(opCtx); - uassertStatusOK(coll.getDb()->dropCollectionEvenIfSystem(opCtx, nss)); + uassertStatusOK(coll.getDb()->dropCollectionEvenIfSystem( + opCtx, nss, {} /* dropOpTime */, true /* markFromMigrate */)); wuow.commit(); }); } diff --git a/src/mongo/db/s/resharding/resharding_donor_service.cpp b/src/mongo/db/s/resharding/resharding_donor_service.cpp index 1a00a5fc3a3..73df389ab68 100644 --- a/src/mongo/db/s/resharding/resharding_donor_service.cpp +++ b/src/mongo/db/s/resharding/resharding_donor_service.cpp @@ -622,6 +622,7 @@ void ReshardingDonorService::DonorStateMachine::_dropOriginalCollectionThenTrans RenameCollectionOptions options; options.dropTarget = true; + options.markFromMigrate = true; uassertStatusOK(renameCollection( opCtx.get(), _metadata.getTempReshardingNss(), _metadata.getSourceNss(), options)); } else { diff --git a/src/mongo/db/s/resharding/resharding_op_observer.h b/src/mongo/db/s/resharding/resharding_op_observer.h index 41f42be9c35..5ddcd31d8d4 100644 --- a/src/mongo/db/s/resharding/resharding_op_observer.h +++ b/src/mongo/db/s/resharding/resharding_op_observer.h @@ -137,6 +137,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -151,6 +152,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) override {} + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -168,6 +170,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) override {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/s/resharding/resharding_recipient_service.cpp b/src/mongo/db/s/resharding/resharding_recipient_service.cpp index 3333b365108..89ec2b91550 100644 --- a/src/mongo/db/s/resharding/resharding_recipient_service.cpp +++ b/src/mongo/db/s/resharding/resharding_recipient_service.cpp @@ -560,6 +560,7 @@ void ReshardingRecipientService::RecipientStateMachine::_renameTemporaryReshardi RenameCollectionOptions options; options.dropTarget = true; + options.markFromMigrate = true; uassertStatusOK(renameCollection( opCtx.get(), _metadata.getTempReshardingNss(), _metadata.getSourceNss(), options)); diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 27e68934a82..29b213aebcb 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -120,6 +120,7 @@ public: void onDropDatabase(OperationContext* opCtx, const std::string& dbName) override {} + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, @@ -132,6 +133,7 @@ public: const std::string& indexName, const BSONObj& indexInfo) override; + using OpObserver::onRenameCollection; void onRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, @@ -149,6 +151,7 @@ public: const BSONObj& storageMetadata, bool isDryRun) override {} + using OpObserver::preRenameCollection; repl::OpTime preRenameCollection(OperationContext* opCtx, const NamespaceString& fromCollection, const NamespaceString& toCollection, diff --git a/src/mongo/db/transaction_participant_retryable_writes_test.cpp b/src/mongo/db/transaction_participant_retryable_writes_test.cpp index 619a9118bf0..037449f1cd1 100644 --- a/src/mongo/db/transaction_participant_retryable_writes_test.cpp +++ b/src/mongo/db/transaction_participant_retryable_writes_test.cpp @@ -153,6 +153,7 @@ public: preparedTransactionCommitted = true; }; + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, diff --git a/src/mongo/db/transaction_participant_test.cpp b/src/mongo/db/transaction_participant_test.cpp index 64b93e4eda2..7e3a3cb28a9 100644 --- a/src/mongo/db/transaction_participant_test.cpp +++ b/src/mongo/db/transaction_participant_test.cpp @@ -134,6 +134,7 @@ public: bool onTransactionAbortThrowsException = false; bool transactionAborted = false; + using OpObserver::onDropCollection; repl::OpTime onDropCollection(OperationContext* opCtx, const NamespaceString& collectionName, OptionalCollectionUUID uuid, |