diff options
author | Daniel Gottlieb <daniel.gottlieb@mongodb.com> | 2021-05-04 16:09:52 -0400 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2021-05-04 21:10:48 +0000 |
commit | 137f0e128c872b601dfce4a3801a1dac8d2f0d28 (patch) | |
tree | c020124db3caf0752672b87233fe6ddd4ca8ffc8 /src | |
parent | e6981fb1f4eade2e07a971af7663f6cbc436be15 (diff) | |
download | mongo-137f0e128c872b601dfce4a3801a1dac8d2f0d28.tar.gz |
SERVER-56373: Introduce `needsRetryImage` field to oplog entries. Refactor OpObserver::onDelete to express image semantics.
Diffstat (limited to 'src')
33 files changed, 155 insertions, 120 deletions
diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript index f2ba6f8af4f..da618eb8a37 100644 --- a/src/mongo/db/SConscript +++ b/src/mongo/db/SConscript @@ -829,9 +829,9 @@ env.Library( '$BUILD_DIR/mongo/base', '$BUILD_DIR/mongo/db/catalog/collection_catalog', '$BUILD_DIR/mongo/db/repl/tenant_migration_access_blocker', - "$BUILD_DIR/mongo/db/timeseries/bucket_catalog", + '$BUILD_DIR/mongo/db/timeseries/bucket_catalog', '$BUILD_DIR/mongo/s/coreshard', - "$BUILD_DIR/mongo/s/grid", + '$BUILD_DIR/mongo/s/grid', 'catalog/collection_options', 'catalog/database_holder', 'op_observer', @@ -842,8 +842,9 @@ env.Library( 'views/views_mongod', ], LIBDEPS_PRIVATE=[ - "$BUILD_DIR/mongo/db/catalog/commit_quorum_options", + '$BUILD_DIR/mongo/db/catalog/commit_quorum_options', '$BUILD_DIR/mongo/db/catalog/import_collection_oplog_entry', + 'repl/repl_server_parameters', 'transaction', ], ) diff --git a/src/mongo/db/auth/auth_op_observer.cpp b/src/mongo/db/auth/auth_op_observer.cpp index 4d1dad3c26e..c05983b875c 100644 --- a/src/mongo/db/auth/auth_op_observer.cpp +++ b/src/mongo/db/auth/auth_op_observer.cpp @@ -88,8 +88,7 @@ void AuthOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { auto& documentId = documentIdDecoration(opCtx); invariant(!documentId.isEmpty()); AuthorizationManager::get(opCtx->getServiceContext()) diff --git a/src/mongo/db/auth/auth_op_observer.h b/src/mongo/db/auth/auth_op_observer.h index 8df53a44696..635babfc83f 100644 --- a/src/mongo/db/auth/auth_op_observer.h +++ b/src/mongo/db/auth/auth_op_observer.h @@ -92,8 +92,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/auth/auth_op_observer_test.cpp b/src/mongo/db/auth/auth_op_observer_test.cpp index 2dad678cc7b..527b465f37d 100644 --- a/src/mongo/db/auth/auth_op_observer_test.cpp +++ b/src/mongo/db/auth/auth_op_observer_test.cpp @@ -134,9 +134,9 @@ TEST_F(AuthOpObserverTest, MultipleAboutToDeleteAndOnDelete) { AutoGetDb autoDb(opCtx.get(), nss.db(), MODE_X); WriteUnitOfWork wunit(opCtx.get()); opObserver.aboutToDelete(opCtx.get(), nss, BSON("_id" << 1)); - opObserver.onDelete(opCtx.get(), nss, uuid, {}, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, uuid, {}, {}); opObserver.aboutToDelete(opCtx.get(), nss, BSON("_id" << 1)); - opObserver.onDelete(opCtx.get(), nss, uuid, {}, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, uuid, {}, {}); } DEATH_TEST_F(AuthOpObserverTest, AboutToDeleteMustPreceedOnDelete, "invariant") { @@ -144,7 +144,7 @@ DEATH_TEST_F(AuthOpObserverTest, AboutToDeleteMustPreceedOnDelete, "invariant") auto opCtx = cc().makeOperationContext(); cc().swapLockState(std::make_unique<LockerNoop>()); NamespaceString nss = {"test", "coll"}; - opObserver.onDelete(opCtx.get(), nss, {}, {}, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, {}, {}, {}); } DEATH_TEST_F(AuthOpObserverTest, EachOnDeleteRequiresAboutToDelete, "invariant") { @@ -153,8 +153,8 @@ DEATH_TEST_F(AuthOpObserverTest, EachOnDeleteRequiresAboutToDelete, "invariant") cc().swapLockState(std::make_unique<LockerNoop>()); NamespaceString nss = {"test", "coll"}; opObserver.aboutToDelete(opCtx.get(), nss, {}); - opObserver.onDelete(opCtx.get(), nss, {}, {}, false, boost::none); - opObserver.onDelete(opCtx.get(), nss, {}, {}, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, {}, {}, {}); + opObserver.onDelete(opCtx.get(), nss, {}, {}, {}); } } // namespace diff --git a/src/mongo/db/catalog/collection_impl.cpp b/src/mongo/db/catalog/collection_impl.cpp index 5ea64a3efcb..7232e02c390 100644 --- a/src/mongo/db/catalog/collection_impl.cpp +++ b/src/mongo/db/catalog/collection_impl.cpp @@ -969,13 +969,12 @@ void CollectionImpl::_cappedDeleteAsNeeded(OperationContext* opCtx, OpObserver* opObserver = opCtx->getServiceContext()->getOpObserver(); opObserver->aboutToDelete(opCtx, ns(), doc); + OpObserver::OplogDeleteEntryArgs args; + // Explicitly setting values despite them being the defaults. + args.deletedDoc = nullptr; + args.fromMigrate = false; // Reserves an optime for the deletion and sets the timestamp for future writes. - opObserver->onDelete(opCtx, - ns(), - uuid(), - kUninitializedStmtId, - /*fromMigrate=*/false, - /*deletedDoc=*/boost::none); + opObserver->onDelete(opCtx, ns(), uuid(), kUninitializedStmtId, args); } int64_t unusedKeysDeleted = 0; @@ -1090,8 +1089,11 @@ void CollectionImpl::deleteDocument(OperationContext* opCtx, _indexCatalog->unindexRecord(opCtx, doc.value(), loc, noWarn, &keysDeleted); _shared->_recordStore->deleteRecord(opCtx, loc); - getGlobalServiceContext()->getOpObserver()->onDelete( - opCtx, ns(), uuid(), stmtId, fromMigrate, deletedDoc); + OpObserver::OplogDeleteEntryArgs deleteArgs{nullptr, fromMigrate, getRecordPreImages()}; + if (deletedDoc) { + deleteArgs.deletedDoc = &(deletedDoc.get()); + } + getGlobalServiceContext()->getOpObserver()->onDelete(opCtx, ns(), uuid(), stmtId, deleteArgs); if (opDebug) { opDebug->additiveMetrics.incrementKeysDeleted(keysDeleted); diff --git a/src/mongo/db/fcv_op_observer.cpp b/src/mongo/db/fcv_op_observer.cpp index 73d8b4b23ee..66546e726e6 100644 --- a/src/mongo/db/fcv_op_observer.cpp +++ b/src/mongo/db/fcv_op_observer.cpp @@ -161,8 +161,7 @@ void FcvOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { // documentKeyDecoration is set in OpObserverImpl::aboutToDelete. So the FcvOpObserver // relies on the OpObserverImpl also being in the opObserverRegistry. auto optDocKey = documentKeyDecoration(opCtx); diff --git a/src/mongo/db/fcv_op_observer.h b/src/mongo/db/fcv_op_observer.h index ba131cf3b2e..9e26f5c483c 100644 --- a/src/mongo/db/fcv_op_observer.h +++ b/src/mongo/db/fcv_op_observer.h @@ -62,8 +62,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onReplicationRollback(OperationContext* opCtx, const RollbackObserverInfo& rbInfo) final; diff --git a/src/mongo/db/free_mon/free_mon_op_observer.cpp b/src/mongo/db/free_mon/free_mon_op_observer.cpp index 29e380c8baa..73861cc31d9 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.cpp +++ b/src/mongo/db/free_mon/free_mon_op_observer.cpp @@ -134,8 +134,7 @@ void FreeMonOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { if (nss != NamespaceString::kServerConfigurationNamespace) { return; } diff --git a/src/mongo/db/free_mon/free_mon_op_observer.h b/src/mongo/db/free_mon/free_mon_op_observer.h index cbc3ac48291..e6b7cc6a422 100644 --- a/src/mongo/db/free_mon/free_mon_op_observer.h +++ b/src/mongo/db/free_mon/free_mon_op_observer.h @@ -92,8 +92,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/op_observer.h b/src/mongo/db/op_observer.h index a6b6a574617..e488f85724a 100644 --- a/src/mongo/db/op_observer.h +++ b/src/mongo/db/op_observer.h @@ -131,24 +131,32 @@ public: virtual void aboutToDelete(OperationContext* opCtx, const NamespaceString& nss, const BSONObj& doc) = 0; + + /** + * "fromMigrate" indicates whether the delete was induced by a chunk migration, and so should be + * ignored by the user as an internal maintenance operation and not a real delete. + */ + struct OplogDeleteEntryArgs { + const BSONObj* deletedDoc = nullptr; + bool fromMigrate = false; + bool preImageRecordingEnabledForCollection = false; + }; + /** * Handles logging before document is deleted. * * "ns" name of the collection from which deleteState.idDoc will be deleted. - * "fromMigrate" indicates whether the delete was induced by a chunk migration, and - * so should be ignored by the user as an internal maintenance operation and not a - * real delete. * - * "deletedDoc" is a reference to an optional copy of the pre-image of the doc before deletion. - * If deletedDoc != boost::none, then the opObserver should assume that the caller intended - * the pre-image to be stored/logged in addition to the documentKey. + * "args" is a reference to information detailing whether the pre-image of the doc should be + * preserved with deletion. If `args.deletedDoc != nullptr`, then the opObserver must store the + * pre-image to be stored in addition to the documentKey. */ virtual void onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) = 0; + const OplogDeleteEntryArgs& args) = 0; + /** * Logs a no-op with "msgObj" in the o field into oplog. * diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp index 2fcb858e6b3..e2715df74aa 100644 --- a/src/mongo/db/op_observer_impl.cpp +++ b/src/mongo/db/op_observer_impl.cpp @@ -53,6 +53,7 @@ #include "mongo/db/read_write_concern_defaults.h" #include "mongo/db/repl/oplog.h" #include "mongo/db/repl/oplog_entry_gen.h" +#include "mongo/db/repl/repl_server_parameters_gen.h" #include "mongo/db/repl/replication_coordinator.h" #include "mongo/db/repl/tenant_migration_access_blocker_util.h" #include "mongo/db/repl/tenant_migration_decoration.h" @@ -221,24 +222,24 @@ OpTimeBundle replLogUpdate(OperationContext* opCtx, */ OpTimeBundle replLogDelete(OperationContext* opCtx, const NamespaceString& nss, + MutableOplogEntry* oplogEntry, OptionalCollectionUUID uuid, StmtId stmtId, bool fromMigrate, const boost::optional<BSONObj>& deletedDoc) { - MutableOplogEntry oplogEntry; - oplogEntry.setNss(nss); - oplogEntry.setUuid(uuid); - oplogEntry.setDestinedRecipient(destinedRecipientDecoration(opCtx)); + oplogEntry->setNss(nss); + oplogEntry->setUuid(uuid); + oplogEntry->setDestinedRecipient(destinedRecipientDecoration(opCtx)); repl::OplogLink oplogLink; - repl::appendOplogEntryChainInfo(opCtx, &oplogEntry, &oplogLink, {stmtId}); + repl::appendOplogEntryChainInfo(opCtx, oplogEntry, &oplogLink, {stmtId}); OpTimeBundle opTimes; // We never want to store pre-images when we're migrating oplog entries from another // replica set. const auto& migrationRecipientInfo = repl::tenantMigrationRecipientInfo(opCtx); if (deletedDoc && !migrationRecipientInfo) { - MutableOplogEntry noopEntry = oplogEntry; + MutableOplogEntry noopEntry = *oplogEntry; noopEntry.setOpType(repl::OpTypeEnum::kNoop); noopEntry.setObject(*deletedDoc); auto noteOplog = logOperation(opCtx, &noopEntry); @@ -246,13 +247,13 @@ OpTimeBundle replLogDelete(OperationContext* opCtx, oplogLink.preImageOpTime = noteOplog; } - oplogEntry.setOpType(repl::OpTypeEnum::kDelete); - oplogEntry.setObject(documentKeyDecoration(opCtx).get().getShardKeyAndId()); - oplogEntry.setFromMigrateIfTrue(fromMigrate); + oplogEntry->setOpType(repl::OpTypeEnum::kDelete); + oplogEntry->setObject(documentKeyDecoration(opCtx).get().getShardKeyAndId()); + oplogEntry->setFromMigrateIfTrue(fromMigrate); // oplogLink could have been changed to include preImageOpTime by the previous no-op write. - repl::appendOplogEntryChainInfo(opCtx, &oplogEntry, &oplogLink, {stmtId}); - opTimes.writeOpTime = logOperation(opCtx, &oplogEntry); - opTimes.wallClockTime = oplogEntry.getWallClockTime(); + repl::appendOplogEntryChainInfo(opCtx, oplogEntry, &oplogLink, {stmtId}); + opTimes.writeOpTime = logOperation(opCtx, oplogEntry); + opTimes.wallClockTime = oplogEntry->getWallClockTime(); return opTimes; } @@ -590,6 +591,17 @@ void OpObserverImpl::onUpdate(OperationContext* opCtx, const OplogUpdateEntryArg oplogEntry.getDurableReplOperation(), css, collDesc); + if (opCtx->getTxnNumber() && !repl::gStoreFindAndModifyImagesInOplog.load()) { + invariant( + repl::feature_flags::gFeatureFlagRetryableFindAndModify.isEnabledAndIgnoreFCV()); + if (args.updateArgs.storeDocOption == CollectionUpdateArgs::StoreDocOption::PreImage) { + oplogEntry.setNeedsRetryImage({repl::RetryImageEnum::kPreImage}); + } else if (args.updateArgs.storeDocOption == + CollectionUpdateArgs::StoreDocOption::PostImage) { + oplogEntry.setNeedsRetryImage({repl::RetryImageEnum::kPostImage}); + } + } + opTime = replLogUpdate(opCtx, args, std::move(oplogEntry)); SessionTxnRecord sessionTxnRecord; @@ -654,8 +666,7 @@ void OpObserverImpl::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { auto optDocKey = documentKeyDecoration(opCtx); invariant(optDocKey, nss.ns()); auto& documentKey = optDocKey.get(); @@ -668,15 +679,28 @@ void OpObserverImpl::onDelete(OperationContext* opCtx, if (inMultiDocumentTransaction) { auto operation = MutableOplogEntry::makeDeleteOperation(nss, uuid.get(), documentKey.getShardKeyAndId()); - if (deletedDoc) { - operation.setPreImage(deletedDoc->getOwned()); + if (args.deletedDoc) { + operation.setPreImage(args.deletedDoc->getOwned()); } operation.setDestinedRecipient(destinedRecipientDecoration(opCtx)); - txnParticipant.addTransactionOperation(opCtx, operation); } else { - opTime = replLogDelete(opCtx, nss, uuid, stmtId, fromMigrate, deletedDoc); + MutableOplogEntry oplogEntry; + if (args.deletedDoc) { + if (!repl::gStoreFindAndModifyImagesInOplog.load()) { + invariant(opCtx->getTxnNumber()); + invariant(repl::feature_flags::gFeatureFlagRetryableFindAndModify + .isEnabledAndIgnoreFCV()); + + oplogEntry.setNeedsRetryImage({repl::RetryImageEnum::kPreImage}); + } + } + + boost::optional<BSONObj> deletedDoc = + args.deletedDoc ? boost::optional<BSONObj>(*(args.deletedDoc)) : boost::none; + opTime = replLogDelete(opCtx, nss, &oplogEntry, uuid, stmtId, args.fromMigrate, deletedDoc); + SessionTxnRecord sessionTxnRecord; sessionTxnRecord.setLastWriteOpTime(opTime.writeOpTime); sessionTxnRecord.setLastWriteDate(opTime.wallClockTime); @@ -684,7 +708,7 @@ void OpObserverImpl::onDelete(OperationContext* opCtx, } if (nss != NamespaceString::kSessionTransactionsTableNamespace) { - if (!fromMigrate) { + if (!args.fromMigrate) { auto* const css = CollectionShardingState::get(opCtx, nss); shardObserveDeleteOp(opCtx, nss, diff --git a/src/mongo/db/op_observer_impl.h b/src/mongo/db/op_observer_impl.h index 7567bea8e60..750cad14d2d 100644 --- a/src/mongo/db/op_observer_impl.h +++ b/src/mongo/db/op_observer_impl.h @@ -106,8 +106,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, const boost::optional<UUID> uuid, diff --git a/src/mongo/db/op_observer_impl_test.cpp b/src/mongo/db/op_observer_impl_test.cpp index f12906e475c..52b6b08ff37 100644 --- a/src/mongo/db/op_observer_impl_test.cpp +++ b/src/mongo/db/op_observer_impl_test.cpp @@ -590,9 +590,9 @@ TEST_F(OpObserverTest, MultipleAboutToDeleteAndOnDelete) { AutoGetDb autoDb(opCtx.get(), nss.db(), MODE_X); WriteUnitOfWork wunit(opCtx.get()); opObserver.aboutToDelete(opCtx.get(), nss, BSON("_id" << 1)); - opObserver.onDelete(opCtx.get(), nss, uuid, kUninitializedStmtId, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, uuid, kUninitializedStmtId, {}); opObserver.aboutToDelete(opCtx.get(), nss, BSON("_id" << 1)); - opObserver.onDelete(opCtx.get(), nss, uuid, kUninitializedStmtId, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, uuid, kUninitializedStmtId, {}); } DEATH_TEST_F(OpObserverTest, AboutToDeleteMustPreceedOnDelete, "invariant") { @@ -600,7 +600,7 @@ DEATH_TEST_F(OpObserverTest, AboutToDeleteMustPreceedOnDelete, "invariant") { auto opCtx = cc().makeOperationContext(); cc().swapLockState(std::make_unique<LockerNoop>()); NamespaceString nss = {"test", "coll"}; - opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, {}); } DEATH_TEST_F(OpObserverTest, EachOnDeleteRequiresAboutToDelete, "invariant") { @@ -609,8 +609,8 @@ DEATH_TEST_F(OpObserverTest, EachOnDeleteRequiresAboutToDelete, "invariant") { cc().swapLockState(std::make_unique<LockerNoop>()); NamespaceString nss = {"test", "coll"}; opObserver.aboutToDelete(opCtx.get(), nss, {}); - opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, false, boost::none); - opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, false, boost::none); + opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, {}); + opObserver.onDelete(opCtx.get(), nss, {}, kUninitializedStmtId, {}); } DEATH_TEST_REGEX_F(OpObserverTest, @@ -790,7 +790,7 @@ TEST_F(OpObserverTransactionTest, TransactionalPrepareTest) { nss1, BSON("_id" << 0 << "data" << "x")); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, {}); { Lock::GlobalLock lk(opCtx(), MODE_IX); @@ -1297,12 +1297,12 @@ TEST_F(OpObserverTransactionTest, TransactionalDeleteTest) { nss1, BSON("_id" << 0 << "data" << "x")); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, {}); opObserver().aboutToDelete(opCtx(), nss2, BSON("_id" << 1 << "data" << "y")); - opObserver().onDelete(opCtx(), nss2, uuid2, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss2, uuid2, 0, {}); auto txnOps = txnParticipant.retrieveCompletedTransactionOperations(opCtx()); opObserver().onUnpreparedTransactionCommit(opCtx(), &txnOps, 0); auto oplogEntry = getSingleOplogEntry(opCtx()); @@ -1528,8 +1528,10 @@ TEST_F(OpObserverMultiEntryTransactionTest, TransactionPreImageTest) { const auto deletedDoc = BSON("_id" << 1 << "data" << "z"); + OpObserver::OplogDeleteEntryArgs args; + args.deletedDoc = &deletedDoc; opObserver().aboutToDelete(opCtx(), nss1, deletedDoc); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, deletedDoc); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, args); auto txnOps = txnParticipant.retrieveCompletedTransactionOperations(opCtx()); opObserver().onUnpreparedTransactionCommit(opCtx(), &txnOps, 2); @@ -1602,8 +1604,10 @@ TEST_F(OpObserverMultiEntryTransactionTest, PreparedTransactionPreImageTest) { const auto deletedDoc = BSON("_id" << 1 << "data" << "z"); + OpObserver::OplogDeleteEntryArgs args; + args.deletedDoc = &deletedDoc; opObserver().aboutToDelete(opCtx(), nss1, deletedDoc); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, deletedDoc); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, args); repl::OpTime prepareOpTime; { @@ -1670,12 +1674,12 @@ TEST_F(OpObserverMultiEntryTransactionTest, TransactionalDeleteTest) { nss1, BSON("_id" << 0 << "data" << "x")); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, {}); opObserver().aboutToDelete(opCtx(), nss2, BSON("_id" << 1 << "data" << "y")); - opObserver().onDelete(opCtx(), nss2, uuid2, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss2, uuid2, 0, {}); auto txnOps = txnParticipant.retrieveCompletedTransactionOperations(opCtx()); opObserver().onUnpreparedTransactionCommit(opCtx(), &txnOps, 0); auto oplogEntryObjs = getNOplogEntries(opCtx(), 2); @@ -1884,12 +1888,12 @@ TEST_F(OpObserverMultiEntryTransactionTest, TransactionalDeletePrepareTest) { nss1, BSON("_id" << 0 << "data" << "x")); - opObserver().onDelete(opCtx(), nss1, uuid1, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss1, uuid1, 0, {}); opObserver().aboutToDelete(opCtx(), nss2, BSON("_id" << 1 << "data" << "y")); - opObserver().onDelete(opCtx(), nss2, uuid2, 0, false, boost::none); + opObserver().onDelete(opCtx(), nss2, uuid2, 0, {}); repl::OpTime prepareOpTime; { diff --git a/src/mongo/db/op_observer_noop.h b/src/mongo/db/op_observer_noop.h index fe32d6706d1..e6fb3ae1db9 100644 --- a/src/mongo/db/op_observer_noop.h +++ b/src/mongo/db/op_observer_noop.h @@ -80,8 +80,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override {} + const OplogDeleteEntryArgs& args) override {} void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, const boost::optional<UUID> uuid, diff --git a/src/mongo/db/op_observer_registry.h b/src/mongo/db/op_observer_registry.h index 0384347236f..e83ca638c20 100644 --- a/src/mongo/db/op_observer_registry.h +++ b/src/mongo/db/op_observer_registry.h @@ -142,11 +142,10 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override { + const OplogDeleteEntryArgs& args) override { ReservedTimes times{opCtx}; for (auto& o : _observers) - o->onDelete(opCtx, nss, uuid, stmtId, fromMigrate, deletedDoc); + o->onDelete(opCtx, nss, uuid, stmtId, args); } void onInternalOpMessage(OperationContext* const opCtx, diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp index 6465720aeda..d5467d20a77 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp @@ -72,12 +72,11 @@ void OplogApplierImplOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OpObserver::OplogDeleteEntryArgs& args) { if (!onDeleteFn) { return; } - onDeleteFn(opCtx, nss, uuid, stmtId, fromMigrate, deletedDoc); + onDeleteFn(opCtx, nss, uuid, stmtId, args); } void OplogApplierImplOpObserver::onUpdate(OperationContext* opCtx, @@ -205,13 +204,12 @@ void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OpObserver::OplogDeleteEntryArgs& args) { applyOpCalled = true; checkOpCtx(opCtx); ASSERT_EQUALS(NamespaceString("test.t"), nss); - ASSERT(deletedDoc); - ASSERT_BSONOBJ_EQ(op.getObject(), *deletedDoc); + ASSERT(args.deletedDoc); + ASSERT_BSONOBJ_EQ(op.getObject(), *(args.deletedDoc)); return Status::OK(); }; diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.h b/src/mongo/db/repl/oplog_applier_impl_test_fixture.h index 9e43f10510e..5cbdbcc1866 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.h +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.h @@ -87,8 +87,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override; + const OplogDeleteEntryArgs& args) override; /** * This function is called whenever OplogApplierImpl updates a document in a collection. @@ -122,8 +121,7 @@ public: const NamespaceString&, OptionalCollectionUUID, StmtId, - bool, - const boost::optional<BSONObj>&)> + const OpObserver::OplogDeleteEntryArgs&)> onDeleteFn; std::function<void(OperationContext*, const OplogUpdateEntryArgs&)> onUpdateFn; diff --git a/src/mongo/db/repl/oplog_entry.cpp b/src/mongo/db/repl/oplog_entry.cpp index 9229f4afa5b..1a06226cb38 100644 --- a/src/mongo/db/repl/oplog_entry.cpp +++ b/src/mongo/db/repl/oplog_entry.cpp @@ -676,6 +676,10 @@ const boost::optional<mongo::repl::OpTime>& OplogEntry::getPostImageOpTime() con return _entry.getPostImageOpTime(); } +const boost::optional<RetryImageEnum> OplogEntry::getNeedsRetryImage() const { + return _entry.getNeedsRetryImage(); +} + OpTime OplogEntry::getOpTime() const { return _entry.getOpTime(); } diff --git a/src/mongo/db/repl/oplog_entry.h b/src/mongo/db/repl/oplog_entry.h index ff4785c2479..b7e24dc16a1 100644 --- a/src/mongo/db/repl/oplog_entry.h +++ b/src/mongo/db/repl/oplog_entry.h @@ -251,6 +251,7 @@ public: using MutableOplogEntry::getFromMigrate; using MutableOplogEntry::getFromTenantMigration; using MutableOplogEntry::getHash; + using MutableOplogEntry::getNeedsRetryImage; using MutableOplogEntry::getNss; using MutableOplogEntry::getObject; using MutableOplogEntry::getObject2; @@ -545,6 +546,7 @@ public: const boost::optional<mongo::UUID>& getFromTenantMigration() const&; const boost::optional<mongo::repl::OpTime>& getPrevWriteOpTimeInTransaction() const&; const boost::optional<mongo::repl::OpTime>& getPostImageOpTime() const&; + const boost::optional<RetryImageEnum> getNeedsRetryImage() const; OpTime getOpTime() const; bool isCommand() const; bool isPartialTransaction() const; diff --git a/src/mongo/db/repl/oplog_entry.idl b/src/mongo/db/repl/oplog_entry.idl index 7ed58a1d353..d23b045c39d 100644 --- a/src/mongo/db/repl/oplog_entry.idl +++ b/src/mongo/db/repl/oplog_entry.idl @@ -49,6 +49,14 @@ enums: kUpdate: "u" kDelete: "d" kNoop: "n" + RetryImage: + description: "Dictates whether a pre-image or post-image is to be stored on behalf of this + retryable write." + type: string + values: + kPreImage: "preImage" + kPostImage: "postImage" + structs: DurableReplOperation: @@ -152,3 +160,8 @@ structs: optional: true description: "The optime of another oplog entry that contains the document after an update was applied." + needsRetryImage: + type: RetryImage + optional: true + description: "Identifies whether a secondary should store a pre-image or post-image + associated with this oplog entry." diff --git a/src/mongo/db/repl/primary_only_service_op_observer.cpp b/src/mongo/db/repl/primary_only_service_op_observer.cpp index 4c939efb7d1..5c094e14a62 100644 --- a/src/mongo/db/repl/primary_only_service_op_observer.cpp +++ b/src/mongo/db/repl/primary_only_service_op_observer.cpp @@ -61,8 +61,7 @@ void PrimaryOnlyServiceOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { auto& documentId = documentIdDecoration(opCtx); invariant(!documentId.isEmpty()); diff --git a/src/mongo/db/repl/primary_only_service_op_observer.h b/src/mongo/db/repl/primary_only_service_op_observer.h index 7c745954de3..c183c3cfbed 100644 --- a/src/mongo/db/repl/primary_only_service_op_observer.h +++ b/src/mongo/db/repl/primary_only_service_op_observer.h @@ -94,8 +94,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/repl/tenant_migration_donor_op_observer.cpp b/src/mongo/db/repl/tenant_migration_donor_op_observer.cpp index f7c032492da..2d8a242507f 100644 --- a/src/mongo/db/repl/tenant_migration_donor_op_observer.cpp +++ b/src/mongo/db/repl/tenant_migration_donor_op_observer.cpp @@ -257,8 +257,7 @@ void TenantMigrationDonorOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { if (nss == NamespaceString::kTenantMigrationDonorsNamespace && tenantIdToDeleteDecoration(opCtx) && !tenant_migration_access_blocker::inRecoveryMode(opCtx)) { diff --git a/src/mongo/db/repl/tenant_migration_donor_op_observer.h b/src/mongo/db/repl/tenant_migration_donor_op_observer.h index d6874272e0a..3e7cfc169ea 100644 --- a/src/mongo/db/repl/tenant_migration_donor_op_observer.h +++ b/src/mongo/db/repl/tenant_migration_donor_op_observer.h @@ -92,8 +92,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp b/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp index e16004be680..0312203101d 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp +++ b/src/mongo/db/repl/tenant_migration_recipient_op_observer.cpp @@ -153,8 +153,7 @@ void TenantMigrationRecipientOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { if (nss == NamespaceString::kTenantMigrationRecipientsNamespace && tenantIdToDeleteDecoration(opCtx) && !tenant_migration_access_blocker::inRecoveryMode(opCtx)) { diff --git a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h index af2899286a5..4a6fae354ab 100644 --- a/src/mongo/db/repl/tenant_migration_recipient_op_observer.h +++ b/src/mongo/db/repl/tenant_migration_recipient_op_observer.h @@ -93,8 +93,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) final; + const OplogDeleteEntryArgs& args) final; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/repl/tenant_oplog_applier_test.cpp b/src/mongo/db/repl/tenant_oplog_applier_test.cpp index 7b587bc95f1..8b448c01974 100644 --- a/src/mongo/db/repl/tenant_oplog_applier_test.cpp +++ b/src/mongo/db/repl/tenant_oplog_applier_test.cpp @@ -526,8 +526,9 @@ TEST_F(TenantOplogApplierTest, ApplyDelete_DatabaseMissing) { const NamespaceString&, OptionalCollectionUUID, StmtId, - bool, - const boost::optional<BSONObj>&) { onDeleteCalled = true; }; + const OpObserver::OplogDeleteEntryArgs&) { + onDeleteCalled = true; + }; pushOps({entry}); auto writerPool = makeTenantMigrationWriterPool(); @@ -550,8 +551,9 @@ TEST_F(TenantOplogApplierTest, ApplyDelete_CollectionMissing) { const NamespaceString&, OptionalCollectionUUID, StmtId, - bool, - const boost::optional<BSONObj>&) { onDeleteCalled = true; }; + const OpObserver::OplogDeleteEntryArgs&) { + onDeleteCalled = true; + }; pushOps({entry}); auto writerPool = makeTenantMigrationWriterPool(); @@ -575,8 +577,9 @@ TEST_F(TenantOplogApplierTest, ApplyDelete_DocumentMissing) { const NamespaceString&, OptionalCollectionUUID, StmtId, - bool, - const boost::optional<BSONObj>&) { onDeleteCalled = true; }; + const OpObserver::OplogDeleteEntryArgs&) { + onDeleteCalled = true; + }; pushOps({entry}); auto writerPool = makeTenantMigrationWriterPool(); @@ -601,14 +604,13 @@ TEST_F(TenantOplogApplierTest, ApplyDelete_Success) { const NamespaceString& nss, OptionalCollectionUUID observer_uuid, StmtId, - bool fromMigrate, - const boost::optional<BSONObj>& o) { + const OpObserver::OplogDeleteEntryArgs& args) { onDeleteCalled = true; ASSERT_TRUE(opCtx); ASSERT_TRUE(opCtx->lockState()->isDbLockedForMode(nss.db(), MODE_IX)); ASSERT_TRUE(opCtx->lockState()->isCollectionLockedForMode(nss, MODE_IX)); ASSERT_TRUE(opCtx->writesAreReplicated()); - ASSERT_FALSE(fromMigrate); + ASSERT_FALSE(args.fromMigrate); ASSERT_EQUALS(nss.db(), dbName); ASSERT_EQUALS(nss.coll(), "bar"); ASSERT_EQUALS(uuid, observer_uuid); diff --git a/src/mongo/db/s/config_server_op_observer.cpp b/src/mongo/db/s/config_server_op_observer.cpp index eb4692e1a75..48ab254d435 100644 --- a/src/mongo/db/s/config_server_op_observer.cpp +++ b/src/mongo/db/s/config_server_op_observer.cpp @@ -61,8 +61,7 @@ void ConfigServerOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { if (nss == VersionType::ConfigNS) { if (!repl::ReplicationCoordinator::get(opCtx)->getMemberState().rollback()) { uasserted(40302, "cannot delete config.version document while in --configsvr mode"); diff --git a/src/mongo/db/s/config_server_op_observer.h b/src/mongo/db/s/config_server_op_observer.h index c8e81aa478d..c5b5e7c90a1 100644 --- a/src/mongo/db/s/config_server_op_observer.h +++ b/src/mongo/db/s/config_server_op_observer.h @@ -94,8 +94,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override; + const OplogDeleteEntryArgs& args) override; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/s/resharding/resharding_op_observer.cpp b/src/mongo/db/s/resharding/resharding_op_observer.cpp index 2b6e9da1ded..201ca033c52 100644 --- a/src/mongo/db/s/resharding/resharding_op_observer.cpp +++ b/src/mongo/db/s/resharding/resharding_op_observer.cpp @@ -237,8 +237,7 @@ void ReshardingOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { if (nss == NamespaceString::kDonorReshardingOperationsNamespace) { _doPin(opCtx); } diff --git a/src/mongo/db/s/resharding/resharding_op_observer.h b/src/mongo/db/s/resharding/resharding_op_observer.h index 90fd9a67bf8..41f42be9c35 100644 --- a/src/mongo/db/s/resharding/resharding_op_observer.h +++ b/src/mongo/db/s/resharding/resharding_op_observer.h @@ -109,8 +109,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override; + const OplogDeleteEntryArgs& args) override; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, diff --git a/src/mongo/db/s/shard_server_op_observer.cpp b/src/mongo/db/s/shard_server_op_observer.cpp index fd933d74bd8..99b25f4e82d 100644 --- a/src/mongo/db/s/shard_server_op_observer.cpp +++ b/src/mongo/db/s/shard_server_op_observer.cpp @@ -446,8 +446,7 @@ void ShardServerOpObserver::onDelete(OperationContext* opCtx, const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) { + const OplogDeleteEntryArgs& args) { auto& documentId = documentIdDecoration(opCtx); invariant(!documentId.isEmpty()); diff --git a/src/mongo/db/s/shard_server_op_observer.h b/src/mongo/db/s/shard_server_op_observer.h index 38b88e11d7a..27e68934a82 100644 --- a/src/mongo/db/s/shard_server_op_observer.h +++ b/src/mongo/db/s/shard_server_op_observer.h @@ -92,8 +92,7 @@ public: const NamespaceString& nss, OptionalCollectionUUID uuid, StmtId stmtId, - bool fromMigrate, - const boost::optional<BSONObj>& deletedDoc) override; + const OplogDeleteEntryArgs& args) override; void onInternalOpMessage(OperationContext* opCtx, const NamespaceString& nss, |