diff options
| author | Daniel Gottlieb <daniel.gottlieb@mongodb.com> | 2023-01-09 22:14:13 -0500 |
|---|---|---|
| committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2023-01-10 04:24:36 +0000 |
| commit | 23acbaa87ef1cbf9e2136c0ad63c32efd0c39bd4 (patch) | |
| tree | 3dc35457544dcc3f454194a76c222c529a9c0b77 /src/mongo/db/repl | |
| parent | bc7e2f3f5a4d7825b67f81443c06f1da49e92c55 (diff) | |
| download | mongo-23acbaa87ef1cbf9e2136c0ad63c32efd0c39bd4.tar.gz | |
SERVER-72500: Change oplog application to work on batches of ApplierOperations instead of OplogEntrys.
ApplierOperations in the general case decays to an OplogEntry. But they can also encapsulate states
pertaining to prepared transactions.
Diffstat (limited to 'src/mongo/db/repl')
| -rw-r--r-- | src/mongo/db/repl/apply_ops.cpp | 2 | ||||
| -rw-r--r-- | src/mongo/db/repl/insert_group.cpp | 6 | ||||
| -rw-r--r-- | src/mongo/db/repl/insert_group.h | 4 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog.cpp | 4 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_impl.cpp | 12 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_impl.h | 8 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_impl_test.cpp | 74 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp | 6 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_utils.cpp | 14 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_applier_utils.h | 8 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_entry_or_grouped_inserts.cpp | 6 | ||||
| -rw-r--r-- | src/mongo/db/repl/oplog_entry_or_grouped_inserts.h | 28 | ||||
| -rw-r--r-- | src/mongo/db/repl/storage_timestamp_test.cpp | 21 | ||||
| -rw-r--r-- | src/mongo/db/repl/tenant_oplog_applier.cpp | 6 | ||||
| -rw-r--r-- | src/mongo/db/repl/tenant_oplog_applier.h | 8 | ||||
| -rw-r--r-- | src/mongo/db/repl/transaction_oplog_application.cpp | 2 |
16 files changed, 119 insertions, 90 deletions
diff --git a/src/mongo/db/repl/apply_ops.cpp b/src/mongo/db/repl/apply_ops.cpp index 6578d8ba18d..8618624e610 100644 --- a/src/mongo/db/repl/apply_ops.cpp +++ b/src/mongo/db/repl/apply_ops.cpp @@ -169,7 +169,7 @@ Status _applyOps(OperationContext* opCtx, const bool isDataConsistent = true; return repl::applyOperation_inlock(opCtx, ctx.db(), - &entry, + ApplierOperation{&entry}, alwaysUpsert, oplogApplicationMode, isDataConsistent); diff --git a/src/mongo/db/repl/insert_group.cpp b/src/mongo/db/repl/insert_group.cpp index eea1cd1f03a..08e84821985 100644 --- a/src/mongo/db/repl/insert_group.cpp +++ b/src/mongo/db/repl/insert_group.cpp @@ -58,7 +58,7 @@ constexpr auto kInsertGroupMaxOpCount = 64; } // namespace -InsertGroup::InsertGroup(std::vector<const OplogEntry*>* ops, +InsertGroup::InsertGroup(std::vector<ApplierOperation>* ops, OperationContext* opCtx, InsertGroup::Mode mode, const bool isDataConsistent, @@ -96,7 +96,7 @@ StatusWith<InsertGroup::ConstIterator> InsertGroup::groupAndApplyInserts( // Make sure to include the first op in the group size. size_t groupSize = entry.getObject().objsize(); - auto opCount = std::vector<const OplogEntry*>::size_type(1); + auto opCount = std::vector<ApplierOperation>::size_type(1); auto groupNamespace = entry.getNss(); /** @@ -114,7 +114,7 @@ StatusWith<InsertGroup::ConstIterator> InsertGroup::groupAndApplyInserts( * will point to the first op that *can't* be added to the current insert group. */ auto endOfGroupableOpsIterator = - std::find_if(it + 1, _end, [&](const OplogEntry* nextEntry) -> bool { + std::find_if(it + 1, _end, [&](ApplierOperation nextEntry) -> bool { auto opNamespace = nextEntry->getNss(); groupSize += nextEntry->getObject().objsize(); opCount += 1; diff --git a/src/mongo/db/repl/insert_group.h b/src/mongo/db/repl/insert_group.h index 4586ee997ab..061740ba5cd 100644 --- a/src/mongo/db/repl/insert_group.h +++ b/src/mongo/db/repl/insert_group.h @@ -48,13 +48,13 @@ class InsertGroup { InsertGroup& operator=(const InsertGroup&) = delete; public: - using ConstIterator = std::vector<const OplogEntry*>::const_iterator; + using ConstIterator = std::vector<ApplierOperation>::const_iterator; using Mode = OplogApplication::Mode; typedef std::function<Status( OperationContext*, const OplogEntryOrGroupedInserts&, OplogApplication::Mode, bool)> ApplyFunc; - InsertGroup(std::vector<const OplogEntry*>* ops, + InsertGroup(std::vector<ApplierOperation>* ops, OperationContext* opCtx, Mode mode, bool isDataConsistent, diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp index 92c9c259fb0..2428f57a566 100644 --- a/src/mongo/db/repl/oplog.cpp +++ b/src/mongo/db/repl/oplog.cpp @@ -1377,7 +1377,7 @@ Status applyOperation_inlock(OperationContext* opCtx, const auto insertOps = opOrGroupedInserts.getGroupedInserts(); WriteUnitOfWork wuow(opCtx); if (!opCtx->writesAreReplicated()) { - for (const auto iOp : insertOps) { + for (const auto& iOp : insertOps) { invariant(iOp->getTerm()); insertObjs.emplace_back( iOp->getObject(), iOp->getTimestamp(), iOp->getTerm().value()); @@ -1388,7 +1388,7 @@ Status applyOperation_inlock(OperationContext* opCtx, // the recipient. std::vector<OplogSlot> slots = getNextOpTimes(opCtx, insertOps.size()); auto slotIter = slots.begin(); - for (const auto iOp : insertOps) { + for (const auto& iOp : insertOps) { insertObjs.emplace_back( iOp->getObject(), slotIter->getTimestamp(), slotIter->getTerm()); slotIter++; diff --git a/src/mongo/db/repl/oplog_applier_impl.cpp b/src/mongo/db/repl/oplog_applier_impl.cpp index 6d4818d5ac7..7f41a03865d 100644 --- a/src/mongo/db/repl/oplog_applier_impl.cpp +++ b/src/mongo/db/repl/oplog_applier_impl.cpp @@ -116,7 +116,7 @@ void _addOplogChainOpsToWriterVectors(OperationContext* opCtx, std::vector<std::vector<OplogEntry>>* derivedOps, OplogEntry* op, CachedCollectionProperties* collPropertiesCache, - std::vector<std::vector<const OplogEntry*>>* writerVectors) { + std::vector<std::vector<ApplierOperation>>* writerVectors) { std::vector<OplogEntry> txnOps; bool shouldSerialize = false; std::tie(txnOps, shouldSerialize) = @@ -544,7 +544,7 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, // and create a pseudo oplog. std::vector<std::vector<OplogEntry>> derivedOps; - std::vector<std::vector<const OplogEntry*>> writerVectors( + std::vector<std::vector<ApplierOperation>> writerVectors( _writerPool->getStats().options.maxThreads); fillWriterVectors(opCtx, &ops, &writerVectors, &derivedOps); @@ -685,7 +685,7 @@ StatusWith<OpTime> OplogApplierImpl::_applyOplogBatch(OperationContext* opCtx, void OplogApplierImpl::_deriveOpsAndFillWriterVectors( OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps, SessionUpdateTracker* sessionUpdateTracker) noexcept { @@ -785,7 +785,7 @@ void OplogApplierImpl::_deriveOpsAndFillWriterVectors( void OplogApplierImpl::fillWriterVectors( OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps) noexcept { SessionUpdateTracker sessionUpdateTracker; @@ -802,7 +802,7 @@ void OplogApplierImpl::fillWriterVectors( void OplogApplierImpl::fillWriterVectors_forTest( OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps) noexcept { fillWriterVectors(opCtx, ops, writerVectors, derivedOps); } @@ -858,7 +858,7 @@ Status applyOplogEntryOrGroupedInserts(OperationContext* opCtx, } Status OplogApplierImpl::applyOplogBatchPerWorker(OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, WorkerMultikeyPathInfo* workerMultikeyPathInfo, const bool isDataConsistent) { UnreplicatedWritesBlock uwb(opCtx); diff --git a/src/mongo/db/repl/oplog_applier_impl.h b/src/mongo/db/repl/oplog_applier_impl.h index 9e8ac05083d..d3ef84bea2d 100644 --- a/src/mongo/db/repl/oplog_applier_impl.h +++ b/src/mongo/db/repl/oplog_applier_impl.h @@ -75,7 +75,7 @@ public: void fillWriterVectors_forTest(OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps) noexcept; private: @@ -103,7 +103,7 @@ private: void _deriveOpsAndFillWriterVectors(OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps, SessionUpdateTracker* sessionUpdateTracker) noexcept; @@ -124,7 +124,7 @@ private: void fillWriterVectors(OperationContext* opCtx, std::vector<OplogEntry>* ops, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, std::vector<std::vector<OplogEntry>>* derivedOps) noexcept; protected: @@ -138,7 +138,7 @@ protected: * application. */ virtual Status applyOplogBatchPerWorker(OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, WorkerMultikeyPathInfo* workerMultikeyPathInfo, bool isDataConsistent); }; diff --git a/src/mongo/db/repl/oplog_applier_impl_test.cpp b/src/mongo/db/repl/oplog_applier_impl_test.cpp index 3ded20a0e83..f244e03670d 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test.cpp @@ -125,7 +125,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsInsertDocumentDataba NamespaceString nss("test.t"); auto op = makeOplogEntry(OpTypeEnum::kInsert, nss, {}); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); } @@ -152,7 +152,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, NamespaceString otherNss("test.othername"); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, {}); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); } @@ -163,7 +163,7 @@ TEST_F(OplogApplierImplTest, NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kInsert, otherNss, kUuid); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); } @@ -192,7 +192,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, kUuid); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); } @@ -204,7 +204,7 @@ TEST_F(OplogApplierImplTest, applyOplogEntryOrGroupedInsertsInsertDocumentCollec // implicitly create the collection. auto op = makeOplogEntry(OpTypeEnum::kInsert, nss, {}); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); ASSERT_FALSE(collectionExists(_opCtx.get(), nss)); } @@ -238,7 +238,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, // error. auto op = makeOplogEntry(OpTypeEnum::kDelete, nss, {}); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); } @@ -272,7 +272,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, repl::createCollection(_opCtx.get(), nss, {}); auto op = makeOplogEntry(OpTypeEnum::kDelete, nss, {}); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NoSuchKey>); } @@ -392,7 +392,7 @@ TEST_F(OplogApplierImplTestEnableSteadyStateConstraints, NamespaceString otherNss(nss.getSisterNS("othername")); auto op = makeOplogEntry(OpTypeEnum::kDelete, otherNss, options.uuid); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NoSuchKey>); } @@ -476,8 +476,8 @@ TEST_F(OplogApplierImplTest, applyOplogEntryToRecordChangeStreamPreImages) { testCase.fromMigrate); // Apply the oplog entry. - ASSERT_OK( - _applyOplogEntryOrGroupedInsertsWrapper(_opCtx.get(), &op, testCase.applicationMode)); + ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( + _opCtx.get(), ApplierOperation{&op}, testCase.applicationMode)); // Load pre-image and cleanup the state. WriteUnitOfWork wuow{_opCtx.get()}; @@ -534,7 +534,7 @@ TEST_F(OplogApplierImplTest, CreateCollectionCommand) { }; auto entry = OplogEntry(op); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kInitialSync)); + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kInitialSync)); ASSERT_TRUE(applyCmdCalled); } @@ -563,7 +563,7 @@ TEST_F(OplogApplierImplTest, CreateCollectionCommandMultitenant) { auto entry = makeCommandOplogEntry(nextOpTime(), nss, op, UUID::gen()); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kSecondary)); ASSERT_TRUE(applyCmdCalled); } @@ -598,7 +598,7 @@ TEST_F(OplogApplierImplTest, CreateCollectionCommandMultitenantRequireTenantIDFa auto entry = OplogEntry(op); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kSecondary)); ASSERT_TRUE(applyCmdCalled); } @@ -646,14 +646,14 @@ TEST_F(OplogApplierImplTest, CreateCollectionCommandMultitenantAlreadyExists) { // This fails silently so we won't see any indication of a collision, but we can also assert // that the opObserver event above won't be called in the event of a collision. ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry1, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry1}, OplogApplication::Mode::kSecondary)); ASSERT_FALSE(applyCmdCalled); ASSERT_TRUE(collectionExists(_opCtx.get(), nssTenant1)); ASSERT_FALSE(collectionExists(_opCtx.get(), nssTenant2)); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry2, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry2}, OplogApplication::Mode::kSecondary)); ASSERT_TRUE(applyCmdCalled); ASSERT_TRUE(collectionExists(_opCtx.get(), nssTenant1)); @@ -681,7 +681,7 @@ TEST_F(OplogApplierImplTest, RenameCollectionCommandMultitenant) { auto op = makeCommandOplogEntry(opTime, sourceNss, oRename, {}); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary)); ASSERT_FALSE(collectionExists(_opCtx.get(), sourceNss)); ASSERT_TRUE(collectionExists(_opCtx.get(), targetNss)); } @@ -707,7 +707,7 @@ TEST_F(OplogApplierImplTest, RenameCollectionCommandMultitenantRequireTenantIDFa auto op = makeCommandOplogEntry(opTime, sourceNss, oRename, {}); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary)); ASSERT_FALSE(collectionExists(_opCtx.get(), sourceNss)); ASSERT_TRUE(collectionExists(_opCtx.get(), targetNss)); } @@ -738,7 +738,7 @@ TEST_F(OplogApplierImplTest, RenameCollectionCommandMultitenantAcrossTenantsRequ ASSERT_EQ(ErrorCodes::IllegalOperation, _applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary)); ASSERT_TRUE(collectionExists(_opCtx.get(), sourceNss)); ASSERT_FALSE(collectionExists(_opCtx.get(), targetNss)); ASSERT_FALSE(collectionExists(_opCtx.get(), wrongTargetNss)); @@ -778,7 +778,7 @@ TEST_F(IdempotencyTest, CollModCommandMultitenant) { auto op = makeCommandOplogEntry(nextOpTime(), nss, collModCmd, kUuid); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary)); ASSERT_TRUE(applyCmdCalled); } @@ -819,7 +819,7 @@ TEST_F(IdempotencyTest, CollModCommandMultitenantWrongTenant) { // A NamespaceNotFound error is absorbed by the applier, but we can still determine the // op_observer callback was never called ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary)); ASSERT_FALSE(applyCmdCalled); } @@ -834,7 +834,7 @@ public: using OplogApplierImpl::OplogApplierImpl; Status applyOplogBatchPerWorker(OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, WorkerMultikeyPathInfo* workerMultikeyPathInfo, bool isDataConsistent) override; @@ -851,7 +851,7 @@ private: Status TrackOpsAppliedApplier::applyOplogBatchPerWorker( OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, WorkerMultikeyPathInfo* workerMultikeyPathInfo, const bool isDataConsistent) { stdx::lock_guard lk(_mutex); @@ -937,7 +937,7 @@ TEST_F(OplogApplierImplTest, NamespaceString nss("local." + _agent.getSuiteName() + "_" + _agent.getTestName()); auto op = makeCreateCollectionOplogEntry({Timestamp(Seconds(1), 0), 1LL}, nss); - std::vector<const OplogEntry*> ops = {&op}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op}}; WorkerMultikeyPathInfo pathInfo; TestApplyOplogGroupApplier oplogApplier( @@ -977,7 +977,7 @@ TEST_F(OplogApplierImplTest, repl::OplogApplier::Options(repl::OplogApplication::Mode::kSecondary), writerPool.get()); - std::vector<std::vector<const OplogEntry*>> writerVectors( + std::vector<std::vector<ApplierOperation>> writerVectors( writerPool->getStats().options.maxThreads); std::vector<std::vector<OplogEntry>> derivedOps; std::vector<OplogEntry> ops{firstRetryableOp, secondRetryableOp}; @@ -2344,7 +2344,7 @@ void testWorkerMultikeyPaths(OperationContext* opCtx, TestApplyOplogGroupApplier oplogApplier( nullptr, nullptr, OplogApplier::Options(OplogApplication::Mode::kSecondary)); WorkerMultikeyPathInfo pathInfo; - std::vector<const OplogEntry*> ops = {&op}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op}}; const bool dataIsConsistent = true; ASSERT_OK(oplogApplier.applyOplogBatchPerWorker(opCtx, &ops, &pathInfo, dataIsConsistent)); ASSERT_EQ(pathInfo.size(), numPaths); @@ -2411,7 +2411,7 @@ TEST_F(OplogApplierImplTest, OplogApplicationThreadFuncAddsMultipleWorkerMultike TestApplyOplogGroupApplier oplogApplier( nullptr, nullptr, OplogApplier::Options(OplogApplication::Mode::kSecondary)); WorkerMultikeyPathInfo pathInfo; - std::vector<const OplogEntry*> ops = {&opA, &opB}; + std::vector<ApplierOperation> ops = {ApplierOperation{&opA}, ApplierOperation{&opB}}; const bool dataIsConsistent = true; ASSERT_OK( oplogApplier.applyOplogBatchPerWorker(_opCtx.get(), &ops, &pathInfo, dataIsConsistent)); @@ -2460,7 +2460,7 @@ TEST_F(OplogApplierImplTest, OplogApplicationThreadFuncFailsWhenCollectionCreati TestApplyOplogGroupApplier oplogApplier( nullptr, nullptr, OplogApplier::Options(OplogApplication::Mode::kSecondary)); - std::vector<const OplogEntry*> ops = {&op}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op}}; const bool dataIsConsistent = true; ASSERT_EQUALS( ErrorCodes::InvalidOptions, @@ -2846,7 +2846,7 @@ TEST_F(OplogApplierImplTest, ApplyGroupIgnoresUpdateOperationIfDocumentIsMissing } auto op = makeUpdateDocumentOplogEntry( {Timestamp(Seconds(1), 0), 1LL}, nss, BSON("_id" << 0), BSON("_id" << 0 << "x" << 2)); - std::vector<const OplogEntry*> ops = {&op}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op}}; WorkerMultikeyPathInfo pathInfo; const bool dataIsConsistent = true; ASSERT_OK( @@ -2871,7 +2871,10 @@ TEST_F(OplogApplierImplTest, auto op1 = makeInsertDocumentOplogEntry({Timestamp(Seconds(2), 0), 1LL}, nss, doc1); auto op2 = makeInsertDocumentOplogEntry({Timestamp(Seconds(3), 0), 1LL}, badNss, doc2); auto op3 = makeInsertDocumentOplogEntry({Timestamp(Seconds(4), 0), 1LL}, nss, doc3); - std::vector<const OplogEntry*> ops = {&op0, &op1, &op2, &op3}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op0}, + ApplierOperation{&op1}, + ApplierOperation{&op2}, + ApplierOperation{&op3}}; WorkerMultikeyPathInfo pathInfo; const bool dataIsConsistent = true; ASSERT_OK( @@ -2901,7 +2904,10 @@ TEST_F(OplogApplierImplTest, auto op2 = makeCreateIndexOplogEntry( {Timestamp(Seconds(3), 0), 1LL}, badNss, "a_1", keyPattern, kUuid); auto op3 = makeInsertDocumentOplogEntry({Timestamp(Seconds(4), 0), 1LL}, nss, doc3); - std::vector<const OplogEntry*> ops = {&op0, &op1, &op2, &op3}; + std::vector<ApplierOperation> ops = {ApplierOperation{&op0}, + ApplierOperation{&op1}, + ApplierOperation{&op2}, + ApplierOperation{&op3}}; WorkerMultikeyPathInfo pathInfo; const bool dataIsConsistent = true; ASSERT_OK( @@ -3310,7 +3316,7 @@ TEST_F(OplogApplierImplWithFastAutoAdvancingClockTest, LogSlowOpApplicationWhenS startCapturingLogMessages(); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kSecondary)); ASSERT_EQUALS( 1, @@ -3333,7 +3339,7 @@ TEST_F(OplogApplierImplWithFastAutoAdvancingClockTest, DoNotLogSlowOpApplication startCapturingLogMessages(); ASSERT_THROWS(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kSecondary), ExceptionFor<ErrorCodes::NamespaceNotFound>); // Use a builder for easier escaping. We expect the operation to *not* be logged @@ -3356,7 +3362,7 @@ TEST_F(OplogApplierImplWithSlowAutoAdvancingClockTest, DoNotLogNonSlowOpApplicat startCapturingLogMessages(); ASSERT_OK(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &entry, OplogApplication::Mode::kSecondary)); + _opCtx.get(), ApplierOperation{&entry}, OplogApplication::Mode::kSecondary)); // Use a builder for easier escaping. We expect the operation to *not* be logged, // since it wasn't slow to apply. @@ -4709,7 +4715,7 @@ TEST_F(IdempotencyTestTxns, CommitPreparedTransactionIgnoresNamespaceNotFoundErr class GlobalIndexTest : public OplogApplierImplTest { protected: - using WriterVectors = std::vector<std::vector<const OplogEntry*>>; + using WriterVectors = std::vector<std::vector<ApplierOperation>>; void setUp() override { OplogApplierImplTest::setUp(); diff --git a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp index 3860726767c..c543979c1d4 100644 --- a/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp +++ b/src/mongo/db/repl/oplog_applier_impl_test_fixture.cpp @@ -274,7 +274,7 @@ void OplogApplierImplTest::_testApplyOplogEntryOrGroupedInsertsCrudOperation( }; ASSERT_EQ(_applyOplogEntryOrGroupedInsertsWrapper( - _opCtx.get(), &op, OplogApplication::Mode::kSecondary), + _opCtx.get(), ApplierOperation{&op}, OplogApplication::Mode::kSecondary), expectedError); ASSERT_EQ(applyOpCalled, expectedApplyOpCalled); } @@ -295,9 +295,9 @@ Status OplogApplierImplTest::runOpsSteadyState(std::vector<OplogEntry> ops) { getConsistencyMarkers(), getStorageInterface(), repl::OplogApplier::Options(repl::OplogApplication::Mode::kSecondary)); - std::vector<const OplogEntry*> opsPtrs; + std::vector<ApplierOperation> opsPtrs; for (auto& op : ops) { - opsPtrs.push_back(&op); + opsPtrs.emplace_back(&op); } WorkerMultikeyPathInfo pathInfo; const bool dataIsConsistent = true; diff --git a/src/mongo/db/repl/oplog_applier_utils.cpp b/src/mongo/db/repl/oplog_applier_utils.cpp index b7fcfeb76d9..39fcb0dfcaa 100644 --- a/src/mongo/db/repl/oplog_applier_utils.cpp +++ b/src/mongo/db/repl/oplog_applier_utils.cpp @@ -109,7 +109,7 @@ void OplogApplierUtils::processCrudOp( uint32_t OplogApplierUtils::addToWriterVector( OperationContext* opCtx, OplogEntry* op, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, CachedCollectionProperties* collPropertiesCache, boost::optional<uint32_t> forceWriterId) { @@ -134,12 +134,12 @@ uint32_t OplogApplierUtils::addToWriterVector( if (writer.empty()) { writer.reserve(8); // Skip a few growth rounds } - writer.push_back(op); + writer.emplace_back(op); return writerId; } -void OplogApplierUtils::stableSortByNamespace(std::vector<const OplogEntry*>* oplogEntryPointers) { - auto nssComparator = [](const OplogEntry* l, const OplogEntry* r) { +void OplogApplierUtils::stableSortByNamespace(std::vector<ApplierOperation>* oplogEntryPointers) { + auto nssComparator = [](ApplierOperation l, ApplierOperation r) { if (l->getNss().isCommand()) { if (r->getNss().isCommand()) // l == r; now compare the namespace @@ -157,7 +157,7 @@ void OplogApplierUtils::stableSortByNamespace(std::vector<const OplogEntry*>* op void OplogApplierUtils::addDerivedOps(OperationContext* opCtx, std::vector<OplogEntry>* derivedOps, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, CachedCollectionProperties* collPropertiesCache, bool serial) { boost::optional<uint32_t> @@ -305,7 +305,7 @@ Status OplogApplierUtils::applyOplogEntryOrGroupedInsertsCommon( Status OplogApplierUtils::applyOplogBatchCommon( OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, OplogApplication::Mode oplogApplicationMode, bool allowNamespaceNotFoundErrorsOnCrudOps, const bool isDataConsistent, @@ -334,7 +334,7 @@ Status OplogApplierUtils::applyOplogBatchCommon( // If we didn't create a group, try to apply the op individually. try { const Status status = applyOplogEntryOrGroupedInserts( - opCtx, &entry, oplogApplicationMode, isDataConsistent); + opCtx, ApplierOperation{&entry}, oplogApplicationMode, isDataConsistent); if (!status.isOK()) { // Tried to apply an update operation but the document is missing, there must be diff --git a/src/mongo/db/repl/oplog_applier_utils.h b/src/mongo/db/repl/oplog_applier_utils.h index 6954fa4be7e..8b1226375bf 100644 --- a/src/mongo/db/repl/oplog_applier_utils.h +++ b/src/mongo/db/repl/oplog_applier_utils.h @@ -69,7 +69,7 @@ public: * This special sort behavior is required because DDL operations need to run before * create/update/delete operations in a multi-doc transaction. */ - static void stableSortByNamespace(std::vector<const OplogEntry*>* oplogEntryPointers); + static void stableSortByNamespace(std::vector<ApplierOperation>* oplogEntryPointers); /** * Updates a CRUD op's hash and isForCappedCollection field if necessary. @@ -87,7 +87,7 @@ public: */ static uint32_t addToWriterVector(OperationContext* opCtx, OplogEntry* op, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, CachedCollectionProperties* collPropertiesCache, boost::optional<uint32_t> forceWriterId = boost::none); /** @@ -97,7 +97,7 @@ public: */ static void addDerivedOps(OperationContext* opCtx, std::vector<OplogEntry>* derivedOps, - std::vector<std::vector<const OplogEntry*>>* writerVectors, + std::vector<std::vector<ApplierOperation>>* writerVectors, CachedCollectionProperties* collPropertiesCache, bool serial); @@ -132,7 +132,7 @@ public: */ static Status applyOplogBatchCommon( OperationContext* opCtx, - std::vector<const OplogEntry*>* ops, + std::vector<ApplierOperation>* ops, OplogApplication::Mode oplogApplicationMode, bool allowNamespaceNotFoundErrorsOnCrudOps, bool isDataConsistent, diff --git a/src/mongo/db/repl/oplog_entry_or_grouped_inserts.cpp b/src/mongo/db/repl/oplog_entry_or_grouped_inserts.cpp index 6173db95e00..5e3c8209435 100644 --- a/src/mongo/db/repl/oplog_entry_or_grouped_inserts.cpp +++ b/src/mongo/db/repl/oplog_entry_or_grouped_inserts.cpp @@ -54,14 +54,14 @@ BSONObj OplogEntryOrGroupedInserts::toBSON() const { // Populate the "ts" field with an array of all the grouped inserts' timestamps. { BSONArrayBuilder tsArrayBuilder(groupedInsertBuilder.subarrayStart("ts")); - for (auto op : _entryOrGroupedInserts) { + for (const auto& op : _entryOrGroupedInserts) { tsArrayBuilder.append(op->getTimestamp()); } } // Populate the "t" (term) field with an array of all the grouped inserts' terms. { BSONArrayBuilder tArrayBuilder(groupedInsertBuilder.subarrayStart("t")); - for (auto op : _entryOrGroupedInserts) { + for (const auto& op : _entryOrGroupedInserts) { long long term = OpTime::kUninitializedTerm; auto parsedTerm = op->getTerm(); if (parsedTerm) @@ -72,7 +72,7 @@ BSONObj OplogEntryOrGroupedInserts::toBSON() const { // Populate the "o" field with an array of all the grouped inserts. { BSONArrayBuilder oArrayBuilder(groupedInsertBuilder.subarrayStart("o")); - for (auto op : _entryOrGroupedInserts) { + for (const auto& op : _entryOrGroupedInserts) { oArrayBuilder.append(op->getObject()); } } diff --git a/src/mongo/db/repl/oplog_entry_or_grouped_inserts.h b/src/mongo/db/repl/oplog_entry_or_grouped_inserts.h index 8b86fbd473f..a93206e939c 100644 --- a/src/mongo/db/repl/oplog_entry_or_grouped_inserts.h +++ b/src/mongo/db/repl/oplog_entry_or_grouped_inserts.h @@ -34,6 +34,24 @@ namespace mongo { namespace repl { + +enum ApplicationInstruction { applyOplogEntry, startSplitPrepare, splitPrepareOplogEntry }; +struct ApplierOperation { + ApplicationInstruction instruction; + const OplogEntry* op; + LogicalSessionId subSession; + + explicit ApplierOperation(const OplogEntry* op) : instruction(applyOplogEntry), op(op) {} + + const OplogEntry* operator->() const { + return op; + } + + const OplogEntry& operator*() const { + return *op; + } +}; + /** * This is a class for a single oplog entry or grouped inserts to be applied in * applyOplogEntryOrGroupedInserts. This class is immutable and can only be initialized using @@ -41,19 +59,19 @@ namespace repl { */ class OplogEntryOrGroupedInserts { public: - using ConstIterator = std::vector<const OplogEntry*>::const_iterator; + using ConstIterator = std::vector<ApplierOperation>::const_iterator; OplogEntryOrGroupedInserts() = delete; // This initializes it as a single oplog entry. - OplogEntryOrGroupedInserts(const OplogEntry* op) : _entryOrGroupedInserts({op}) {} + OplogEntryOrGroupedInserts(ApplierOperation op) : _entryOrGroupedInserts({op}) {} // This initializes it as grouped inserts. OplogEntryOrGroupedInserts(ConstIterator begin, ConstIterator end) : _entryOrGroupedInserts(begin, end) { // Performs sanity checks to confirm that the batch is valid. invariant(!_entryOrGroupedInserts.empty()); - for (auto op : _entryOrGroupedInserts) { + for (const auto& op : _entryOrGroupedInserts) { // Every oplog entry must be an insert. invariant(op->getOpType() == OpTypeEnum::kInsert); // Every oplog entry must be in the same namespace. @@ -70,7 +88,7 @@ public: return _entryOrGroupedInserts.size() > 1; } - const std::vector<const OplogEntry*>& getGroupedInserts() const { + const std::vector<ApplierOperation>& getGroupedInserts() const { invariant(isGroupedInserts()); return _entryOrGroupedInserts; } @@ -80,7 +98,7 @@ public: private: // A single oplog entry or a batch of grouped insert oplog entries to be applied. - std::vector<const OplogEntry*> _entryOrGroupedInserts; + std::vector<ApplierOperation> _entryOrGroupedInserts; }; } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/storage_timestamp_test.cpp b/src/mongo/db/repl/storage_timestamp_test.cpp index 70a54a98b63..466fdf3de34 100644 --- a/src/mongo/db/repl/storage_timestamp_test.cpp +++ b/src/mongo/db/repl/storage_timestamp_test.cpp @@ -876,7 +876,7 @@ TEST_F(StorageTimestampTest, SecondaryArrayInsertTimes) { std::vector<repl::OplogEntry> oplogEntries; oplogEntries.reserve(docsToInsert); - std::vector<const repl::OplogEntry*> opPtrs; + std::vector<repl::ApplierOperation> opPtrs; BSONObjBuilder oplogEntryBuilders[docsToInsert]; for (std::int32_t idx = 0; idx < docsToInsert; ++idx) { auto o = BSON("_id" << idx); @@ -892,7 +892,7 @@ TEST_F(StorageTimestampTest, SecondaryArrayInsertTimes) { oplogEntryBuilders[idx].appendElementsUnique(oplogCommon); // Insert ops to be applied. oplogEntries.push_back(repl::OplogEntry(oplogEntryBuilders[idx].done())); - opPtrs.push_back(&(oplogEntries.back())); + opPtrs.emplace_back(&(oplogEntries.back())); } repl::OplogEntryOrGroupedInserts groupedInserts(opPtrs.cbegin(), opPtrs.cend()); @@ -2524,7 +2524,7 @@ public: _taskFuture(taskFuture) {} Status applyOplogBatchPerWorker(OperationContext* opCtx, - std::vector<const repl::OplogEntry*>* operationsToApply, + std::vector<repl::ApplierOperation>* operationsToApply, WorkerMultikeyPathInfo* pathInfo, bool isDataConsistent) override; @@ -2542,7 +2542,7 @@ private: // and this test case fails without crashing the entire suite. Status SecondaryReadsDuringBatchApplicationAreAllowedApplier::applyOplogBatchPerWorker( OperationContext* opCtx, - std::vector<const repl::OplogEntry*>* operationsToApply, + std::vector<repl::ApplierOperation>* operationsToApply, WorkerMultikeyPathInfo* pathInfo, const bool isDataConsistent) { if (!_testOpCtx->lockState()->isLockHeldForMode(resourceIdParallelBatchWriterMode, MODE_X)) { @@ -2874,8 +2874,11 @@ TEST_F(StorageTimestampTest, TimestampIndexOplogApplicationOnPrimary) { auto start = repl::makeStartIndexBuildOplogEntry( startBuildOpTime, nss, "field_1", keyPattern, collUUID, indexBuildUUID); const bool dataIsConsistent = true; - ASSERT_OK(repl::applyOplogEntryOrGroupedInserts( - _opCtx, &start, repl::OplogApplication::Mode::kSecondary, dataIsConsistent)); + ASSERT_OK( + repl::applyOplogEntryOrGroupedInserts(_opCtx, + repl::ApplierOperation{&start}, + repl::OplogApplication::Mode::kSecondary, + dataIsConsistent)); // We cannot use the OperationContext to wait for the thread to reach the fail point // because it also uses the ClockSourceMock. @@ -2901,8 +2904,10 @@ TEST_F(StorageTimestampTest, TimestampIndexOplogApplicationOnPrimary) { auto commit = repl::makeCommitIndexBuildOplogEntry( startBuildOpTime, nss, "field_1", keyPattern, collUUID, indexBuildUUID); const bool dataIsConsistent = true; - ASSERT_OK(repl::applyOplogEntryOrGroupedInserts( - _opCtx, &commit, repl::OplogApplication::Mode::kSecondary, dataIsConsistent)); + ASSERT_OK(repl::applyOplogEntryOrGroupedInserts(_opCtx, + repl::ApplierOperation{&commit}, + repl::OplogApplication::Mode::kSecondary, + dataIsConsistent)); // Reacquire read lock to check index metadata. AutoGetCollection autoColl(_opCtx, nss, LockMode::MODE_IS); diff --git a/src/mongo/db/repl/tenant_oplog_applier.cpp b/src/mongo/db/repl/tenant_oplog_applier.cpp index a0321a95ef3..8bfb81dec34 100644 --- a/src/mongo/db/repl/tenant_oplog_applier.cpp +++ b/src/mongo/db/repl/tenant_oplog_applier.cpp @@ -928,9 +928,9 @@ void TenantOplogApplier::_writeNoOpsForRange(OpObserver* opObserver, }); } -std::vector<std::vector<const OplogEntry*>> TenantOplogApplier::_fillWriterVectors( +std::vector<std::vector<ApplierOperation>> TenantOplogApplier::_fillWriterVectors( OperationContext* opCtx, TenantOplogBatch* batch) { - std::vector<std::vector<const OplogEntry*>> writerVectors( + std::vector<std::vector<ApplierOperation>> writerVectors( _writerPool->getStats().options.maxThreads); CachedCollectionProperties collPropertiesCache; @@ -1062,7 +1062,7 @@ Status TenantOplogApplier::_applyOplogEntryOrGroupedInserts( return status; } -Status TenantOplogApplier::_applyOplogBatchPerWorker(std::vector<const OplogEntry*>* ops) { +Status TenantOplogApplier::_applyOplogBatchPerWorker(std::vector<ApplierOperation>* ops) { auto opCtx = cc().makeOperationContext(); opCtx->setEnforceConstraints(false); tenantMigrationInfo(opCtx.get()) = boost::make_optional<TenantMigrationInfo>(_migrationUuid); diff --git a/src/mongo/db/repl/tenant_oplog_applier.h b/src/mongo/db/repl/tenant_oplog_applier.h index db50201afad..23824bb5670 100644 --- a/src/mongo/db/repl/tenant_oplog_applier.h +++ b/src/mongo/db/repl/tenant_oplog_applier.h @@ -122,11 +122,11 @@ private: bool _shouldStopApplying(Status status); void _applyOplogBatch(TenantOplogBatch* batch); - Status _applyOplogBatchPerWorker(std::vector<const OplogEntry*>* ops); + Status _applyOplogBatchPerWorker(std::vector<ApplierOperation>* ops); void _checkNsAndUuidsBelongToTenant(OperationContext* opCtx, const TenantOplogBatch& batch); OpTimePair _writeNoOpEntries(OperationContext* opCtx, const TenantOplogBatch& batch); - using TenantNoOpEntry = std::pair<const OplogEntry*, std::vector<OplogSlot>::iterator>; + using TenantNoOpEntry = std::pair<ApplierOperation, std::vector<OplogSlot>::iterator>; void _writeNoOpsForRange(OpObserver* opObserver, std::vector<TenantNoOpEntry>::const_iterator begin, std::vector<TenantNoOpEntry>::const_iterator end); @@ -137,8 +137,8 @@ private: const OplogEntryOrGroupedInserts& entryOrGroupedInserts, OplogApplication::Mode oplogApplicationMode, bool isDataConsistent); - std::vector<std::vector<const OplogEntry*>> _fillWriterVectors(OperationContext* opCtx, - TenantOplogBatch* batch); + std::vector<std::vector<ApplierOperation>> _fillWriterVectors(OperationContext* opCtx, + TenantOplogBatch* batch); /** * Sets the _finalStatus to the new status if and only if the old status is "OK". diff --git a/src/mongo/db/repl/transaction_oplog_application.cpp b/src/mongo/db/repl/transaction_oplog_application.cpp index a56fb3bbf4e..44d58963350 100644 --- a/src/mongo/db/repl/transaction_oplog_application.cpp +++ b/src/mongo/db/repl/transaction_oplog_application.cpp @@ -81,7 +81,7 @@ Status _applyOperationsForTransaction(OperationContext* opCtx, const bool isDataConsistent = true; auto status = repl::applyOperation_inlock(opCtx, coll.getDb(), - &op, + repl::ApplierOperation{&op}, false /*alwaysUpsert*/, oplogApplicationMode, isDataConsistent); |
