SERVER-40919 Remove stmtId from transaction oplog entries

10 files changed, 43 insertions, 107 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 74a146de680..5b1838a0ca7 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -9902,7 +9902,6 @@ buildvariants:
     test_flags: >-
         --mongodSetParameters="{useMultipleOplogEntryFormatForTransactions: true, maxNumberOfTransactionOperationsInSingleOplogEntry: 2}"
         --excludeWithAnyTags=exclude_from_large_txns
-        --excludeWithAnyTags=exclude_from_large_txns_due_to_stmtids
         --excludeWithAnyTags=exclude_from_large_txns_due_to_change_streams
   tasks:
   - name: compile_TG
diff --git a/jstests/core/txns/await_prepared_transactions_on_FCV_downgrade.js b/jstests/core/txns/await_prepared_transactions_on_FCV_downgrade.js
index 47f90aab4dd..a6623101196 100644
--- a/jstests/core/txns/await_prepared_transactions_on_FCV_downgrade.js
+++ b/jstests/core/txns/await_prepared_transactions_on_FCV_downgrade.js
@@ -1,6 +1,6 @@
 /**
  * Test that we wait for prepared transactions to finish during downgrade to FCV 4.0.
- * @tags: [uses_transactions, uses_prepare_transaction, exclude_from_large_txns_due_to_stmtids]
+ * @tags: [uses_transactions, uses_prepare_transaction]
  */
 (function() {
     "use strict";
diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp
index c0d38b64504..1e87d9681f7 100644
--- a/src/mongo/db/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer_impl.cpp
@@ -85,7 +85,7 @@ repl::OpTime logOperation(OperationContext* opCtx,
                           bool fromMigrate,
                           Date_t wallClockTime,
                           const OperationSessionInfo& sessionInfo,
-                          StmtId stmtId,
+                          boost::optional<StmtId> stmtId,
                           const repl::OplogLink& oplogLink,
                           const OplogSlot& oplogSlot) {
     auto& times = OpObserver::Times::get(opCtx).reservedOpTimes;
@@ -297,7 +297,7 @@ OpTimeBundle replLogApplyOps(OperationContext* opCtx,
                              const NamespaceString& cmdNss,
                              const BSONObj& applyOpCmd,
                              const OperationSessionInfo& sessionInfo,
-                             StmtId stmtId,
+                             boost::optional<StmtId> stmtId,
                              const repl::OplogLink& oplogLink,
                              const OplogSlot& oplogSlot) {
     OpTimeBundle times;
@@ -1018,7 +1018,7 @@ OpTimeBundle logApplyOpsForTransaction(OperationContext* opCtx,
                                        BSONObjBuilder* applyOpsBuilder,
                                        const OplogSlot& oplogSlot,
                                        repl::OpTime prevWriteOpTime,
-                                       StmtId stmtId,
+                                       boost::optional<StmtId> stmtId,
                                        const bool prepare,
                                        const bool isPartialTxn,
                                        const bool shouldWriteStateField,
@@ -1066,13 +1066,8 @@ OpTimeBundle logApplyOpsForTransaction(OperationContext* opCtx,
         }();
 
         if (updateTxnTable) {
-            onWriteOpCompleted(opCtx,
-                               cmdNss,
-                               {stmtId},
-                               times.writeOpTime,
-                               times.wallClockTime,
-                               txnState,
-                               startOpTime);
+            onWriteOpCompleted(
+                opCtx, cmdNss, {}, times.writeOpTime, times.wallClockTime, txnState, startOpTime);
         }
         return times;
     } catch (const AssertionException& e) {
@@ -1094,7 +1089,7 @@ OpTimeBundle logApplyOpsForTransaction(OperationContext* opCtx,
                                        const std::vector<repl::ReplOperation>& statements,
                                        const OplogSlot& oplogSlot,
                                        repl::OpTime prevWriteOpTime,
-                                       StmtId stmtId,
+                                       boost::optional<StmtId> stmtId,
                                        const bool prepare,
                                        const bool isPartialTxn,
                                        const bool shouldWriteStateField,
@@ -1117,33 +1112,6 @@ OpTimeBundle logApplyOpsForTransaction(OperationContext* opCtx,
                                      startOpTime);
 }
 
-OpTimeBundle logReplOperationForTransaction(OperationContext* opCtx,
-                                            const OperationSessionInfo& sessionInfo,
-                                            repl::OpTime prevOpTime,
-                                            StmtId stmtId,
-                                            const repl::ReplOperation& stmt,
-                                            const OplogSlot& oplogSlot) {
-    repl::OplogLink oplogLink;
-    oplogLink.prevOpTime = prevOpTime;
-    OpTimeBundle times;
-    // The IDL serializer always returns null-terminated string literals, so rawData is safe.
-    const char* optype = repl::OpType_serializer(stmt.getOpType()).rawData();
-    times.wallClockTime = getWallClockTimeForOpLog(opCtx);
-    times.writeOpTime = logOperation(opCtx,
-                                     optype,
-                                     stmt.getNss(),
-                                     stmt.getUuid(),
-                                     stmt.getObject(),
-                                     stmt.getObject2() ? &*stmt.getObject2() : nullptr,
-                                     false /* fromMigrate*/,
-                                     times.wallClockTime,
-                                     sessionInfo,
-                                     stmtId,
-                                     oplogLink,
-                                     oplogSlot);
-    return times;
-}
-
 // Logs transaction oplog entries for preparing a transaction or committing an unprepared
 // transaction. This includes the in-progress 'partialTxn' oplog entries followed by the implicit
 // prepare or commit entry. If the 'prepare' argument is true, it will log entries for a prepared
@@ -1172,7 +1140,7 @@ int logOplogEntriesForTransaction(OperationContext* opCtx,
 
     const auto txnParticipant = TransactionParticipant::get(opCtx);
     OpTimeBundle prevWriteOpTime;
-    StmtId stmtId = 0;
+    auto numEntriesWritten = 0;
     writeConflictRetry(
         opCtx, "logOplogEntriesForTransaction", NamespaceString::kRsOplogNamespace.ns(), [&] {
             // Writes to the oplog only require a Global intent lock. Guaranteed by
@@ -1182,8 +1150,6 @@ int logOplogEntriesForTransaction(OperationContext* opCtx,
             WriteUnitOfWork wuow(opCtx);
 
             prevWriteOpTime.writeOpTime = txnParticipant.getLastWriteOpTime();
-            // Note the logged statement IDs are not the same as the user-chosen statement IDs.
-            stmtId = 0;
             auto currOplogSlot = oplogSlots.begin();
 
             // At the beginning of each loop iteration below, 'stmtsIter' will always point to the
@@ -1233,7 +1199,7 @@ int logOplogEntriesForTransaction(OperationContext* opCtx,
                                                             &applyOpsBuilder,
                                                             oplogSlot,
                                                             prevWriteOpTime.writeOpTime,
-                                                            stmtId,
+                                                            boost::none /* stmtId */,
                                                             implicitPrepare,
                                                             isPartialTxn,
                                                             true /* shouldWriteStateField */,
@@ -1242,11 +1208,11 @@ int logOplogEntriesForTransaction(OperationContext* opCtx,
                                                             startOpTime);
                 // Advance the iterator to the beginning of the remaining unpacked statements.
                 stmtsIter = nextStmt;
-                stmtId++;
+                numEntriesWritten++;
             }
             wuow.commit();
         });
-    return stmtId;
+    return numEntriesWritten;
 }
 
 void logCommitOrAbortForTransaction(OperationContext* opCtx,
@@ -1263,22 +1229,6 @@ void logCommitOrAbortForTransaction(OperationContext* opCtx,
     auto txnParticipant = TransactionParticipant::get(opCtx);
     oplogLink.prevOpTime = txnParticipant.getLastWriteOpTime();
 
-    StmtId stmtId(1);
-    if (gUseMultipleOplogEntryFormatForTransactions &&
-        serverGlobalParams.featureCompatibility.getVersion() ==
-            ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo42 &&
-        durableState != DurableTxnStateEnum::kAborted) {
-        // Statement ids are only required to increase monotonically within a transaction. They
-        // don't need to be strictly consecutive. The statements from this transaction may have been
-        // packed into some number of oplog entries <= the total number of statements, so the
-        // statement id we use for the commit here should always be safe.
-        stmtId = txnParticipant.retrieveCompletedTransactionOperations(opCtx).size() + 1;
-    }
-    // When we abort a transaction on stepup, we won't know the number of operations since we
-    // don't reconstruct them. Using the max integer would be safe.
-    if (durableState == DurableTxnStateEnum::kAborted) {
-        stmtId = std::numeric_limits<StmtId>::max();
-    }
     const auto wallClockTime = getWallClockTimeForOpLog(opCtx);
 
     // There should not be a parent WUOW outside of this one. This guarantees the safety of the
@@ -1307,14 +1257,14 @@ void logCommitOrAbortForTransaction(OperationContext* opCtx,
                                                   false /* fromMigrate */,
                                                   wallClockTime,
                                                   sessionInfo,
-                                                  stmtId,
+                                                  boost::none /* stmtId */,
                                                   oplogLink,
                                                   oplogSlot);
             invariant(oplogSlot.isNull() || oplogSlot == oplogOpTime);
 
             onWriteOpCompleted(opCtx,
                                cmdNss,
-                               {stmtId},
+                               {},
                                oplogOpTime,
                                wallClockTime,
                                durableState,
@@ -1434,7 +1384,7 @@ void OpObserverImpl::onTransactionPrepare(OperationContext* opCtx,
                     statements,
                     prepareOpTime,
                     lastWriteOpTime,
-                    StmtId(0),
+                    boost::none /* stmtId */,
                     true /* prepare */,
                     false /* isPartialTxn */,
                     fcv >= ServerGlobalParams::FeatureCompatibility::Version::kUpgradingTo42,
@@ -1477,7 +1427,7 @@ void OpObserverImpl::onTransactionPrepare(OperationContext* opCtx,
                                               &applyOpsBuilder,
                                               oplogSlot,
                                               repl::OpTime() /* prevWriteOpTime */,
-                                              StmtId(0),
+                                              boost::none /* stmtId */,
                                               true /* prepare */,
                                               false /* isPartialTxn */,
                                               true /* shouldWriteStateField */,
diff --git a/src/mongo/db/op_observer_impl_test.cpp b/src/mongo/db/op_observer_impl_test.cpp
index f4b73434b70..5bb742454cc 100644
--- a/src/mongo/db/op_observer_impl_test.cpp
+++ b/src/mongo/db/op_observer_impl_test.cpp
@@ -2008,7 +2008,6 @@ TEST_F(OpObserverMultiEntryTransactionTest, CommitPreparedTest) {
     }
     oplogEntryObjs = getNOplogEntries(opCtx(), 3);
     const auto commitOplogObj = oplogEntryObjs.back();
-    // Statement id's for the insert and implicit prepare should be 0 and 1, respectively.
     checkSessionAndTransactionFields(commitOplogObj);
     auto commitEntry = assertGet(OplogEntry::parse(commitOplogObj));
     auto o = commitEntry.getObject();
@@ -2083,7 +2082,6 @@ TEST_F(OpObserverMultiEntryTransactionTest, AbortPreparedTest) {
 
     oplogEntryObjs = getNOplogEntries(opCtx(), 2);
     auto abortOplogObj = oplogEntryObjs.back();
-    // Statement id for the implicit prepare should be 0.
     checkSessionAndTransactionFields(abortOplogObj);
     auto abortEntry = assertGet(OplogEntry::parse(abortOplogObj));
     auto o = abortEntry.getObject();
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 6517e60fa83..1c785e6186b 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -336,21 +336,25 @@ namespace {
  */
 void appendSessionInfo(OperationContext* opCtx,
                        BSONObjBuilder* builder,
-                       StmtId statementId,
+                       boost::optional<StmtId> statementId,
                        const OperationSessionInfo& sessionInfo,
                        const OplogLink& oplogLink) {
     if (!sessionInfo.getTxnNumber()) {
         return;
     }
 
-    // Note: certain operations, like implicit collection creation will not have a stmtId.
+    // Note: certain non-transaction operations, like implicit collection creation will have an
+    // uninitialized statementId.
     if (statementId == kUninitializedStmtId) {
         return;
     }
 
     sessionInfo.serialize(builder);
 
-    builder->append(OplogEntryBase::kStatementIdFieldName, statementId);
+    // Only non-transaction operations will have a statementId.
+    if (statementId) {
+        builder->append(OplogEntryBase::kStatementIdFieldName, *statementId);
+    }
     oplogLink.prevOpTime.append(builder,
                                 OplogEntryBase::kPrevWriteOpTimeInTransactionFieldName.toString());
 
@@ -375,7 +379,7 @@ OplogDocWriter _logOpWriter(OperationContext* opCtx,
                             OpTime optime,
                             Date_t wallTime,
                             const OperationSessionInfo& sessionInfo,
-                            StmtId statementId,
+                            boost::optional<StmtId> statementId,
                             const OplogLink& oplogLink) {
     BSONObjBuilder b(256);
 
@@ -485,7 +489,7 @@ OpTime logOp(OperationContext* opCtx,
              bool fromMigrate,
              Date_t wallClockTime,
              const OperationSessionInfo& sessionInfo,
-             StmtId statementId,
+             boost::optional<StmtId> statementId,
              const OplogLink& oplogLink,
              const OplogSlot& oplogSlot) {
     // All collections should have UUIDs now, so all insert, update, and delete oplog entries should
@@ -502,10 +506,11 @@ OpTime logOp(OperationContext* opCtx,
     // For commands, the test below is on the command ns and therefore does not check for
     // specific namespaces such as system.profile. This is the caller's responsibility.
     if (replCoord->isOplogDisabledFor(opCtx, nss)) {
+        invariant(statementId);
         uassert(ErrorCodes::IllegalOperation,
                 str::stream() << "retryable writes is not supported for unreplicated ns: "
                               << nss.ns(),
-                statementId == kUninitializedStmtId);
+                *statementId == kUninitializedStmtId);
         return {};
     }
 
diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h
index 16a4880ac1c..b8729038179 100644
--- a/src/mongo/db/repl/oplog.h
+++ b/src/mongo/db/repl/oplog.h
@@ -120,6 +120,8 @@ std::vector<OpTime> logInsertOps(OperationContext* opCtx,
  * wallClockTime this specifies the wall-clock timestamp of then this oplog entry was generated. It
  *   is purely informational, may not be monotonically increasing and is not interpreted in any way
  *   by the replication subsystem.
+ * stmtId specifies the statementId of an operation. For transaction operations, stmtId is always
+ *   boost::none.
  * oplogLink this contains the timestamp that points to the previous write that will be
  *   linked via prevTs, and the timestamps of the oplog entry that contains the document
  *   before/after update was applied. The timestamps are ignored if isNull() is true.
@@ -138,7 +140,7 @@ OpTime logOp(OperationContext* opCtx,
              bool fromMigrate,
              Date_t wallClockTime,
              const OperationSessionInfo& sessionInfo,
-             StmtId stmtId,
+             boost::optional<StmtId> stmtId,
              const OplogLink& oplogLink,
              const OplogSlot& oplogSlot);
 
diff --git a/src/mongo/db/repl/oplog_entry.idl b/src/mongo/db/repl/oplog_entry.idl
index 19ce453ce19..aa11922f513 100644
--- a/src/mongo/db/repl/oplog_entry.idl
+++ b/src/mongo/db/repl/oplog_entry.idl
@@ -124,8 +124,7 @@ structs:
             stmtId:
                 cpp_name: statementId
                 type: StmtId
-                optional: true # If txnNumber is missing, this will also be absent, otherwise it
-                               # must exist.
+                optional: true
                 description: "Identifier of the transaction statement which generated this oplog
                               entry"
             prevOpTime:
diff --git a/src/mongo/db/repl/rs_rollback.cpp b/src/mongo/db/repl/rs_rollback.cpp
index e0063ec6ead..e27d463b0f7 100644
--- a/src/mongo/db/repl/rs_rollback.cpp
+++ b/src/mongo/db/repl/rs_rollback.cpp
@@ -274,7 +274,6 @@ Status rollback_internal::updateFixUpInfoFromLocalOplogEntry(OperationContext* o
     if (txnNumber) {
         auto sessionId = operationSessionInfo.getSessionId();
         invariant(sessionId);
-        invariant(oplogEntry.getStatementId());
 
         auto transactionTableUUID = fixUpInfo.transactionTableUUID;
         if (transactionTableUUID) {
diff --git a/src/mongo/db/repl/rs_rollback_test.cpp b/src/mongo/db/repl/rs_rollback_test.cpp
index 10fec44b23b..f9f122309d4 100644
--- a/src/mongo/db/repl/rs_rollback_test.cpp
+++ b/src/mongo/db/repl/rs_rollback_test.cpp
@@ -2113,30 +2113,6 @@ DEATH_TEST_F(RSRollbackTest, LocalEntryWithTxnNumberWithoutSessionIdIsFatal, "in
                   RSFatalException);
 }
 
-DEATH_TEST_F(RSRollbackTest, LocalEntryWithTxnNumberWithoutStmtIdIsFatal, "invariant") {
-    auto validOplogEntry = BSON("ts" << Timestamp(Seconds(1), 0) << "t" << 1LL << "op"
-                                     << "i"
-                                     << "ui"
-                                     << UUID::gen()
-                                     << "ns"
-                                     << "test.t"
-                                     << "o"
-                                     << BSON("_id" << 1 << "a" << 1));
-    FixUpInfo fui;
-    ASSERT_OK(updateFixUpInfoFromLocalOplogEntry(
-        nullptr /* opCtx */, OplogInterfaceMock(), fui, validOplogEntry, false));
-
-    const auto txnNumber = BSON("txnNumber" << 1LL);
-    const auto noSessionIdOrStmtId = validOplogEntry.addField(txnNumber.firstElement());
-
-    const auto lsid = makeLogicalSessionIdForTest();
-    const auto sessionId = BSON("lsid" << lsid.toBSON());
-    const auto noStmtId = noSessionIdOrStmtId.addField(sessionId.firstElement());
-    ASSERT_THROWS(updateFixUpInfoFromLocalOplogEntry(
-                      nullptr /* opCtx */, OplogInterfaceMock(), fui, noStmtId, false),
-                  RSFatalException);
-}
-
 TEST_F(RSRollbackTest, LocalEntryWithTxnNumberWithoutTxnTableUUIDIsFatal) {
     // If txnNumber is present, but the transaction collection has no UUID, rollback fails.
     UUID uuid = UUID::gen();
diff --git a/src/mongo/db/transaction_participant.cpp b/src/mongo/db/transaction_participant.cpp
index 4cd71c8f5b3..e5e819624fc 100644
--- a/src/mongo/db/transaction_participant.cpp
+++ b/src/mongo/db/transaction_participant.cpp
@@ -135,29 +135,37 @@ ActiveTransactionHistory fetchActiveTransactionHistory(OperationContext* opCtx,
     if ((serverGlobalParams.featureCompatibility.getVersion() >=
          ServerGlobalParams::FeatureCompatibility::Version::kDowngradingTo40)) {
 
+        // The state being kPrepared marks a prepared transaction. We should never be refreshing
+        // a prepared transaction from storage since it should already be in a valid state after
+        // replication recovery.
+        invariant(result.lastTxnRecord->getState() != DurableTxnStateEnum::kPrepared);
+
         // The state being kCommitted marks the commit of a transaction.
         if (result.lastTxnRecord->getState() == DurableTxnStateEnum::kCommitted) {
             result.state = result.TxnRecordState::kCommitted;
+            return result;
         }
 
         // The state being kAborted marks the abort of a prepared transaction since we do not write
         // down abortTransaction oplog entries in 4.0.
         if (result.lastTxnRecord->getState() == DurableTxnStateEnum::kAborted) {
             result.state = result.TxnRecordState::kAbortedWithPrepare;
+            return result;
         }
 
-        // The state being kPrepared marks a prepared transaction. We should never be refreshing
-        // a prepared transaction from storage since it should already be in a valid state after
-        // replication recovery.
-        invariant(result.lastTxnRecord->getState() != DurableTxnStateEnum::kPrepared);
+        if (result.lastTxnRecord->getState() == DurableTxnStateEnum::kInProgress) {
+            return result;
+        }
     }
 
     auto it = TransactionHistoryIterator(result.lastTxnRecord->getLastWriteOpTime());
     while (it.hasNext()) {
         try {
             const auto entry = it.next(opCtx);
-            invariant(entry.getStatementId());
 
+            // Each entry should correspond to a retryable write or a FCV4.0 format transaction.
+            // These oplog entries must have statementIds.
+            invariant(entry.getStatementId());
             if (*entry.getStatementId() == kIncompleteHistoryStmtId) {
                 // Only the dead end sentinel can have this id for oplog write history
                 invariant(entry.getObject2());