SERVER-56373: BACKPORT-8899 from 8447dea added needsRetryImage support to log serializer for oplog

8 files changed, 97 insertions, 46 deletions

diff --git a/src/mongo/db/SConscript b/src/mongo/db/SConscript
index c69ff52cbef..5fc695bb2d7 100644
--- a/src/mongo/db/SConscript
+++ b/src/mongo/db/SConscript
@@ -907,6 +907,7 @@ env.Library(
         'views/views_mongod',
         '$BUILD_DIR/mongo/base',
         '$BUILD_DIR/mongo/db/catalog/collection_catalog',
+        '$BUILD_DIR/mongo/db/repl/repl_server_parameters',
         '$BUILD_DIR/mongo/s/coreshard',
         "$BUILD_DIR/mongo/s/grid",
     ],
diff --git a/src/mongo/db/commands/dbcheck.cpp b/src/mongo/db/commands/dbcheck.cpp
index 6b068c53616..5a690659597 100644
--- a/src/mongo/db/commands/dbcheck.cpp
+++ b/src/mongo/db/commands/dbcheck.cpp
@@ -484,7 +484,8 @@ private:
                                                   {},
                                                   kUninitializedStmtId,
                                                   {},
-                                                  OplogSlot());
+                                                  OplogSlot(),
+                                                  {});
                 uow.commit();
                 return result;
             });
diff --git a/src/mongo/db/op_observer_impl.cpp b/src/mongo/db/op_observer_impl.cpp
index 9254605f578..527d989203f 100644
--- a/src/mongo/db/op_observer_impl.cpp
+++ b/src/mongo/db/op_observer_impl.cpp
@@ -49,6 +49,7 @@
 #include "mongo/db/operation_context.h"
 #include "mongo/db/repl/oplog.h"
 #include "mongo/db/repl/oplog_entry_gen.h"
+#include "mongo/db/repl/repl_server_parameters_gen.h"
 #include "mongo/db/repl/replication_coordinator.h"
 #include "mongo/db/s/collection_sharding_state.h"
 #include "mongo/db/server_options.h"
@@ -88,7 +89,8 @@ repl::OpTime logOperation(OperationContext* opCtx,
                           const OperationSessionInfo& sessionInfo,
                           boost::optional<StmtId> stmtId,
                           const repl::OplogLink& oplogLink,
-                          const OplogSlot& oplogSlot) {
+                          const OplogSlot& oplogSlot,
+                          boost::optional<repl::RetryImageEnum> needsRetryImage) {
     auto& times = OpObserver::Times::get(opCtx).reservedOpTimes;
     auto opTime = repl::logOp(opCtx,
                               opstr,
@@ -101,7 +103,8 @@ repl::OpTime logOperation(OperationContext* opCtx,
                               sessionInfo,
                               stmtId,
                               oplogLink,
-                              oplogSlot);
+                              oplogSlot,
+                              needsRetryImage);
 
     times.push_back(opTime);
     return opTime;
@@ -169,11 +172,18 @@ struct OpTimeBundle {
  */
 OpTimeBundle replLogUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs& args) {
     BSONObj storeObj;
+    boost::optional<repl::RetryImageEnum> needsRetryImage;
     if (args.updateArgs.storeDocOption == CollectionUpdateArgs::StoreDocOption::PreImage) {
         invariant(args.updateArgs.preImageDoc);
         storeObj = *args.updateArgs.preImageDoc;
+        if (repl::gStoreFindAndModifyImagesInSideCollection.load() && opCtx->getTxnNumber()) {
+            needsRetryImage = repl::RetryImageEnum::kPreImage;
+        }
     } else if (args.updateArgs.storeDocOption == CollectionUpdateArgs::StoreDocOption::PostImage) {
         storeObj = args.updateArgs.updatedDoc;
+        if (repl::gStoreFindAndModifyImagesInSideCollection.load() && opCtx->getTxnNumber()) {
+            needsRetryImage = repl::RetryImageEnum::kPostImage;
+        }
     }
 
     OperationSessionInfo sessionInfo;
@@ -189,7 +199,8 @@ OpTimeBundle replLogUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs&
     OpTimeBundle opTimes;
     opTimes.wallClockTime = getWallClockTimeForOpLog(opCtx);
 
-    if (!storeObj.isEmpty() && opCtx->getTxnNumber()) {
+    if (!repl::gStoreFindAndModifyImagesInSideCollection.load() && !storeObj.isEmpty() &&
+        opCtx->getTxnNumber()) {
         auto noteUpdateOpTime = logOperation(opCtx,
                                              "n",
                                              args.nss,
@@ -201,7 +212,8 @@ OpTimeBundle replLogUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs&
                                              sessionInfo,
                                              args.updateArgs.stmtId,
                                              {},
-                                             OplogSlot());
+                                             OplogSlot(),
+                                             {});
 
         opTimes.prePostImageOpTime = noteUpdateOpTime;
 
@@ -224,7 +236,8 @@ OpTimeBundle replLogUpdate(OperationContext* opCtx, const OplogUpdateEntryArgs&
                                        sessionInfo,
                                        args.updateArgs.stmtId,
                                        oplogLink,
-                                       OplogSlot());
+                                       OplogSlot(),
+                                       needsRetryImage);
 
     return opTimes;
 }
@@ -251,21 +264,27 @@ OpTimeBundle replLogDelete(OperationContext* opCtx,
     OpTimeBundle opTimes;
     opTimes.wallClockTime = getWallClockTimeForOpLog(opCtx);
 
+    boost::optional<repl::RetryImageEnum> needsRetryImage;
     if (deletedDoc && opCtx->getTxnNumber()) {
-        auto noteOplog = logOperation(opCtx,
-                                      "n",
-                                      nss,
-                                      uuid,
-                                      deletedDoc.get(),
-                                      nullptr,
-                                      false,
-                                      opTimes.wallClockTime,
-                                      sessionInfo,
-                                      stmtId,
-                                      {},
-                                      OplogSlot());
-        opTimes.prePostImageOpTime = noteOplog;
-        oplogLink.preImageOpTime = noteOplog;
+        if (repl::gStoreFindAndModifyImagesInSideCollection.load()) {
+            needsRetryImage = repl::RetryImageEnum::kPreImage;
+        } else {
+            auto noteOplog = logOperation(opCtx,
+                                          "n",
+                                          nss,
+                                          uuid,
+                                          deletedDoc.get(),
+                                          nullptr,
+                                          false,
+                                          opTimes.wallClockTime,
+                                          sessionInfo,
+                                          stmtId,
+                                          {},
+                                          OplogSlot(),
+                                          {});
+            opTimes.prePostImageOpTime = noteOplog;
+            oplogLink.preImageOpTime = noteOplog;
+        }
     }
 
     auto& documentKey = documentKeyDecoration(opCtx);
@@ -280,7 +299,8 @@ OpTimeBundle replLogDelete(OperationContext* opCtx,
                                        sessionInfo,
                                        stmtId,
                                        oplogLink,
-                                       OplogSlot());
+                                       OplogSlot(),
+                                       needsRetryImage);
     return opTimes;
 }
 
@@ -307,7 +327,8 @@ OpTimeBundle replLogApplyOps(OperationContext* opCtx,
                                      sessionInfo,
                                      stmtId,
                                      oplogLink,
-                                     oplogSlot);
+                                     oplogSlot,
+                                     {});
     return times;
 }
 
@@ -345,7 +366,8 @@ void OpObserverImpl::onCreateIndex(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 void OpObserverImpl::onStartIndexBuild(OperationContext* opCtx,
@@ -381,7 +403,8 @@ void OpObserverImpl::onStartIndexBuild(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 void OpObserverImpl::onCommitIndexBuild(OperationContext* opCtx,
@@ -417,7 +440,8 @@ void OpObserverImpl::onCommitIndexBuild(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 void OpObserverImpl::onAbortIndexBuild(OperationContext* opCtx,
@@ -453,7 +477,8 @@ void OpObserverImpl::onAbortIndexBuild(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 void OpObserverImpl::onInserts(OperationContext* opCtx,
@@ -668,7 +693,8 @@ void OpObserverImpl::onInternalOpMessage(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 void OpObserverImpl::onCreateCollection(OperationContext* opCtx,
@@ -694,7 +720,8 @@ void OpObserverImpl::onCreateCollection(OperationContext* opCtx,
                      {},
                      kUninitializedStmtId,
                      {},
-                     createOpTime);
+                     createOpTime,
+                     {});
     }
 }
 
@@ -732,7 +759,8 @@ void OpObserverImpl::onCollMod(OperationContext* opCtx,
                      {},
                      kUninitializedStmtId,
                      {},
-                     OplogSlot());
+                     OplogSlot(),
+                     {});
     }
 
     // Make sure the UUID values in the Collection metadata, the Collection object, and the UUID
@@ -766,7 +794,8 @@ void OpObserverImpl::onDropDatabase(OperationContext* opCtx, const std::string&
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 
     uassert(
         50714, "dropping the admin database is not allowed.", dbName != NamespaceString::kAdminDb);
@@ -798,7 +827,8 @@ repl::OpTime OpObserverImpl::onDropCollection(OperationContext* opCtx,
                      {},
                      kUninitializedStmtId,
                      {},
-                     OplogSlot());
+                     OplogSlot(),
+                     {});
     }
 
     uassert(50715,
@@ -849,7 +879,8 @@ void OpObserverImpl::onDropIndex(OperationContext* opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 }
 
 
@@ -889,7 +920,8 @@ repl::OpTime OpObserverImpl::preRenameCollection(OperationContext* const opCtx,
                  {},
                  kUninitializedStmtId,
                  {},
-                 OplogSlot());
+                 OplogSlot(),
+                 {});
 
     return {};
 }
@@ -945,7 +977,8 @@ void OpObserverImpl::onEmptyCapped(OperationContext* opCtx,
                      {},
                      kUninitializedStmtId,
                      {},
-                     OplogSlot());
+                     OplogSlot(),
+                     {});
     }
 }
 
@@ -1259,7 +1292,8 @@ void logCommitOrAbortForPreparedTransaction(OperationContext* opCtx,
                                                   sessionInfo,
                                                   boost::none /* stmtId */,
                                                   oplogLink,
-                                                  oplogSlot);
+                                                  oplogSlot,
+                                                  {});
             invariant(oplogSlot.isNull() || oplogSlot == oplogOpTime);
 
             SessionTxnRecord sessionTxnRecord;
diff --git a/src/mongo/db/repl/oplog.cpp b/src/mongo/db/repl/oplog.cpp
index 7dc0d5e6b85..04b693f8285 100644
--- a/src/mongo/db/repl/oplog.cpp
+++ b/src/mongo/db/repl/oplog.cpp
@@ -380,7 +380,8 @@ OplogDocWriter _logOpWriter(OperationContext* opCtx,
                             Date_t wallTime,
                             const OperationSessionInfo& sessionInfo,
                             boost::optional<StmtId> statementId,
-                            const OplogLink& oplogLink) {
+                            const OplogLink& oplogLink,
+                            boost::optional<repl::RetryImageEnum> needsRetryImage) {
     BSONObjBuilder b(256);
 
     b.append("ts", optime.getTimestamp());
@@ -402,6 +403,10 @@ OplogDocWriter _logOpWriter(OperationContext* opCtx,
     if (o2)
         b.append("o2", *o2);
 
+    if (needsRetryImage) {
+        b.append("needsRetryImage", repl::RetryImage_serializer(*needsRetryImage));
+    }
+
     invariant(wallTime != Date_t{});
     b.appendDate(OplogEntryBase::kWallClockTimeFieldName, wallTime);
 
@@ -490,7 +495,8 @@ OpTime logOp(OperationContext* opCtx,
              const OperationSessionInfo& sessionInfo,
              boost::optional<StmtId> statementId,
              const OplogLink& oplogLink,
-             const OplogSlot& oplogSlot) {
+             const OplogSlot& oplogSlot,
+             boost::optional<repl::RetryImageEnum> needsRetryImage) {
     // All collections should have UUIDs now, so all insert, update, and delete oplog entries should
     // also have uuids. Some no-op (n) and command (c) entries may still elide the uuid field.
     invariant(uuid || 'n' == *opstr || 'c' == *opstr,
@@ -539,7 +545,8 @@ OpTime logOp(OperationContext* opCtx,
                                wallClockTime,
                                sessionInfo,
                                statementId,
-                               oplogLink);
+                               oplogLink,
+                               needsRetryImage);
     const DocWriter* basePtr = &writer;
     auto timestamp = slot.getTimestamp();
     _logOpsInner(opCtx, nss, &basePtr, &timestamp, 1, oplog, slot, wallClockTime);
@@ -610,7 +617,8 @@ std::vector<OpTime> logInsertOps(OperationContext* opCtx,
                                           wallClockTime,
                                           sessionInfo,
                                           begin[i].stmtId,
-                                          oplogLink));
+                                          oplogLink,
+                                          {}));
         oplogLink.prevOpTime = insertStatementOplogSlot;
         timestamps[i] = oplogLink.prevOpTime.getTimestamp();
         opTimes.push_back(insertStatementOplogSlot);
diff --git a/src/mongo/db/repl/oplog.h b/src/mongo/db/repl/oplog.h
index 155272bb417..48e59c18384 100644
--- a/src/mongo/db/repl/oplog.h
+++ b/src/mongo/db/repl/oplog.h
@@ -142,7 +142,8 @@ OpTime logOp(OperationContext* opCtx,
              const OperationSessionInfo& sessionInfo,
              boost::optional<StmtId> stmtId,
              const OplogLink& oplogLink,
-             const OplogSlot& oplogSlot);
+             const OplogSlot& oplogSlot,
+             boost::optional<repl::RetryImageEnum> needsRetryImage);
 
 // Flush out the cached pointer to the oplog.
 void clearLocalOplogPtr();
diff --git a/src/mongo/db/repl/oplog_test.cpp b/src/mongo/db/repl/oplog_test.cpp
index 0715f665f8c..e252d77f059 100644
--- a/src/mongo/db/repl/oplog_test.cpp
+++ b/src/mongo/db/repl/oplog_test.cpp
@@ -112,7 +112,8 @@ TEST_F(OplogTest, LogOpReturnsOpTimeOnSuccessfulInsertIntoOplogCollection) {
                        {},
                        kUninitializedStmtId,
                        {},
-                       OplogSlot());
+                       OplogSlot(),
+                       {});
         ASSERT_FALSE(opTime.isNull());
         wunit.commit();
     }
@@ -235,7 +236,8 @@ OpTime _logOpNoopWithMsg(OperationContext* opCtx,
                         {},
                         kUninitializedStmtId,
                         {},
-                        OplogSlot());
+                        OplogSlot(),
+                        {});
     ASSERT_FALSE(opTime.isNull());
 
     ASSERT(opTimeNssMap->find(opTime) == opTimeNssMap->end())
diff --git a/src/mongo/db/s/session_catalog_migration_destination.cpp b/src/mongo/db/s/session_catalog_migration_destination.cpp
index 7c4ab7d50a9..6a7a3ee75de 100644
--- a/src/mongo/db/s/session_catalog_migration_destination.cpp
+++ b/src/mongo/db/s/session_catalog_migration_destination.cpp
@@ -291,7 +291,8 @@ ProcessOplogResult processSessionOplog(const BSONObj& oplogBSON,
                                            sessionInfo,
                                            stmtId,
                                            oplogLink,
-                                           OplogSlot());
+                                           OplogSlot(),
+                                           {});
 
             const auto& oplogOpTime = result.oplogTime;
             uassert(40633,
diff --git a/src/mongo/db/transaction_participant_retryable_writes_test.cpp b/src/mongo/db/transaction_participant_retryable_writes_test.cpp
index c982c6e6305..e0e2ac68387 100644
--- a/src/mongo/db/transaction_participant_retryable_writes_test.cpp
+++ b/src/mongo/db/transaction_participant_retryable_writes_test.cpp
@@ -218,7 +218,8 @@ protected:
                            osi,
                            stmtId,
                            link,
-                           OplogSlot());
+                           OplogSlot(),
+                           {});
     }
 
     repl::OpTime writeTxnRecord(TxnNumber txnNum,
@@ -646,7 +647,8 @@ TEST_F(TransactionParticipantRetryableWritesTest, ErrorOnlyWhenStmtIdBeingChecke
                                   osi,
                                   1,
                                   {},
-                                  OplogSlot());
+                                  OplogSlot(),
+                                  {});
 
         SessionTxnRecord sessionTxnRecord;
         sessionTxnRecord.setSessionId(sessionId);
@@ -679,7 +681,8 @@ TEST_F(TransactionParticipantRetryableWritesTest, ErrorOnlyWhenStmtIdBeingChecke
                                   osi,
                                   kIncompleteHistoryStmtId,
                                   link,
-                                  OplogSlot());
+                                  OplogSlot(),
+                                  {});
 
         SessionTxnRecord sessionTxnRecord;
         sessionTxnRecord.setSessionId(sessionId);