summaryrefslogtreecommitdiff
path: root/src/mongo/db/repl/transaction_oplog_application.cpp
blob: 40e38b0515210bdde5c435c6111e3cd865d3e237 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
/**
 *    Copyright (C) 2018-present MongoDB, Inc.
 *
 *    This program is free software: you can redistribute it and/or modify
 *    it under the terms of the Server Side Public License, version 1,
 *    as published by MongoDB, Inc.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    Server Side Public License for more details.
 *
 *    You should have received a copy of the Server Side Public License
 *    along with this program. If not, see
 *    <http://www.mongodb.com/licensing/server-side-public-license>.
 *
 *    As a special exception, the copyright holders give permission to link the
 *    code of portions of this program with the OpenSSL library under certain
 *    conditions as described in each individual source file and distribute
 *    linked combinations including the program with the OpenSSL library. You
 *    must comply with the Server Side Public License in all respects for
 *    all of the code used other than as permitted herein. If you modify file(s)
 *    with this exception, you may extend this exception to your version of the
 *    file(s), but you are not obligated to do so. If you do not wish to do so,
 *    delete this exception statement from your version. If you delete this
 *    exception statement from all source files in the program, then also delete
 *    it in the license file.
 */


#include "mongo/platform/basic.h"

#include "mongo/db/repl/transaction_oplog_application.h"

#include "mongo/db/catalog_raii.h"
#include "mongo/db/commands/txn_cmds_gen.h"
#include "mongo/db/concurrency/exception_util.h"
#include "mongo/db/dbdirectclient.h"
#include "mongo/db/index_builds_coordinator.h"
#include "mongo/db/op_observer/op_observer.h"
#include "mongo/db/repl/apply_ops.h"
#include "mongo/db/repl/storage_interface_impl.h"
#include "mongo/db/repl/timestamp_block.h"
#include "mongo/db/session/session_catalog_mongod.h"
#include "mongo/db/shard_role.h"
#include "mongo/db/transaction/transaction_history_iterator.h"
#include "mongo/db/transaction/transaction_participant.h"
#include "mongo/logv2/log.h"

#define MONGO_LOGV2_DEFAULT_COMPONENT ::mongo::logv2::LogComponent::kReplication


namespace mongo {

using repl::ApplierOperation;
using repl::OplogEntry;

namespace {
// If enabled, causes _applyPrepareTransaction to hang before preparing the transaction participant.
MONGO_FAIL_POINT_DEFINE(applyOpsHangBeforePreparingTransaction);

// Failpoint that will cause reconstructPreparedTransactions to return early.
MONGO_FAIL_POINT_DEFINE(skipReconstructPreparedTransactions);

// Failpoint that causes apply prepare transaction oplog entry's ops to fail with write
// conflict error.
MONGO_FAIL_POINT_DEFINE(applyPrepareTxnOpsFailsWithWriteConflict);

MONGO_FAIL_POINT_DEFINE(hangBeforeSessionCheckOutForApplyPrepare);

// Given a vector of OplogEntry pointers, copy and return a vector of OplogEntry's.
std::vector<OplogEntry> _copyOps(const std::vector<const OplogEntry*>& ops) {
    std::vector<OplogEntry> res;
    res.reserve(ops.size());
    std::transform(ops.begin(),
                   ops.end(),
                   std::back_inserter(res),
                   [](const auto* op) -> const OplogEntry& { return *op; });

    return res;
}

// Returns all the committed statement IDs from the transaction operations if the transaction is
// a retryable internal transcation.
template <typename Operation>
boost::optional<std::vector<StmtId>> _getCommittedStmtIds(const LogicalSessionId& lsid,
                                                          const std::vector<Operation>& txnOps) {
    // The template type 'Operation' is expected to be either 'OplogEntry'
    // or 'OplogEntry*', so these functions are used to convert it to the
    // latter regardless of which one is the instantiated
    struct OpConverter {
        static const OplogEntry* asPtr(const OplogEntry& op) {
            return &op;
        }
        static const OplogEntry* asPtr(const OplogEntry* op) {
            return op;
        }
    };

    // Only retryable internal transactions need to deal with statement IDs.
    if (isInternalSessionForRetryableWrite(lsid)) {
        std::vector<StmtId> committedStmtIds;
        for (const auto& op : txnOps) {
            const auto& stmtIds = OpConverter::asPtr(op)->getStatementIds();
            committedStmtIds.insert(committedStmtIds.end(), stmtIds.begin(), stmtIds.end());
        }
        return committedStmtIds;
    }

    return boost::none;
}

// Apply the oplog entries for a prepare or a prepared commit during recovery/initial sync.
Status _applyOperationsForTransaction(OperationContext* opCtx,
                                      const std::vector<OplogEntry>& txnOps,
                                      repl::OplogApplication::Mode oplogApplicationMode) noexcept {
    // Apply each the operations via repl::applyOperation.
    for (const auto& op : txnOps) {
        try {
            if (op.getOpType() == repl::OpTypeEnum::kNoop) {
                continue;
            }

            // Presently, it is not allowed to run a prepared transaction with a command
            // inside. TODO(SERVER-46105)
            invariant(!op.isCommand());
            auto coll = acquireCollection(
                opCtx,
                CollectionAcquisitionRequest(op.getNss(),
                                             AcquisitionPrerequisites::kPretendUnsharded,
                                             repl::ReadConcernArgs::get(opCtx),
                                             AcquisitionPrerequisites::kWrite),
                MODE_IX);
            const bool isDataConsistent = true;
            auto status = repl::applyOperation_inlock(opCtx,
                                                      coll,
                                                      ApplierOperation{&op},
                                                      false /*alwaysUpsert*/,
                                                      oplogApplicationMode,
                                                      isDataConsistent);
            if (!status.isOK()) {
                return status;
            }
        } catch (const DBException& ex) {
            // Ignore NamespaceNotFound errors if we are in initial sync or recovering mode.
            // During recovery we reconsutuct prepared transactions at the end after applying all
            // the oplogs, so 'NamespaceNotFound' error shouldn't be hit whether it is a stable or
            // unstable recovery. However we have some scenarios when this error should be skipped:
            //  1- This code path can be called while applying commit oplog during unstable recovery
            //     when 'startupRecoveryForRestore' is set.
            //  2- During selective backup:
            //     - During restore when 'recoverFromOplogAsStandalone' is set which is usually be
            //       done in a stable recovery mode.
            //     - After the restore finished as the standalone node started with the flag
            //       'takeUnstableCheckpointOnShutdown' so after restarting the node as a replica
            //       set member it will go through unstable recovery.
            const bool ignoreException = ex.code() == ErrorCodes::NamespaceNotFound &&
                (oplogApplicationMode == repl::OplogApplication::Mode::kInitialSync ||
                 repl::OplogApplication::inRecovering(oplogApplicationMode));

            if (ex.code() == ErrorCodes::NamespaceNotFound &&
                oplogApplicationMode == repl::OplogApplication::Mode::kStableRecovering) {
                repl::OplogApplication::checkOnOplogFailureForRecovery(
                    opCtx, op.getNss(), redact(op.toBSONForLogging()), redact(ex));
            }

            if (!ignoreException) {
                LOGV2_DEBUG(
                    21845,
                    1,
                    "Error applying operation in transaction. {error}- oplog entry: {oplogEntry}",
                    "Error applying operation in transaction",
                    "error"_attr = redact(ex),
                    "oplogEntry"_attr = redact(op.toBSONForLogging()));
                return exceptionToStatus();
            }
            LOGV2_DEBUG(21846,
                        1,
                        "Encountered but ignoring error: {error} while applying operations for "
                        "transaction because we are either in initial "
                        "sync or recovering mode - oplog entry: {oplogEntry}",
                        "Encountered but ignoring error while applying operations for transaction "
                        "because we are either in initial sync or recovering mode",
                        "error"_attr = redact(ex),
                        "oplogEntry"_attr = redact(op.toBSONForLogging()),
                        "oplogApplicationMode"_attr =
                            repl::OplogApplication::modeToString(oplogApplicationMode));
        }
    }
    return Status::OK();
}

/**
 * Helper that will read the entire sequence of oplog entries for the transaction and apply each of
 * them.
 *
 * Currently used for oplog application of a commitTransaction oplog entry during recovery and
 * rollback.
 */
Status _applyTransactionFromOplogChain(OperationContext* opCtx,
                                       const OplogEntry& entry,
                                       repl::OplogApplication::Mode mode,
                                       Timestamp commitTimestamp,
                                       Timestamp durableTimestamp) {
    invariant(repl::OplogApplication::inRecovering(mode));

    auto ops = readTransactionOperationsFromOplogChain(opCtx, entry, {});

    const auto dbName = entry.getNss().dbName();
    Status status = Status::OK();

    writeConflictRetry(opCtx, "replaying prepared transaction", NamespaceString(dbName), [&] {
        WriteUnitOfWork wunit(opCtx);

        // We might replay a prepared transaction behind oldest timestamp.
        opCtx->recoveryUnit()->setRoundUpPreparedTimestamps(true);

        BSONObjBuilder resultWeDontCareAbout;

        status = _applyOperationsForTransaction(opCtx, ops, mode);
        if (status.isOK()) {
            // If the transaction was empty then we have no locks, ensure at least Global IX.
            Lock::GlobalLock lk(opCtx, MODE_IX);
            opCtx->recoveryUnit()->setPrepareTimestamp(commitTimestamp);
            wunit.prepare();

            // Calls setCommitTimestamp() to set commit timestamp of the transaction and
            // clears the commit timestamp in the recovery unit when tsBlock goes out of the
            // scope. It is necessary that we clear the commit timestamp because there can be
            // another transaction in the same recovery unit calling setTimestamp().
            TimestampBlock tsBlock(opCtx, commitTimestamp);
            opCtx->recoveryUnit()->setDurableTimestamp(durableTimestamp);
            wunit.commit();
        }
    });
    return status;
}

/**
 * This is the part of applyCommitTransaction which is common to both split and non-split commit
 * ops in secondary oplog application mode.
 */
Status _applyCommitTransaction(OperationContext* opCtx,
                               const OplogEntry& commitOp,
                               const LogicalSessionId& lsid,
                               TxnNumber txnNumber,
                               Timestamp commitTimestamp) {
    {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->setLogicalSessionId(lsid);
        opCtx->setTxnNumber(txnNumber);
        if (auto txnRetryCounter = commitOp.getOperationSessionInfo().getTxnRetryCounter()) {
            opCtx->setTxnRetryCounter(*txnRetryCounter);
        }
        opCtx->setInMultiDocumentTransaction();
    }
    // This opCtx can be used to apply later operations in the batch, clean up before reusing.
    ON_BLOCK_EXIT([opCtx]() {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->resetMultiDocumentTransactionState();
    });

    // The write on transaction table may be applied concurrently, so refreshing state
    // from disk may read that write, causing starting a new transaction on an existing
    // txnNumber. Thus, we start a new transaction without refreshing state from disk.
    auto mongoDSessionCatalog = MongoDSessionCatalog::get(opCtx);
    auto sessionCheckout = mongoDSessionCatalog->checkOutSessionWithoutRefresh(opCtx);

    auto txnParticipant = TransactionParticipant::get(opCtx);
    invariant(txnParticipant);
    txnParticipant.unstashTransactionResources(opCtx, "commitTransaction");
    txnParticipant.commitPreparedTransaction(opCtx, commitTimestamp, commitOp.getOpTime());

    return Status::OK();
}

/**
 * This is the part of applyAbortTransaction which is common to both split and non-split abort
 * ops in secondary oplog application mode.
 */
Status _applyAbortTransaction(OperationContext* opCtx,
                              const OplogEntry& abortOp,
                              const LogicalSessionId& lsid,
                              TxnNumber txnNumber) {
    {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->setLogicalSessionId(lsid);
        opCtx->setTxnNumber(txnNumber);
        if (auto txnRetryCounter = abortOp.getOperationSessionInfo().getTxnRetryCounter()) {
            opCtx->setTxnRetryCounter(*txnRetryCounter);
        }
        opCtx->setInMultiDocumentTransaction();
    }
    // This opCtx can be used to apply later operations in the batch, clean up before reusing.
    ON_BLOCK_EXIT([opCtx]() {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->resetMultiDocumentTransactionState();
    });

    // The write on transaction table may be applied concurrently, so refreshing state
    // from disk may read that write, causing starting a new transaction on an existing
    // txnNumber. Thus, we start a new transaction without refreshing state from disk.
    auto mongoDSessionCatalog = MongoDSessionCatalog::get(opCtx);
    auto sessionCheckout = mongoDSessionCatalog->checkOutSessionWithoutRefresh(opCtx);

    auto txnParticipant = TransactionParticipant::get(opCtx);
    invariant(txnParticipant);
    txnParticipant.unstashTransactionResources(opCtx, "abortTransaction");
    txnParticipant.abortTransaction(opCtx);

    return Status::OK();
}
}  // namespace

Status applyCommitTransaction(OperationContext* opCtx,
                              const ApplierOperation& op,
                              repl::OplogApplication::Mode mode) {
    IDLParserContext ctx("commitTransaction");
    auto commitCommand = CommitTransactionOplogObject::parse(ctx, op->getObject());
    auto commitTimestamp = *commitCommand.getCommitTimestamp();

    switch (mode) {
        case repl::OplogApplication::Mode::kUnstableRecovering:
        case repl::OplogApplication::Mode::kStableRecovering: {
            return _applyTransactionFromOplogChain(
                opCtx, *op, mode, commitTimestamp, op->getOpTime().getTimestamp());
        }
        case repl::OplogApplication::Mode::kInitialSync: {
            // Initial sync should never apply 'commitTransaction' since it unpacks committed
            // transactions onto various applier threads.
            MONGO_UNREACHABLE;
        }
        case repl::OplogApplication::Mode::kApplyOpsCmd: {
            // Return error if run via applyOps command.
            uasserted(50987, "commitTransaction is only used internally by secondaries.");
        }
        case repl::OplogApplication::Mode::kSecondary: {
            switch (op.instruction) {
                case repl::ApplicationInstruction::applyOplogEntry:
                case repl::ApplicationInstruction::applyTopLevelPreparedTxnOp: {
                    // Checkout the session and apply non-split or top-level commit op.
                    invariant(!op.subSession);
                    invariant(!op.preparedTxnOps);
                    return _applyCommitTransaction(
                        opCtx, *op, *op->getSessionId(), *op->getTxnNumber(), commitTimestamp);
                }
                case repl::ApplicationInstruction::applySplitPreparedTxnOp: {
                    // Checkout the session and apply split commit op.
                    invariant(op.subSession);
                    invariant(!op.preparedTxnOps);
                    return _applyCommitTransaction(opCtx,
                                                   *op,
                                                   (*op.subSession).getSessionId(),
                                                   (*op.subSession).getTxnNumber(),
                                                   commitTimestamp);
                }
            }
        }
    }
    MONGO_UNREACHABLE;
}

Status applyAbortTransaction(OperationContext* opCtx,
                             const ApplierOperation& op,
                             repl::OplogApplication::Mode mode) {
    switch (mode) {
        case repl::OplogApplication::Mode::kUnstableRecovering:
        case repl::OplogApplication::Mode::kStableRecovering: {
            // We don't put transactions into the prepare state until the end of recovery,
            // so there is no transaction to abort.
            return Status::OK();
        }
        case repl::OplogApplication::Mode::kInitialSync: {
            // We don't put transactions into the prepare state until the end of initial sync,
            // so there is no transaction to abort.
            return Status::OK();
        }
        case repl::OplogApplication::Mode::kApplyOpsCmd: {
            // Return error if run via applyOps command.
            uasserted(50972, "abortTransaction is only used internally by secondaries.");
        }
        case repl::OplogApplication::Mode::kSecondary: {
            switch (op.instruction) {
                case repl::ApplicationInstruction::applyOplogEntry:
                case repl::ApplicationInstruction::applyTopLevelPreparedTxnOp: {
                    // Checkout the session and apply non-split or top-level abort op.
                    invariant(!op.subSession);
                    invariant(!op.preparedTxnOps);
                    return _applyAbortTransaction(
                        opCtx, *op, *op->getSessionId(), *op->getTxnNumber());
                }
                case repl::ApplicationInstruction::applySplitPreparedTxnOp: {
                    // Checkout the session and apply split abort op.
                    invariant(op.subSession);
                    invariant(!op.preparedTxnOps);
                    return _applyAbortTransaction(opCtx,
                                                  *op,
                                                  (*op.subSession).getSessionId(),
                                                  (*op.subSession).getTxnNumber());
                }
            }
        }
    }
    MONGO_UNREACHABLE;
}

std::pair<std::vector<OplogEntry>, bool> _readTransactionOperationsFromOplogChain(
    OperationContext* opCtx,
    const OplogEntry& lastEntryInTxn,
    const std::vector<OplogEntry*>& cachedOps,
    const bool checkForCommands) noexcept {
    bool isTransactionWithCommand = false;
    // Ensure future transactions read without a timestamp.
    invariant(RecoveryUnit::ReadSource::kNoTimestamp ==
              opCtx->recoveryUnit()->getTimestampReadSource());

    std::vector<OplogEntry> ops;

    // The cachedOps are the ops for this transaction that are from the same oplog application batch
    // as the commit or prepare, those which have not necessarily been written to the oplog.  These
    // ops are in order of increasing timestamp.
    const auto oldestEntryInBatch = cachedOps.empty() ? lastEntryInTxn : *cachedOps.front();

    // The lastEntryWrittenToOplogOpTime is the OpTime of the latest entry for this
    // transaction which is expected to be present in the oplog.  It is the entry
    // before the first cachedOp, unless there are no cachedOps in which case it is
    // the entry before the commit or prepare.
    const auto lastEntryWrittenToOplogOpTime = oldestEntryInBatch.getPrevWriteOpTimeInTransaction();
    invariant(lastEntryWrittenToOplogOpTime < lastEntryInTxn.getOpTime());

    TransactionHistoryIterator iter(lastEntryWrittenToOplogOpTime.value());

    // If we started with a prepared commit, we want to forget about that operation
    // and move onto the prepare.
    auto prepareOrUnpreparedCommit = lastEntryInTxn;
    if (lastEntryInTxn.isPreparedCommit()) {
        // A prepared-commit must be in its own batch and thus have no cached ops.
        invariant(cachedOps.empty());
        invariant(iter.hasNext());
        prepareOrUnpreparedCommit = iter.nextFatalOnErrors(opCtx);
    }
    invariant(prepareOrUnpreparedCommit.getCommandType() == OplogEntry::CommandType::kApplyOps);

    // The non-DurableReplOperation fields of the extracted transaction operations
    // will match those of the lastEntryInTxn. For a prepared commit, this will
    // include the commit oplog entry's 'ts' field, which is what we want.
    auto lastEntryInTxnObj = lastEntryInTxn.getEntry().toBSON();

    // First retrieve and transform the ops from the oplog, which will be retrieved
    // in reverse order.
    while (iter.hasNext()) {
        const auto& operationEntry = iter.nextFatalOnErrors(opCtx);
        invariant(operationEntry.isPartialTransaction());
        auto prevOpsEnd = ops.size();
        repl::ApplyOps::extractOperationsTo(operationEntry, lastEntryInTxnObj, &ops);

        // Because BSONArrays do not have fast way of determining size without
        // iterating through them, and we also have no way of knowing how many oplog
        // entries are in a transaction without iterating, reversing each applyOps
        // and then reversing the whole array is about as good as we can do to get
        // the entire thing in chronological order.  Fortunately STL arrays of BSON
        // objects should be fast to reverse (just pointer copies).
        std::reverse(ops.begin() + prevOpsEnd, ops.end());
    }
    std::reverse(ops.begin(), ops.end());

    // Next retrieve and transform the ops from the current batch, which are in
    // increasing timestamp order.
    for (auto* cachedOp : cachedOps) {
        const auto& operationEntry = *cachedOp;
        invariant(operationEntry.isPartialTransaction());
        repl::ApplyOps::extractOperationsTo(operationEntry, lastEntryInTxnObj, &ops);
    }

    // Reconstruct the operations from the prepare or unprepared commit oplog entry.
    repl::ApplyOps::extractOperationsTo(prepareOrUnpreparedCommit, lastEntryInTxnObj, &ops);

    // It is safe to assume that any commands inside `ops` are real commands to be
    // applied, as opposed to auxiliary commands such as "commit" and "abort".
    if (checkForCommands) {
        for (auto&& op : ops) {
            if (op.isCommand()) {
                isTransactionWithCommand = true;
                break;
            }
        }
    }
    return {std::move(ops), isTransactionWithCommand};
}

std::vector<OplogEntry> readTransactionOperationsFromOplogChain(
    OperationContext* opCtx,
    const OplogEntry& lastEntryInTxn,
    const std::vector<OplogEntry*>& cachedOps) noexcept {
    auto [txnOps, _] = _readTransactionOperationsFromOplogChain(
        opCtx, lastEntryInTxn, cachedOps, false /*checkForCommands*/);
    return std::move(txnOps);
}

std::pair<std::vector<OplogEntry>, bool> readTransactionOperationsFromOplogChainAndCheckForCommands(
    OperationContext* opCtx,
    const OplogEntry& lastEntryInTxn,
    const std::vector<OplogEntry*>& cachedOps) noexcept {
    return _readTransactionOperationsFromOplogChain(
        opCtx, lastEntryInTxn, cachedOps, true /*checkForCommands*/);
}

namespace {
/**
 * This is the part of applyPrepareTransaction which is common to steady state, initial
 * sync and recovery oplog application.
 *
 * Note: when this is called to apply a split prepared transaction, the txnOps here represents
 * a subset of all the ops in the transaction, but when being called by a top-level prepared
 * transaction, it's just an empty array. Future changes that depend on transaction operations
 * should be careful about the differences.
 */
Status _applyPrepareTransaction(OperationContext* opCtx,
                                const OplogEntry& prepareOp,
                                const LogicalSessionId& lsid,
                                TxnNumber txnNumber,
                                const std::vector<OplogEntry>& txnOps,
                                repl::OplogApplication::Mode mode,
                                boost::optional<std::vector<StmtId>> stmtIds = boost::none) {

    // Block application of prepare oplog entries on secondaries when a concurrent
    // background index build is running. This will prevent hybrid index builds from
    // corrupting an index on secondary nodes if a prepared transaction becomes prepared
    // during a build but commits after the index build commits. When two-phase index
    // builds are in use, this is both unnecessary and unsafe. Due to locking, we can
    // guarantee that a transaction prepared on a primary during an index build will
    // always commit before that index build completes. Because two-phase index builds
    // replicate start and commit oplog entries, it will never be possible to replicate
    // a prepared transaction, commit an index build, then commit the transaction, the
    // bug described above. This blocking behavior can also introduce a deadlock with
    // two-phase index builds on a secondary if a prepared transaction blocks on an
    // index build, but the index build can't re-acquire its X lock because of the
    // transaction.
    for (const auto& op : txnOps) {
        if (op.getOpType() == repl::OpTypeEnum::kNoop) {
            continue;
        }
        auto indexBuildsCoord = IndexBuildsCoordinator::get(opCtx);
        auto ns = op.getNss();
        auto uuid = *op.getUuid();
        if (indexBuildsCoord->inProgForCollection(uuid, IndexBuildProtocol::kSinglePhase)) {
            LOGV2_WARNING(21849,
                          "Blocking replication until single-phase index builds are finished on "
                          "collection, due to prepared transaction",
                          "namespace"_attr = redact(toStringForLogging(ns)),
                          "uuid"_attr = uuid);
            indexBuildsCoord->awaitNoIndexBuildInProgressForCollection(
                opCtx, uuid, IndexBuildProtocol::kSinglePhase);
        }
    }

    {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->setLogicalSessionId(lsid);
        opCtx->setTxnNumber(txnNumber);
        if (auto txnRetryCounter = prepareOp.getOperationSessionInfo().getTxnRetryCounter()) {
            opCtx->setTxnRetryCounter(*txnRetryCounter);
        }
        opCtx->setInMultiDocumentTransaction();
    }
    // This opCtx can be used to apply later operations in the batch, clean up before
    // reusing.
    ON_BLOCK_EXIT([opCtx]() {
        stdx::lock_guard<Client> lk(*opCtx->getClient());
        opCtx->resetMultiDocumentTransactionState();
    });

    return writeConflictRetry(opCtx, "applying prepare transaction", prepareOp.getNss(), [&] {
        // The write on transaction table may be applied concurrently, so refreshing
        // state from disk may read that write, causing starting a new transaction
        // on an existing txnNumber. Thus, we start a new transaction without
        // refreshing state from disk.
        hangBeforeSessionCheckOutForApplyPrepare.pauseWhileSet();
        auto mongoDSessionCatalog = MongoDSessionCatalog::get(opCtx);
        auto sessionCheckout = mongoDSessionCatalog->checkOutSessionWithoutRefresh(opCtx);
        auto txnParticipant = TransactionParticipant::get(opCtx);

        // We reset the recovery unit on retries, so make sure that we set the
        // necessary states.

        // When querying indexes, we return the record matching the key if it exists,
        // or an adjacent document. This means that it is possible for us to hit a
        // prepare conflict if we query for an incomplete key and an adjacent key is
        // prepared. We ignore prepare conflicts on recovering nodes because they may
        // may encounter prepare conflicts that did not occur on the primary.
        opCtx->recoveryUnit()->setPrepareConflictBehavior(
            PrepareConflictBehavior::kIgnoreConflictsAllowWrites);
        // We might replay a prepared transaction behind oldest timestamp.
        if (repl::OplogApplication::inRecovering(mode) ||
            mode == repl::OplogApplication::Mode::kInitialSync) {
            opCtx->recoveryUnit()->setRoundUpPreparedTimestamps(true);
        }

        // Release WUOW, transaction lock resources and abort storage transaction
        // so that the writeConflictRetry loop will be able to retry applying the
        // transactional ops on WCE error.
        ScopeGuard abortOnError([&txnParticipant, opCtx] {
            // Abort transaction and invalidate the session it is associated with.
            txnParticipant.abortTransaction(opCtx);
            txnParticipant.invalidate(opCtx);
        });

        // Starts the WUOW.
        txnParticipant.unstashTransactionResources(opCtx, "prepareTransaction");

        // Set this in case the application of any ops needs to use the prepare timestamp
        // of this transaction. It should be cleared automatically when the txn finishes.
        if (repl::OplogApplication::inRecovering(mode) ||
            mode == repl::OplogApplication::Mode::kInitialSync) {
            txnParticipant.setPrepareOpTimeForRecovery(opCtx, prepareOp.getOpTime());
        }

        auto status = _applyOperationsForTransaction(opCtx, txnOps, mode);

        // Add committed statement IDs if this is a retryable internal transaction.
        // They are used when this node becomes primary to avoid re-executing
        // committed txn statements.
        const auto& committedStmtIds = stmtIds ? stmtIds : _getCommittedStmtIds(lsid, txnOps);
        if (committedStmtIds) {
            txnParticipant.addCommittedStmtIds(opCtx, *committedStmtIds, prepareOp.getOpTime());
        }

        if (MONGO_unlikely(applyPrepareTxnOpsFailsWithWriteConflict.shouldFail())) {
            LOGV2(4947101, "Hit applyPrepareTxnOpsFailsWithWriteConflict failpoint");
            status = Status(ErrorCodes::WriteConflict,
                            "Prepare transaction apply ops failed due to write conflict");
        }

        if (status == ErrorCodes::WriteConflict) {
            throwWriteConflictException(
                "Conflict encountered when applying a prepare transaction.");
        }
        fassert(31137, status);

        if (MONGO_unlikely(applyOpsHangBeforePreparingTransaction.shouldFail())) {
            LOGV2(21847, "Hit applyOpsHangBeforePreparingTransaction failpoint");
            applyOpsHangBeforePreparingTransaction.pauseWhileSet(opCtx);
        }

        txnParticipant.prepareTransaction(opCtx, prepareOp.getOpTime());

        auto opObserver = opCtx->getServiceContext()->getOpObserver();
        invariant(opObserver);
        opObserver->onTransactionPrepareNonPrimary(
            opCtx, *prepareOp.getSessionId(), txnOps, prepareOp.getOpTime());

        // Prepare transaction success.
        abortOnError.dismiss();

        txnParticipant.stashTransactionResources(opCtx);
        return Status::OK();
    });
}

/**
 * Apply a prepared transaction when we are reconstructing prepared transactions.
 */
void _reconstructPreparedTransaction(OperationContext* opCtx,
                                     const OplogEntry& prepareOp,
                                     repl::OplogApplication::Mode mode) {
    repl::UnreplicatedWritesBlock uwb(opCtx);

    // Snapshot transaction can never conflict with the PBWM lock.
    opCtx->lockState()->setShouldConflictWithSecondaryBatchApplication(false);

    // The operations here are reconstructed at their prepare time. However, that time
    // will be ignored because there is an outer write unit of work during their
    // application. The prepare time of the transaction is set explicitly below.
    auto ops = readTransactionOperationsFromOplogChain(opCtx, prepareOp, {});

    // Checks out the session, applies the operations and prepares the transaction.
    uassertStatusOK(_applyPrepareTransaction(
        opCtx, prepareOp, *prepareOp.getSessionId(), *prepareOp.getTxnNumber(), ops, mode));
}
}  // namespace

/**
 * Make sure that if we are in replication recovery, we don't apply the prepare
 * transaction oplog entry until we either see a commit transaction oplog entry or are
 * at the very end of recovery. Otherwise, only apply the prepare transaction oplog
 * entry if we are a secondary. We shouldn't get here for initial sync and applyOps
 * should error.
 */
Status applyPrepareTransaction(OperationContext* opCtx,
                               const ApplierOperation& op,
                               repl::OplogApplication::Mode mode) {
    switch (mode) {
        case repl::OplogApplication::Mode::kUnstableRecovering:
        case repl::OplogApplication::Mode::kStableRecovering: {
            if (!serverGlobalParams.enableMajorityReadConcern) {
                LOGV2_ERROR(21850,
                            "Cannot replay a prepared transaction when "
                            "'enableMajorityReadConcern' is "
                            "set to false. Restart the server with "
                            "--enableMajorityReadConcern=true "
                            "to complete recovery");
                fassertFailed(51146);
            }

            // Don't apply the operations from the prepared transaction until either we
            // see a commit transaction oplog entry during recovery or are at the end of
            // recovery.
            return Status::OK();
        }
        case repl::OplogApplication::Mode::kInitialSync: {
            // Initial sync should never apply 'prepareTransaction' since it unpacks
            // committed transactions onto various applier threads at commit time.
            MONGO_UNREACHABLE;
        }
        case repl::OplogApplication::Mode::kApplyOpsCmd: {
            // Return error if run via applyOps command.
            uasserted(51145,
                      "prepare applyOps oplog entry is only used internally by secondaries.");
        }
        case repl::OplogApplication::Mode::kSecondary: {
            switch (op.instruction) {
                case repl::ApplicationInstruction::applyOplogEntry: {
                    // Checkout the session and apply non-split prepare op.
                    // TODO (SERVER-70578): This can no longer happen once the feature flag
                    // is removed.
                    invariant(!op.subSession);
                    invariant(!op.preparedTxnOps);
                    auto ops = readTransactionOperationsFromOplogChain(opCtx, *op, {});
                    return _applyPrepareTransaction(
                        opCtx, *op, *op->getSessionId(), *op->getTxnNumber(), ops, mode);
                }
                case repl::ApplicationInstruction::applySplitPreparedTxnOp: {
                    // Checkout the session and apply split prepare op.
                    invariant(op.subSession);
                    invariant(op.preparedTxnOps);
                    return _applyPrepareTransaction(opCtx,
                                                    *op,
                                                    (*op.subSession).getSessionId(),
                                                    (*op.subSession).getTxnNumber(),
                                                    _copyOps(*op.preparedTxnOps),
                                                    repl::OplogApplication::Mode::kSecondary);
                }
                case repl::ApplicationInstruction::applyTopLevelPreparedTxnOp: {
                    // Checkout the session and apply top-level prepare op.
                    invariant(!op.subSession);
                    invariant(op.preparedTxnOps);
                    const auto& txnOps = *op.preparedTxnOps;
                    // For a top-level transaction, the actual transaction operations should've
                    // already been applied by its split transactions. So here we just pass it
                    // an empty array of transaction operations. However if this is a retryable
                    // internal transaction, we need to pass the committed statement IDs.
                    auto stmtIds = _getCommittedStmtIds(*op->getSessionId(), txnOps);
                    return _applyPrepareTransaction(opCtx,
                                                    *op,
                                                    *op->getSessionId(),
                                                    *op->getTxnNumber(),
                                                    {},
                                                    mode,
                                                    std::move(stmtIds));
                }
            }
        }
    }
    MONGO_UNREACHABLE;
}

void reconstructPreparedTransactions(OperationContext* opCtx, repl::OplogApplication::Mode mode) {
    if (MONGO_unlikely(skipReconstructPreparedTransactions.shouldFail())) {
        LOGV2(21848, "Hit skipReconstructPreparedTransactions failpoint");
        return;
    }

    // Ensure future transactions read without a timestamp.
    invariant(RecoveryUnit::ReadSource::kNoTimestamp ==
              opCtx->recoveryUnit()->getTimestampReadSource());

    DBDirectClient client(opCtx);
    FindCommandRequest findRequest{NamespaceString::kSessionTransactionsTableNamespace};
    findRequest.setFilter(BSON("state"
                               << "prepared"));
    const auto cursor = client.find(std::move(findRequest));

    // Iterate over each entry in the transactions table that has a prepared
    // transaction.
    while (cursor->more()) {
        const auto txnRecordObj = cursor->next();
        const auto txnRecord = SessionTxnRecord::parse(
            IDLParserContext("recovering prepared transaction"), txnRecordObj);

        invariant(txnRecord.getState() == DurableTxnStateEnum::kPrepared);

        // Get the prepareTransaction oplog entry corresponding to this transactions
        // table entry.
        const auto prepareOpTime = txnRecord.getLastWriteOpTime();
        invariant(!prepareOpTime.isNull());
        TransactionHistoryIterator iter(prepareOpTime);
        invariant(iter.hasNext());
        auto prepareOplogEntry = iter.nextFatalOnErrors(opCtx);

        {
            // Make a new opCtx so that we can set the lsid when applying the prepare
            // transaction oplog entry.
            auto newClient =
                opCtx->getServiceContext()->makeClient("reconstruct-prepared-transactions");

            // TODO(SERVER-74656): Please revisit if this thread could be made killable.
            {
                stdx::lock_guard<Client> lk(*newClient.get());
                newClient.get()->setSystemOperationUnkillableByStepdown(lk);
            }

            AlternativeClientRegion acr(newClient);
            const auto newOpCtx = cc().makeOperationContext();

            _reconstructPreparedTransaction(newOpCtx.get(), prepareOplogEntry, mode);
        }
    }
}
}  // namespace mongo