diff options
author | William Schultz <william.schultz@mongodb.com> | 2019-05-22 20:31:37 -0400 |
---|---|---|
committer | William Schultz <william.schultz@mongodb.com> | 2019-05-22 20:31:37 -0400 |
commit | 853f03613e329c391881338b3314a52d2abcf948 (patch) | |
tree | cae10f01076f1bef1ee561c59f457fbb91827d06 /src | |
parent | 8cf664042e2cb74c12beaf37ef1a3fbe12d90b73 (diff) | |
download | mongo-853f03613e329c391881338b3314a52d2abcf948.tar.gz |
SERVER-39804 Extend idempotency unit tests to cover the new large transaction format
Diffstat (limited to 'src')
-rw-r--r-- | src/mongo/db/repl/idempotency_test_fixture.cpp | 30 | ||||
-rw-r--r-- | src/mongo/db/repl/idempotency_test_fixture.h | 11 | ||||
-rw-r--r-- | src/mongo/db/repl/sync_tail_test.cpp | 343 |
3 files changed, 373 insertions, 11 deletions
diff --git a/src/mongo/db/repl/idempotency_test_fixture.cpp b/src/mongo/db/repl/idempotency_test_fixture.cpp index 949627960dd..3a84112a1ad 100644 --- a/src/mongo/db/repl/idempotency_test_fixture.cpp +++ b/src/mongo/db/repl/idempotency_test_fixture.cpp @@ -467,7 +467,8 @@ OplogEntry IdempotencyTest::dropIndex(const std::string& indexName, const UUID& OplogEntry IdempotencyTest::prepare(LogicalSessionId lsid, TxnNumber txnNum, StmtId stmtId, - const BSONArray& ops) { + const BSONArray& ops, + OpTime prevOpTime) { OperationSessionInfo info; info.setSessionId(lsid); info.setTxnNumber(txnNum); @@ -480,18 +481,19 @@ OplogEntry IdempotencyTest::prepare(LogicalSessionId lsid, Date_t::min() /* wallClockTime -- required but not checked */, stmtId, boost::none /* uuid */, - OpTime()); + prevOpTime); } OplogEntry IdempotencyTest::commitUnprepared(LogicalSessionId lsid, TxnNumber txnNum, StmtId stmtId, - const BSONArray& ops) { + const BSONArray& ops, + OpTime prevOpTime) { OperationSessionInfo info; info.setSessionId(lsid); info.setTxnNumber(txnNum); return makeCommandOplogEntryWithSessionInfoAndStmtId( - nextOpTime(), nss, BSON("applyOps" << ops), lsid, txnNum, stmtId, OpTime()); + nextOpTime(), nss, BSON("applyOps" << ops), lsid, txnNum, stmtId, prevOpTime); } OplogEntry IdempotencyTest::commitPrepared(LogicalSessionId lsid, @@ -516,6 +518,26 @@ OplogEntry IdempotencyTest::abortPrepared(LogicalSessionId lsid, nextOpTime(), nss, BSON("abortTransaction" << 1), lsid, txnNum, stmtId, prepareOpTime); } +OplogEntry IdempotencyTest::partialTxn(LogicalSessionId lsid, + TxnNumber txnNum, + StmtId stmtId, + OpTime prevOpTime, + const BSONArray& ops) { + OperationSessionInfo info; + info.setSessionId(lsid); + info.setTxnNumber(txnNum); + return makeOplogEntry(nextOpTime(), + OpTypeEnum::kCommand, + nss.getCommandNS(), + BSON("applyOps" << ops << "partialTxn" << true), + boost::none /* o2 */, + info /* sessionInfo */, + Date_t::min() /* wallClockTime -- required but not checked */, + stmtId, + boost::none /* uuid */, + prevOpTime); +} + std::string IdempotencyTest::computeDataHash(Collection* collection) { auto desc = collection->getIndexCatalog()->findIdIndex(_opCtx.get()); ASSERT_TRUE(desc); diff --git a/src/mongo/db/repl/idempotency_test_fixture.h b/src/mongo/db/repl/idempotency_test_fixture.h index 325effc7e22..82eb4215b59 100644 --- a/src/mongo/db/repl/idempotency_test_fixture.h +++ b/src/mongo/db/repl/idempotency_test_fixture.h @@ -100,11 +100,13 @@ protected: OplogEntry prepare(LogicalSessionId lsid, TxnNumber txnNum, StmtId stmtId, - const BSONArray& ops); + const BSONArray& ops, + OpTime prevOpTime = OpTime()); OplogEntry commitUnprepared(LogicalSessionId lsid, TxnNumber txnNum, StmtId stmtId, - const BSONArray& ops); + const BSONArray& ops, + OpTime prevOpTime = OpTime()); OplogEntry commitPrepared(LogicalSessionId lsid, TxnNumber txnNum, StmtId stmtId, @@ -113,6 +115,11 @@ protected: TxnNumber txnNum, StmtId stmtId, OpTime prepareOpTime); + OplogEntry partialTxn(LogicalSessionId lsid, + TxnNumber txnNum, + StmtId stmtId, + OpTime prevOpTime, + const BSONArray& ops); virtual Status resetState(); /** diff --git a/src/mongo/db/repl/sync_tail_test.cpp b/src/mongo/db/repl/sync_tail_test.cpp index cd063989a04..7a366c16971 100644 --- a/src/mongo/db/repl/sync_tail_test.cpp +++ b/src/mongo/db/repl/sync_tail_test.cpp @@ -3013,10 +3013,13 @@ TEST_F(IdempotencyTest, ConvertToCappedNamespaceNotFound) { ASSERT_FALSE(autoColl.getDb()); } +class IdempotencyTestTxns : public IdempotencyTest {}; + // Document used by transaction idempotency tests. const BSONObj doc = fromjson("{_id: 1}"); +const BSONObj doc2 = fromjson("{_id: 2}"); -TEST_F(IdempotencyTest, CommitUnpreparedTransaction) { +TEST_F(IdempotencyTestTxns, CommitUnpreparedTransaction) { createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto lsid = makeLogicalSessionId(_opCtx.get()); @@ -3039,7 +3042,7 @@ TEST_F(IdempotencyTest, CommitUnpreparedTransaction) { ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); } -TEST_F(IdempotencyTest, CommitUnpreparedTransactionDataPartiallyApplied) { +TEST_F(IdempotencyTestTxns, CommitUnpreparedTransactionDataPartiallyApplied) { createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto lsid = makeLogicalSessionId(_opCtx.get()); @@ -3078,7 +3081,7 @@ TEST_F(IdempotencyTest, CommitUnpreparedTransactionDataPartiallyApplied) { ASSERT_TRUE(docExists(_opCtx.get(), nss2, doc)); } -TEST_F(IdempotencyTest, CommitPreparedTransaction) { +TEST_F(IdempotencyTestTxns, CommitPreparedTransaction) { createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto lsid = makeLogicalSessionId(_opCtx.get()); @@ -3103,7 +3106,7 @@ TEST_F(IdempotencyTest, CommitPreparedTransaction) { ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); } -TEST_F(IdempotencyTest, CommitPreparedTransactionDataPartiallyApplied) { +TEST_F(IdempotencyTestTxns, CommitPreparedTransactionDataPartiallyApplied) { createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto lsid = makeLogicalSessionId(_opCtx.get()); @@ -3144,7 +3147,7 @@ TEST_F(IdempotencyTest, CommitPreparedTransactionDataPartiallyApplied) { ASSERT_TRUE(docExists(_opCtx.get(), nss2, doc)); } -TEST_F(IdempotencyTest, AbortPreparedTransaction) { +TEST_F(IdempotencyTestTxns, AbortPreparedTransaction) { createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); auto uuid = createCollectionWithUuid(_opCtx.get(), nss); auto lsid = makeLogicalSessionId(_opCtx.get()); @@ -3167,6 +3170,336 @@ TEST_F(IdempotencyTest, AbortPreparedTransaction) { DurableTxnStateEnum::kAborted); ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); } + +TEST_F(IdempotencyTestTxns, SinglePartialTxnOp) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp}); + auto expectedStartOpTime = partialOp.getOpTime(); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + partialOp.getOpTime(), + *partialOp.getWallClockTime(), + expectedStartOpTime, + DurableTxnStateEnum::kInProgress); + + // Document should not be visible yet. + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); +} + +TEST_F(IdempotencyTestTxns, MultiplePartialTxnOps) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp1 = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto partialOp2 = partialTxn(lsid, + txnNum, + StmtId(1), + partialOp1.getOpTime(), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2))); + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp1, partialOp2}); + auto expectedStartOpTime = partialOp1.getOpTime(); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + partialOp1.getOpTime(), + *partialOp1.getWallClockTime(), + expectedStartOpTime, + DurableTxnStateEnum::kInProgress); + // Document should not be visible yet. + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, CommitUnpreparedTransactionWithPartialTxnOps) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + + auto commitOp = commitUnprepared(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, commitOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + commitOp.getOpTime(), + *commitOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kCommitted); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, CommitUnpreparedTransactionWithPartialTxnOpsAndDataPartiallyApplied) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + + auto commitOp = commitUnprepared(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + + // Manually insert the first document so that the data will partially reflect the transaction + // when the commitTransaction oplog entry is applied during initial sync. This simulates the + // case where the transaction committed on the sync source at a point during the initial sync, + // such that we cloned 'doc' but missed 'doc2'. + ASSERT_OK(getStorageInterface()->insertDocument(_opCtx.get(), + nss, + {doc, commitOp.getOpTime().getTimestamp()}, + commitOp.getOpTime().getTerm())); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, commitOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + commitOp.getOpTime(), + *commitOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kCommitted); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, PrepareTransactionWithPartialTxnOps) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto prepareOp = prepare(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, prepareOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + prepareOp.getOpTime(), + *prepareOp.getWallClockTime(), + partialOp.getOpTime(), + DurableTxnStateEnum::kPrepared); + // Document should not be visible yet. + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); +} + +TEST_F(IdempotencyTestTxns, EmptyPrepareTransaction) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + // It is possible to have an empty prepare oplog entry. + auto prepareOp = prepare(lsid, txnNum, StmtId(1), BSONArray(), OpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({prepareOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + prepareOp.getOpTime(), + *prepareOp.getWallClockTime(), + prepareOp.getOpTime(), + DurableTxnStateEnum::kPrepared); +} + +TEST_F(IdempotencyTestTxns, CommitPreparedTransactionWithPartialTxnOps) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto prepareOp = prepare(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + auto commitOp = commitPrepared(lsid, txnNum, StmtId(2), prepareOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, prepareOp, commitOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + commitOp.getOpTime(), + *commitOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kCommitted); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, CommitPreparedTransactionWithPartialTxnOpsAndDataPartiallyApplied) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto prepareOp = prepare(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + auto commitOp = commitPrepared(lsid, txnNum, StmtId(2), prepareOp.getOpTime()); + + // Manually insert the first document so that the data will partially reflect the transaction + // when the commitTransaction oplog entry is applied during initial sync. This simulates the + // case where the transaction committed on the sync source at a point during the initial sync, + // such that we cloned 'doc' but missed 'doc2'. + ASSERT_OK(getStorageInterface()->insertDocument(_opCtx.get(), + nss, + {doc, commitOp.getOpTime().getTimestamp()}, + commitOp.getOpTime().getTerm())); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, prepareOp, commitOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + commitOp.getOpTime(), + *commitOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kCommitted); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc)); + ASSERT_TRUE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, AbortPreparedTransactionWithPartialTxnOps) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto prepareOp = prepare(lsid, + txnNum, + StmtId(1), + BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc2)), + partialOp.getOpTime()); + auto abortOp = abortPrepared(lsid, txnNum, StmtId(2), prepareOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, prepareOp, abortOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + abortOp.getOpTime(), + *abortOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kAborted); + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc2)); +} + +TEST_F(IdempotencyTestTxns, AbortInProgressTransaction) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + auto uuid = createCollectionWithUuid(_opCtx.get(), nss); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto partialOp = partialTxn( + lsid, txnNum, StmtId(0), OpTime(), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc))); + auto abortOp = abortPrepared(lsid, txnNum, StmtId(1), partialOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + testOpsAreIdempotent({partialOp, abortOp}); + repl::checkTxnTable(_opCtx.get(), + lsid, + txnNum, + abortOp.getOpTime(), + *abortOp.getWallClockTime(), + boost::none, + DurableTxnStateEnum::kAborted); + ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); +} + +TEST_F(IdempotencyTestTxns, CommitPreparedTransactionIgnoresNamespaceNotFoundErrors) { + createCollectionWithUuid(_opCtx.get(), NamespaceString::kSessionTransactionsTableNamespace); + + // Instead of creating a collection, we generate an arbitrary UUID to use for the operations + // below. This simulates the case where, during initial sync, a document D was inserted into a + // collection C on the sync source and then collection C was dropped, after we started fetching + // oplog entries but before we started collection cloning. In this case, we would not clone + // collection C, but when we try to apply the insertion of document D after collection cloning + // has finished, the collection would not exist since we never created it. It is acceptable to + // ignore the NamespaceNotFound error in this case since we know the collection will be dropped + // later on. + auto uuid = UUID::gen(); + auto lsid = makeLogicalSessionId(_opCtx.get()); + TxnNumber txnNum(0); + + auto prepareOp = prepare( + lsid, txnNum, StmtId(0), BSON_ARRAY(makeInsertApplyOpsEntry(nss, uuid, doc)), OpTime()); + auto commitOp = commitPrepared(lsid, txnNum, StmtId(1), prepareOp.getOpTime()); + + ASSERT_OK(ReplicationCoordinator::get(getGlobalServiceContext()) + ->setFollowerMode(MemberState::RS_RECOVERING)); + + // TODO (SERVER-41113): Enable these assertions once this ticket is complete. + // ASSERT_OK(runOpsInitialSync({prepareOp, commitOp})); + + // The op should have thrown a NamespaceNotFound error, which should have been ignored, so the + // operation has no effect. + // ASSERT_FALSE(docExists(_opCtx.get(), nss, doc)); +} + + } // namespace } // namespace repl } // namespace mongo |