diff options
author | Lamont Nelson <lamont.nelson@mongodb.com> | 2019-06-20 19:20:24 -0400 |
---|---|---|
committer | Lamont Nelson <lamont.nelson@mongodb.com> | 2019-06-24 15:51:45 -0400 |
commit | 18ad465f76bd339e8a902d13bf2dc74171d6e2f1 (patch) | |
tree | 14f6dc8cd1491a3150f5e62e55ce59b5f1ae3be1 /src | |
parent | 5c5832b459baa56b6bee172ee13fca97518436e2 (diff) | |
download | mongo-18ad465f76bd339e8a902d13bf2dc74171d6e2f1.tar.gz |
SERVER-41572 use FailPointEnableBlock to control failpoints in transaction_coordinator_test
Diffstat (limited to 'src')
-rw-r--r-- | src/mongo/db/s/transaction_coordinator_test.cpp | 118 | ||||
-rw-r--r-- | src/mongo/util/fail_point_service.cpp | 14 | ||||
-rw-r--r-- | src/mongo/util/fail_point_service.h | 2 |
3 files changed, 61 insertions, 73 deletions
diff --git a/src/mongo/db/s/transaction_coordinator_test.cpp b/src/mongo/db/s/transaction_coordinator_test.cpp index 215e315fc97..5bda5f4cfad 100644 --- a/src/mongo/db/s/transaction_coordinator_test.cpp +++ b/src/mongo/db/s/transaction_coordinator_test.cpp @@ -1574,9 +1574,7 @@ TEST_F(TransactionCoordinatorMetricsTest, expectedMetrics.totalStartedTwoPhaseCommit++; expectedMetrics.currentWritingParticipantList++; - setGlobalFailPoint("hangBeforeWaitingForParticipantListWriteConcern", - BSON("mode" - << "alwaysOn")); + FailPointEnableBlock fp("hangBeforeWaitingForParticipantListWriteConcern"); coordinator.runCommit(kTwoShardIdList); waitUntilCoordinatorDocIsPresent(); @@ -1600,11 +1598,6 @@ TEST_F(TransactionCoordinatorMetricsTest, // Slow log line is not logged since the coordination did not complete successfully. stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("two-phase commit parameters:")); - - // Clear the failpoint before the next test. - setGlobalFailPoint("hangBeforeWaitingForParticipantListWriteConcern", - BSON("mode" - << "off")); } TEST_F(TransactionCoordinatorMetricsTest, @@ -1703,9 +1696,8 @@ TEST_F(TransactionCoordinatorMetricsTest, expectedMetrics.totalStartedTwoPhaseCommit++; expectedMetrics.currentWritingDecision++; - setGlobalFailPoint("hangBeforeWaitingForDecisionWriteConcern", - BSON("mode" - << "alwaysOn")); + FailPointEnableBlock fp("hangBeforeWaitingForDecisionWriteConcern"); + coordinator.runCommit(kTwoShardIdList); // Respond to the second prepare request in a separate thread, because the coordinator will // hijack that thread to run its continuation. @@ -1733,11 +1725,6 @@ TEST_F(TransactionCoordinatorMetricsTest, // Slow log line is not logged since the coordination did not complete successfully. stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("two-phase commit parameters:")); - - // Clear the failpoint before the next test. - setGlobalFailPoint("hangBeforeWaitingForDecisionWriteConcern", - BSON("mode" - << "off")); } TEST_F(TransactionCoordinatorMetricsTest, @@ -1842,16 +1829,15 @@ TEST_F(TransactionCoordinatorMetricsTest, CoordinatorsAWSIsShutDownWhileCoordina expectedMetrics.totalStartedTwoPhaseCommit++; expectedMetrics.currentDeletingCoordinatorDoc++; - setGlobalFailPoint("hangAfterDeletingCoordinatorDoc", - BSON("mode" - << "alwaysOn")); + FailPointEnableBlock fp("hangAfterDeletingCoordinatorDoc"); + coordinator.runCommit(kTwoShardIdList); // Respond to the second prepare request in a separate thread, because the coordinator will // hijack that thread to run its continuation. assertPrepareSentAndRespondWithSuccess(); auto future = launchAsync([this] { assertPrepareSentAndRespondWithSuccess(); }); - // The last thing the coordinator will do on the hijacked prepare response thread is schedule - // the commitTransaction network requests. + // The last thing the coordinator will do on the hijacked prepare response thread is + // schedule the commitTransaction network requests. future.timed_get(kLongFutureTimeout); waitUntilMessageSent(); // Respond to the second commit request in a separate thread, because the coordinator will @@ -1872,8 +1858,8 @@ TEST_F(TransactionCoordinatorMetricsTest, CoordinatorsAWSIsShutDownWhileCoordina expectedMetrics.currentDeletingCoordinatorDoc--; awsPtr->shutdown({ErrorCodes::TransactionCoordinatorSteppingDown, "dummy"}); - // The last thing the coordinator will do on the hijacked commit response thread is signal the - // coordinator's completion. + // The last thing the coordinator will do on the hijacked commit response thread is signal + // the coordinator's completion. future.timed_get(kLongFutureTimeout); coordinator.onCompletion().get(); @@ -1883,11 +1869,6 @@ TEST_F(TransactionCoordinatorMetricsTest, CoordinatorsAWSIsShutDownWhileCoordina // Slow log line is not logged since the coordination did not complete successfully. stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("two-phase commit parameters:")); - - // Clear the failpoint before the next test. - setGlobalFailPoint("hangAfterDeletingCoordinatorDoc", - BSON("mode" - << "off")); } TEST_F(TransactionCoordinatorMetricsTest, LogsTransactionAtLogLevelOne) { @@ -2046,70 +2027,63 @@ TEST_F(TransactionCoordinatorMetricsTest, SlowLogLineIncludesStepDurationsAndTot std::make_unique<txn::AsyncWorkScheduler>(getServiceContext()), Date_t::max()); - setGlobalFailPoint("hangBeforeWaitingForParticipantListWriteConcern", - BSON("mode" - << "alwaysOn" - << "data" - << BSON("useUninterruptibleSleep" << 1))); - coordinator.runCommit(kTwoShardIdList); - waitUntilCoordinatorDocIsPresent(); + { + FailPointEnableBlock fp("hangBeforeWaitingForParticipantListWriteConcern", + BSON("useUninterruptibleSleep" << 1)); - // Increase the duration spent writing the participant list. - tickSource()->advance(Milliseconds(100)); + coordinator.runCommit(kTwoShardIdList); + waitUntilCoordinatorDocIsPresent(); + + // Increase the duration spent writing the participant list. + tickSource()->advance(Milliseconds(100)); + } - setGlobalFailPoint("hangBeforeWaitingForParticipantListWriteConcern", - BSON("mode" - << "off")); waitUntilMessageSent(); // Increase the duration spent waiting for votes. tickSource()->advance(Milliseconds(100)); - setGlobalFailPoint("hangBeforeWaitingForDecisionWriteConcern", - BSON("mode" - << "alwaysOn" - << "data" - << BSON("useUninterruptibleSleep" << 1))); - // Respond to the second prepare request in a separate thread, because the coordinator will - // hijack that thread to run its continuation. - assertPrepareSentAndRespondWithSuccess(); - auto future = launchAsync([this] { assertPrepareSentAndRespondWithSuccess(); }); - waitUntilCoordinatorDocHasDecision(); + boost::optional<executor::NetworkTestEnv::FutureHandle<void>> futureOption; - // Increase the duration spent writing the decision. - tickSource()->advance(Milliseconds(100)); + { + FailPointEnableBlock fp("hangBeforeWaitingForDecisionWriteConcern", + BSON("useUninterruptibleSleep" << 1)); + + // Respond to the second prepare request in a separate thread, because the coordinator will + // hijack that thread to run its continuation. + assertPrepareSentAndRespondWithSuccess(); + futureOption.emplace(launchAsync([this] { assertPrepareSentAndRespondWithSuccess(); })); + waitUntilCoordinatorDocHasDecision(); + + // Increase the duration spent writing the decision. + tickSource()->advance(Milliseconds(100)); + } - setGlobalFailPoint("hangBeforeWaitingForDecisionWriteConcern", - BSON("mode" - << "off")); // The last thing the coordinator will do on the hijacked prepare response thread is schedule // the commitTransaction network requests. - future.timed_get(kLongFutureTimeout); + futureOption->timed_get(kLongFutureTimeout); waitUntilMessageSent(); // Increase the duration spent waiting for decision acks. tickSource()->advance(Milliseconds(100)); - setGlobalFailPoint("hangAfterDeletingCoordinatorDoc", - BSON("mode" - << "alwaysOn" - << "data" - << BSON("useUninterruptibleSleep" << 1))); - // Respond to the second commit request in a separate thread, because the coordinator will - // hijack that thread to run its continuation. - assertCommitSentAndRespondWithSuccess(); - future = launchAsync([this] { assertCommitSentAndRespondWithSuccess(); }); - waitUntilNoCoordinatorDocIsPresent(); + { + FailPointEnableBlock fp("hangAfterDeletingCoordinatorDoc", + BSON("useUninterruptibleSleep" << 1)); - // Increase the duration spent deleting the coordinator doc. - tickSource()->advance(Milliseconds(100)); + // Respond to the second commit request in a separate thread, because the coordinator will + // hijack that thread to run its continuation. + assertCommitSentAndRespondWithSuccess(); + futureOption.emplace(launchAsync([this] { assertCommitSentAndRespondWithSuccess(); })); + waitUntilNoCoordinatorDocIsPresent(); + + // Increase the duration spent deleting the coordinator doc. + tickSource()->advance(Milliseconds(100)); + } - setGlobalFailPoint("hangAfterDeletingCoordinatorDoc", - BSON("mode" - << "off")); // The last thing the coordinator will do on the hijacked commit response thread is signal the // coordinator's completion. - future.timed_get(kLongFutureTimeout); + futureOption->timed_get(kLongFutureTimeout); coordinator.onCompletion().get(); stopCapturingLogMessages(); diff --git a/src/mongo/util/fail_point_service.cpp b/src/mongo/util/fail_point_service.cpp index 9b286d6fc32..77068f82100 100644 --- a/src/mongo/util/fail_point_service.cpp +++ b/src/mongo/util/fail_point_service.cpp @@ -78,14 +78,26 @@ void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) warning() << "failpoint: " << failPointName << " set to: " << failPoint->toBSON(); } -FailPointEnableBlock::FailPointEnableBlock(const std::string& failPointName) { +FailPointEnableBlock::FailPointEnableBlock(const std::string& failPointName) + : _failPointName(failPointName) { _failPoint = getGlobalFailPointRegistry()->getFailPoint(failPointName); invariant(_failPoint != nullptr); _failPoint->setMode(FailPoint::alwaysOn); + warning() << "failpoint: " << failPointName << " set to: " << _failPoint->toBSON(); } +FailPointEnableBlock::FailPointEnableBlock(const std::string& failPointName, const BSONObj& data) + : _failPointName(failPointName) { + _failPoint = getGlobalFailPointRegistry()->getFailPoint(failPointName); + invariant(_failPoint != nullptr); + _failPoint->setMode(FailPoint::alwaysOn, 0, data); + warning() << "failpoint: " << failPointName << " set to: " << _failPoint->toBSON(); +} + + FailPointEnableBlock::~FailPointEnableBlock() { _failPoint->setMode(FailPoint::off); + warning() << "failpoint: " << _failPointName << " set to: " << _failPoint->toBSON(); } } // namespace mongo diff --git a/src/mongo/util/fail_point_service.h b/src/mongo/util/fail_point_service.h index 4852b673b2a..64c511942fc 100644 --- a/src/mongo/util/fail_point_service.h +++ b/src/mongo/util/fail_point_service.h @@ -69,10 +69,12 @@ void setGlobalFailPoint(const std::string& failPointName, const BSONObj& cmdObj) class FailPointEnableBlock { public: FailPointEnableBlock(const std::string& failPointName); + FailPointEnableBlock(const std::string& failPointName, const BSONObj& cmdObj); ~FailPointEnableBlock(); private: FailPoint* _failPoint; + const std::string _failPointName; }; } // namespace mongo |