diff options
author | Andy Schwerin <Andy Schwerin schwerin@mongodb.com> | 2017-04-07 14:59:54 -0400 |
---|---|---|
committer | Andy Schwerin <Andy Schwerin schwerin@mongodb.com> | 2017-04-10 16:12:58 -0400 |
commit | 3dfb9f67d20188042ce019d2720593935da27b33 (patch) | |
tree | 0676b435067e1f9195038574441a5755dfd26676 | |
parent | 61339fc6cf60938cbc83fd5b6772b10da55d4289 (diff) | |
download | mongo-3dfb9f67d20188042ce019d2720593935da27b33.tar.gz |
SERVER-28719 Use a rescheduling fail point instead of blocking on a condition variable in some unit tests.
7 files changed, 39 insertions, 37 deletions
diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp index 0c7302bf701..a177d0008b8 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.cpp @@ -53,7 +53,6 @@ ReplicationCoordinatorExternalStateMock::ReplicationCoordinatorExternalStateMock _canAcquireGlobalSharedLock(true), _storeLocalConfigDocumentStatus(Status::OK()), _storeLocalLastVoteDocumentStatus(Status::OK()), - _storeLocalConfigDocumentShouldHang(false), _storeLocalLastVoteDocumentShouldHang(false), _connectionsClosed(false), _threadsStarted(false) {} @@ -127,12 +126,6 @@ StatusWith<BSONObj> ReplicationCoordinatorExternalStateMock::loadLocalConfigDocu Status ReplicationCoordinatorExternalStateMock::storeLocalConfigDocument(OperationContext* opCtx, const BSONObj& config) { - { - stdx::unique_lock<stdx::mutex> lock(_shouldHangConfigMutex); - while (_storeLocalConfigDocumentShouldHang) { - _shouldHangConfigCondVar.wait(lock); - } - } if (_storeLocalConfigDocumentStatus.isOK()) { setLocalConfigDocument(StatusWith<BSONObj>(config)); return Status::OK(); @@ -188,14 +181,6 @@ void ReplicationCoordinatorExternalStateMock::setStoreLocalConfigDocumentStatus( _storeLocalConfigDocumentStatus = status; } -void ReplicationCoordinatorExternalStateMock::setStoreLocalConfigDocumentToHang(bool hang) { - stdx::unique_lock<stdx::mutex> lock(_shouldHangConfigMutex); - _storeLocalConfigDocumentShouldHang = hang; - if (!hang) { - _shouldHangConfigCondVar.notify_all(); - } -} - bool ReplicationCoordinatorExternalStateMock::threadsStarted() const { return _threadsStarted; } diff --git a/src/mongo/db/repl/replication_coordinator_external_state_mock.h b/src/mongo/db/repl/replication_coordinator_external_state_mock.h index 7f6268e8969..aeb27de0fce 100644 --- a/src/mongo/db/repl/replication_coordinator_external_state_mock.h +++ b/src/mongo/db/repl/replication_coordinator_external_state_mock.h @@ -141,12 +141,6 @@ public: void setStoreLocalConfigDocumentStatus(Status status); /** - * Sets whether or not subsequent calls to storeLocalConfigDocument() should hang - * indefinitely or not based on the value of "hang". - */ - void setStoreLocalConfigDocumentToHang(bool hang); - - /** * Sets the return value for subsequent calls to storeLocalLastVoteDocument(). * If "status" is Status::OK(), the subsequent calls will call the underlying funtion. */ @@ -196,13 +190,9 @@ private: bool _canAcquireGlobalSharedLock; Status _storeLocalConfigDocumentStatus; Status _storeLocalLastVoteDocumentStatus; - // mutex and cond var for controlling stroeLocalConfigDocument()'s hanging - stdx::mutex _shouldHangConfigMutex; - stdx::condition_variable _shouldHangConfigCondVar; // mutex and cond var for controlling stroeLocalLastVoteDocument()'s hanging stdx::mutex _shouldHangLastVoteMutex; stdx::condition_variable _shouldHangLastVoteCondVar; - bool _storeLocalConfigDocumentShouldHang; bool _storeLocalLastVoteDocumentShouldHang; bool _connectionsClosed; HostAndPort _clientHostAndPort; diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp index 5724fb43bf6..e0ba997c564 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp @@ -42,6 +42,7 @@ #include "mongo/db/repl/topology_coordinator_impl.h" #include "mongo/executor/network_interface_mock.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -383,8 +384,9 @@ TEST_F(ReplCoordElectTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); getReplCoord()->setMyLastAppliedOpTime(OpTime(Timestamp(100, 0), 0)); - // set hbreconfig to hang while in progress - getExternalState()->setStoreLocalConfigDocumentToHang(true); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::alwaysOn); // hb reconfig NetworkInterfaceMock* net = getNet(); @@ -455,7 +457,9 @@ TEST_F(ReplCoordElectTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { ASSERT_EQUALS(1, countLogLinesContaining("Not standing for election; processing " "a configuration change")); - getExternalState()->setStoreLocalConfigDocumentToHang(false); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::off); } TEST_F(ReplCoordElectTest, StepsDownRemoteIfNodeHasHigherPriorityThanCurrentPrimary) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp index 34bb0a9c101..c26e5af56ea 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1_test.cpp @@ -42,6 +42,7 @@ #include "mongo/db/repl/topology_coordinator_impl.h" #include "mongo/executor/network_interface_mock.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -462,8 +463,9 @@ TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { getReplCoord()->setMyLastAppliedOpTime(OpTime(Timestamp(100, 0), 0)); getReplCoord()->setMyLastDurableOpTime(OpTime(Timestamp(100, 0), 0)); - // set hbreconfig to hang while in progress - getExternalState()->setStoreLocalConfigDocumentToHang(true); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::alwaysOn); // hb reconfig NetworkInterfaceMock* net = getNet(); @@ -547,7 +549,9 @@ TEST_F(ReplCoordTest, NodeWillNotStandForElectionDuringHeartbeatReconfig) { ASSERT_EQUALS(1, countLogLinesContaining("Not standing for election; processing " "a configuration change")); - getExternalState()->setStoreLocalConfigDocumentToHang(false); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::off); } TEST_F(ReplCoordTest, ElectionFailsWhenInsufficientVotesAreReceivedDuringRequestVotes) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 1a91b82c33d..ac532ba036c 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -68,6 +68,7 @@ using CBHandle = ReplicationExecutor::CallbackHandle; using CBHStatus = StatusWith<CBHandle>; MONGO_FP_DECLARE(blockHeartbeatStepdown); +MONGO_FP_DECLARE(blockHeartbeatReconfigFinish); } // namespace @@ -538,10 +539,22 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( const ReplicationExecutor::CallbackArgs& cbData, const ReplSetConfig& newConfig, StatusWith<int> myIndex) { + if (cbData.status == ErrorCodes::CallbackCanceled) { return; } + if (MONGO_FAIL_POINT(blockHeartbeatReconfigFinish)) { + _replExecutor.scheduleWorkAt( + _replExecutor.now() + Milliseconds{10}, + stdx::bind(&ReplicationCoordinatorImpl::_heartbeatReconfigFinish, + this, + stdx::placeholders::_1, + newConfig, + myIndex)); + return; + } + stdx::unique_lock<stdx::mutex> lk(_mutex); invariant(_rsConfigState == kConfigHBReconfiguring); diff --git a/src/mongo/db/repl/replication_coordinator_impl_reconfig_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_reconfig_test.cpp index 926b4a47a0f..761359552ed 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_reconfig_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_reconfig_test.cpp @@ -41,6 +41,7 @@ #include "mongo/db/repl/replication_coordinator_test_fixture.h" #include "mongo/executor/network_interface_mock.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -510,8 +511,9 @@ TEST_F( simulateSuccessfulV1Election(); ASSERT_TRUE(getReplCoord()->getMemberState().primary()); - // set hbreconfig to hang while in progress - getExternalState()->setStoreLocalConfigDocumentToHang(true); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::alwaysOn); // hb reconfig NetworkInterfaceMock* net = getNet(); @@ -549,7 +551,9 @@ TEST_F( ASSERT_EQUALS(ErrorCodes::ConfigurationInProgress, getReplCoord()->processReplSetReconfig(opCtx.get(), args, &result)); - getExternalState()->setStoreLocalConfigDocumentToHang(false); + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::off); } TEST_F(ReplCoordTest, NodeDoesNotAcceptHeartbeatReconfigWhileInTheMidstOfReconfig) { diff --git a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp index 21fe828ee95..4e241ee07ae 100644 --- a/src/mongo/db/repl/replication_coordinator_test_fixture.cpp +++ b/src/mongo/db/repl/replication_coordinator_test_fixture.cpp @@ -47,6 +47,7 @@ #include "mongo/stdx/functional.h" #include "mongo/stdx/memory.h" #include "mongo/unittest/unittest.h" +#include "mongo/util/fail_point_service.h" #include "mongo/util/log.h" namespace mongo { @@ -85,9 +86,10 @@ void ReplCoordTest::setUp() { } void ReplCoordTest::tearDown() { - if (_externalState) { - _externalState->setStoreLocalConfigDocumentToHang(false); - } + getGlobalFailPointRegistry() + ->getFailPoint("blockHeartbeatReconfigFinish") + ->setMode(FailPoint::off); + if (_callShutdown) { auto opCtx = makeOperationContext(); shutdown(opCtx.get()); |