diff options
Diffstat (limited to 'src/mongo/db/repl/check_quorum_for_config_change_test.cpp')
-rw-r--r-- | src/mongo/db/repl/check_quorum_for_config_change_test.cpp | 1440 |
1 files changed, 727 insertions, 713 deletions
diff --git a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp index 8f0e01ec00b..7b3d869b47b 100644 --- a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp +++ b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp @@ -45,756 +45,770 @@ #include "mongo/unittest/unittest.h" #include "mongo/util/net/hostandport.h" -#define ASSERT_REASON_CONTAINS(STATUS, PATTERN) do { \ - const mongo::Status s_ = (STATUS); \ - ASSERT_FALSE(s_.reason().find(PATTERN) == std::string::npos) << \ - #STATUS ".reason() == " << s_.reason(); \ +#define ASSERT_REASON_CONTAINS(STATUS, PATTERN) \ + do { \ + const mongo::Status s_ = (STATUS); \ + ASSERT_FALSE(s_.reason().find(PATTERN) == std::string::npos) \ + << #STATUS ".reason() == " << s_.reason(); \ } while (false) -#define ASSERT_NOT_REASON_CONTAINS(STATUS, PATTERN) do { \ - const mongo::Status s_ = (STATUS); \ - ASSERT_TRUE(s_.reason().find(PATTERN) == std::string::npos) << \ - #STATUS ".reason() == " << s_.reason(); \ +#define ASSERT_NOT_REASON_CONTAINS(STATUS, PATTERN) \ + do { \ + const mongo::Status s_ = (STATUS); \ + ASSERT_TRUE(s_.reason().find(PATTERN) == std::string::npos) \ + << #STATUS ".reason() == " << s_.reason(); \ } while (false) namespace mongo { namespace repl { namespace { - using executor::NetworkInterfaceMock; - - class CheckQuorumTest : public mongo::unittest::Test { - protected: - CheckQuorumTest(); - - void startQuorumCheck(const ReplicaSetConfig& config, int myIndex); - Status waitForQuorumCheck(); - bool isQuorumCheckDone(); - - NetworkInterfaceMock* _net; - StorageInterfaceMock* _storage; - std::unique_ptr<ReplicationExecutor> _executor; - - private: - void setUp(); - void tearDown(); - - void _runQuorumCheck(const ReplicaSetConfig& config, int myIndex); - virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) = 0; - - std::unique_ptr<stdx::thread> _executorThread; - std::unique_ptr<stdx::thread> _quorumCheckThread; - Status _quorumCheckStatus; - stdx::mutex _mutex; - bool _isQuorumCheckDone; - }; - - CheckQuorumTest::CheckQuorumTest() : - _quorumCheckStatus(ErrorCodes::InternalError, "Not executed") { - } - - void CheckQuorumTest::setUp() { - _net = new NetworkInterfaceMock; - _storage = new StorageInterfaceMock; - _executor.reset(new ReplicationExecutor(_net, _storage, 1 /* prng */ )); - _executorThread.reset(new stdx::thread(stdx::bind(&ReplicationExecutor::run, - _executor.get()))); - } - - void CheckQuorumTest::tearDown() { - _executor->shutdown(); - _executorThread->join(); - } - - void CheckQuorumTest::startQuorumCheck(const ReplicaSetConfig& config, int myIndex) { - ASSERT_FALSE(_quorumCheckThread); - _isQuorumCheckDone = false; - _quorumCheckThread.reset(new stdx::thread(stdx::bind(&CheckQuorumTest::_runQuorumCheck, - this, - config, - myIndex))); - } - - Status CheckQuorumTest::waitForQuorumCheck() { - ASSERT_TRUE(_quorumCheckThread); - _quorumCheckThread->join(); - return _quorumCheckStatus; +using executor::NetworkInterfaceMock; + +class CheckQuorumTest : public mongo::unittest::Test { +protected: + CheckQuorumTest(); + + void startQuorumCheck(const ReplicaSetConfig& config, int myIndex); + Status waitForQuorumCheck(); + bool isQuorumCheckDone(); + + NetworkInterfaceMock* _net; + StorageInterfaceMock* _storage; + std::unique_ptr<ReplicationExecutor> _executor; + +private: + void setUp(); + void tearDown(); + + void _runQuorumCheck(const ReplicaSetConfig& config, int myIndex); + virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) = 0; + + std::unique_ptr<stdx::thread> _executorThread; + std::unique_ptr<stdx::thread> _quorumCheckThread; + Status _quorumCheckStatus; + stdx::mutex _mutex; + bool _isQuorumCheckDone; +}; + +CheckQuorumTest::CheckQuorumTest() + : _quorumCheckStatus(ErrorCodes::InternalError, "Not executed") {} + +void CheckQuorumTest::setUp() { + _net = new NetworkInterfaceMock; + _storage = new StorageInterfaceMock; + _executor.reset(new ReplicationExecutor(_net, _storage, 1 /* prng */)); + _executorThread.reset(new stdx::thread(stdx::bind(&ReplicationExecutor::run, _executor.get()))); +} + +void CheckQuorumTest::tearDown() { + _executor->shutdown(); + _executorThread->join(); +} + +void CheckQuorumTest::startQuorumCheck(const ReplicaSetConfig& config, int myIndex) { + ASSERT_FALSE(_quorumCheckThread); + _isQuorumCheckDone = false; + _quorumCheckThread.reset( + new stdx::thread(stdx::bind(&CheckQuorumTest::_runQuorumCheck, this, config, myIndex))); +} + +Status CheckQuorumTest::waitForQuorumCheck() { + ASSERT_TRUE(_quorumCheckThread); + _quorumCheckThread->join(); + return _quorumCheckStatus; +} + +bool CheckQuorumTest::isQuorumCheckDone() { + stdx::lock_guard<stdx::mutex> lk(_mutex); + return _isQuorumCheckDone; +} + +void CheckQuorumTest::_runQuorumCheck(const ReplicaSetConfig& config, int myIndex) { + _quorumCheckStatus = _runQuorumCheckImpl(config, myIndex); + stdx::lock_guard<stdx::mutex> lk(_mutex); + _isQuorumCheckDone = true; +} + +class CheckQuorumForInitiate : public CheckQuorumTest { +private: + virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) { + return checkQuorumForInitiate(_executor.get(), config, myIndex); } +}; - bool CheckQuorumTest::isQuorumCheckDone() { - stdx::lock_guard<stdx::mutex> lk(_mutex); - return _isQuorumCheckDone; +class CheckQuorumForReconfig : public CheckQuorumTest { +protected: + virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) { + return checkQuorumForReconfig(_executor.get(), config, myIndex); } - - void CheckQuorumTest::_runQuorumCheck(const ReplicaSetConfig& config, int myIndex) { - _quorumCheckStatus = _runQuorumCheckImpl(config, myIndex); - stdx::lock_guard<stdx::mutex> lk(_mutex); - _isQuorumCheckDone = true; - } - - class CheckQuorumForInitiate : public CheckQuorumTest { - private: - virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) { - return checkQuorumForInitiate(_executor.get(), config, myIndex); - } - }; - - class CheckQuorumForReconfig : public CheckQuorumTest { - protected: - virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) { - return checkQuorumForReconfig(_executor.get(), config, myIndex); - } - }; - - ReplicaSetConfig assertMakeRSConfig(const BSONObj& configBson) { - ReplicaSetConfig config; - ASSERT_OK(config.initialize(configBson)); - ASSERT_OK(config.validate()); - return config; +}; + +ReplicaSetConfig assertMakeRSConfig(const BSONObj& configBson) { + ReplicaSetConfig config; + ASSERT_OK(config.initialize(configBson)); + ASSERT_OK(config.validate()); + return config; +} + +TEST_F(CheckQuorumForInitiate, ValidSingleNodeSet) { + ReplicaSetConfig config = assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1")))); + startQuorumCheck(config, 0); + ASSERT_OK(waitForQuorumCheck()); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckCanceledByShutdown) { + _executor->shutdown(); + ReplicaSetConfig config = assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1")))); + startQuorumCheck(config, 0); + ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, waitForQuorumCheck()); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSeveralDownNodes) { + // In this test, "we" are host "h3:1". All other nodes time out on + // their heartbeat request, and so the quorum check for initiate + // will fail because some members were unavailable. + ReplicaSetConfig config = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + startQuorumCheck(config, 2); + _net->enterNetwork(); + const Date_t startDate = _net->now(); + const int numCommandsExpected = config.getNumMembers() - 1; + for (int i = 0; i < numCommandsExpected; ++i) { + _net->scheduleResponse(_net->getNextReadyRequest(), + startDate + Milliseconds(10), + ResponseStatus(ErrorCodes::NoSuchKey, "No reply")); } - - TEST_F(CheckQuorumForInitiate, ValidSingleNodeSet) { - ReplicaSetConfig config = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1")))); - startQuorumCheck(config, 0); - ASSERT_OK(waitForQuorumCheck()); + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + ASSERT_EQUALS(startDate + Milliseconds(10), _net->now()); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); + ASSERT_REASON_CONTAINS( + status, "replSetInitiate quorum check failed because not all proposed set members"); + ASSERT_REASON_CONTAINS(status, "h1:1"); + ASSERT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_REASON_CONTAINS(status, "h4:1"); + ASSERT_REASON_CONTAINS(status, "h5:1"); +} + +const BSONObj makeHeartbeatRequest(const ReplicaSetConfig& rsConfig, int myConfigIndex) { + const MemberConfig& myConfig = rsConfig.getMemberAt(myConfigIndex); + ReplSetHeartbeatArgs hbArgs; + hbArgs.setSetName(rsConfig.getReplSetName()); + hbArgs.setProtocolVersion(1); + hbArgs.setConfigVersion(rsConfig.getConfigVersion()); + hbArgs.setCheckEmpty(rsConfig.getConfigVersion() == 1); + hbArgs.setSenderHost(myConfig.getHostAndPort()); + hbArgs.setSenderId(myConfig.getId()); + return hbArgs.toBSON(); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckSuccessForFiveNodes) { + // In this test, "we" are host "h3:1". All nodes respond successfully to their heartbeat + // requests, and the quorum check succeeds. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); } - - TEST_F(CheckQuorumForInitiate, QuorumCheckCanceledByShutdown) { - _executor->shutdown(); - ReplicaSetConfig config = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1")))); - startQuorumCheck(config, 0); - ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, waitForQuorumCheck()); - } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSeveralDownNodes) { - // In this test, "we" are host "h3:1". All other nodes time out on - // their heartbeat request, and so the quorum check for initiate - // will fail because some members were unavailable. - ReplicaSetConfig config = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - startQuorumCheck(config, 2); - _net->enterNetwork(); - const Date_t startDate = _net->now(); - const int numCommandsExpected = config.getNumMembers() - 1; - for (int i = 0; i < numCommandsExpected; ++i) { - _net->scheduleResponse(_net->getNextReadyRequest(), + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + ASSERT_OK(waitForQuorumCheck()); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToOneDownNode) { + // In this test, "we" are host "h3:1". All nodes except "h2:1" respond + // successfully to their heartbeat requests, but quorum check fails because + // all nodes must be available for initiate. This is so even though "h2" + // is neither voting nor electable. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1" + << "priority" << 0 << "votes" << 0) + << BSON("_id" << 3 << "host" + << "h3:1") << BSON("_id" << 4 << "host" + << "h4:1") + << BSON("_id" << 5 << "host" + << "h5:1") << BSON("_id" << 6 << "host" + << "h6:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h2", 1)) { + _net->scheduleResponse(noi, startDate + Milliseconds(10), - ResponseStatus(ErrorCodes::NoSuchKey, "No reply")); + ResponseStatus(ErrorCodes::NoSuchKey, "No response")); + } else { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - ASSERT_EQUALS(startDate + Milliseconds(10), _net->now()); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); - ASSERT_REASON_CONTAINS( - status, "replSetInitiate quorum check failed because not all proposed set members"); - ASSERT_REASON_CONTAINS(status, "h1:1"); - ASSERT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_REASON_CONTAINS(status, "h4:1"); - ASSERT_REASON_CONTAINS(status, "h5:1"); } - - const BSONObj makeHeartbeatRequest(const ReplicaSetConfig& rsConfig, int myConfigIndex) { - const MemberConfig& myConfig = rsConfig.getMemberAt(myConfigIndex); - ReplSetHeartbeatArgs hbArgs; - hbArgs.setSetName(rsConfig.getReplSetName()); - hbArgs.setProtocolVersion(1); - hbArgs.setConfigVersion(rsConfig.getConfigVersion()); - hbArgs.setCheckEmpty(rsConfig.getConfigVersion() == 1); - hbArgs.setSenderHost(myConfig.getHostAndPort()); - hbArgs.setSenderId(myConfig.getId()); - return hbArgs.toBSON(); - } - - TEST_F(CheckQuorumForInitiate, QuorumCheckSuccessForFiveNodes) { - // In this test, "we" are host "h3:1". All nodes respond successfully to their heartbeat - // requests, and the quorum check succeeds. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); + ASSERT_REASON_CONTAINS( + status, "replSetInitiate quorum check failed because not all proposed set members"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h6:1"); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSetNameMismatch) { + // In this test, "we" are host "h3:1". All nodes respond + // successfully to their heartbeat requests, but quorum check fails because + // "h4" declares that the requested replica set name was not what it expected. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h4", 1)) { _net->scheduleResponse(noi, startDate + Milliseconds(10), ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), Milliseconds(8)))); + BSON("ok" << 0 << "mismatch" << true), Milliseconds(8)))); + } else { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - ASSERT_OK(waitForQuorumCheck()); } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToOneDownNode) { - // In this test, "we" are host "h3:1". All nodes except "h2:1" respond - // successfully to their heartbeat requests, but quorum check fails because - // all nodes must be available for initiate. This is so even though "h2" - // is neither voting nor electable. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1" << - "priority" << 0 << "votes" << 0) << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1") << - BSON("_id" << 6 << "host" << "h6:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h2", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(ErrorCodes::NoSuchKey, "No response")); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), Milliseconds(8)))); - } - } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); - ASSERT_REASON_CONTAINS( - status, "replSetInitiate quorum check failed because not all proposed set members"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h6:1"); - } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSetNameMismatch) { - // In this test, "we" are host "h3:1". All nodes respond - // successfully to their heartbeat requests, but quorum check fails because - // "h4" declares that the requested replica set name was not what it expected. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h4", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 0 << "mismatch" << true), - Milliseconds(8)))); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), Milliseconds(8)))); - } - } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); - ASSERT_REASON_CONTAINS( - status, "Our set name did not match"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_REASON_CONTAINS(status, "h4:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); - } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNode) { - // In this test, "we" are host "h3:1". All nodes respond - // successfully to their heartbeat requests, but quorum check fails because - // "h5" declares that it is already initialized. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h5", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 0 << - "set" << "rs0" << - "v" << 1), - Milliseconds(8)))); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), Milliseconds(8)))); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); + ASSERT_REASON_CONTAINS(status, "Our set name did not match"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_REASON_CONTAINS(status, "h4:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNode) { + // In this test, "we" are host "h3:1". All nodes respond + // successfully to their heartbeat requests, but quorum check fails because + // "h5" declares that it is already initialized. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h5", 1)) { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set" + << "rs0" + << "v" << 1), + Milliseconds(8)))); + } else { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); - ASSERT_REASON_CONTAINS( - status, "Our config version of"); - ASSERT_REASON_CONTAINS( - status, "is no larger than the version"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); - ASSERT_REASON_CONTAINS(status, "h5:1"); } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNodeOnlyOneRespondent) { - // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes, - // and quorum check fails because "h5" declares that it is already initialized. - // - // Compare to QuorumCheckFailedDueToInitializedNode, above. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h5", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 0 << - "set" << "rs0" << - "v" << 1), - Milliseconds(8)))); - } - else { - _net->blackHole(noi); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); + ASSERT_REASON_CONTAINS(status, "Our config version of"); + ASSERT_REASON_CONTAINS(status, "is no larger than the version"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); + ASSERT_REASON_CONTAINS(status, "h5:1"); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNodeOnlyOneRespondent) { + // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes, + // and quorum check fails because "h5" declares that it is already initialized. + // + // Compare to QuorumCheckFailedDueToInitializedNode, above. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h5", 1)) { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set" + << "rs0" + << "v" << 1), + Milliseconds(8)))); + } else { + _net->blackHole(noi); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); - ASSERT_REASON_CONTAINS( - status, "Our config version of"); - ASSERT_REASON_CONTAINS( - status, "is no larger than the version"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); - ASSERT_REASON_CONTAINS(status, "h5:1"); } - - TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToNodeWithData) { - // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes, - // and quorum check fails because "h5" declares that it has data already. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 1 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1") << - BSON("_id" << 5 << "host" << "h5:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - ReplSetHeartbeatResponse hbResp; - hbResp.setConfigVersion(0); - hbResp.noteHasData(); - if (request.target == HostAndPort("h5", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - hbResp.toBSON(false), - Milliseconds(8)))); - } - else { - _net->blackHole(noi); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); + ASSERT_REASON_CONTAINS(status, "Our config version of"); + ASSERT_REASON_CONTAINS(status, "is no larger than the version"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); + ASSERT_REASON_CONTAINS(status, "h5:1"); +} + +TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToNodeWithData) { + // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes, + // and quorum check fails because "h5" declares that it has data already. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1") << BSON("_id" << 5 << "host" + << "h5:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + ReplSetHeartbeatResponse hbResp; + hbResp.setConfigVersion(0); + hbResp.noteHasData(); + if (request.target == HostAndPort("h5", 1)) { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(hbResp.toBSON(false), Milliseconds(8)))); + } else { + _net->blackHole(noi); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::CannotInitializeNodeWithData, status); - ASSERT_REASON_CONTAINS( - status, "has data already"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); - ASSERT_REASON_CONTAINS(status, "h5:1"); } - TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToHigherConfigVersion) { - // In this test, "we" are host "h3:1". The request to "h2" does not arrive before the end - // of the test, and the request to "h1" comes back indicating a higher config version. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 2 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h1", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 0 << - "set" << "rs0" << - "v" << 5), - Milliseconds(8)))); - } - else { - _net->blackHole(noi); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::CannotInitializeNodeWithData, status); + ASSERT_REASON_CONTAINS(status, "has data already"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); + ASSERT_REASON_CONTAINS(status, "h5:1"); +} +TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToHigherConfigVersion) { + // In this test, "we" are host "h3:1". The request to "h2" does not arrive before the end + // of the test, and the request to "h1" comes back indicating a higher config version. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h1", 1)) { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set" + << "rs0" + << "v" << 5), + Milliseconds(8)))); + } else { + _net->blackHole(noi); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); - ASSERT_REASON_CONTAINS( - status, "Our config version of"); - ASSERT_REASON_CONTAINS( - status, "is no larger than the version"); - ASSERT_REASON_CONTAINS(status, "h1:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); } - - TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToIncompatibleSetName) { - // In this test, "we" are host "h3:1". The request to "h1" times out, - // and the request to "h2" comes back indicating an incompatible set name. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 2 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1")))); - const int myConfigIndex = 2; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h2", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 0 << "mismatch" << true), - Milliseconds(8)))); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(ErrorCodes::NoSuchKey, "No response")); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); + ASSERT_REASON_CONTAINS(status, "Our config version of"); + ASSERT_REASON_CONTAINS(status, "is no larger than the version"); + ASSERT_REASON_CONTAINS(status, "h1:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); +} + +TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToIncompatibleSetName) { + // In this test, "we" are host "h3:1". The request to "h1" times out, + // and the request to "h2" comes back indicating an incompatible set name. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1")))); + const int myConfigIndex = 2; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h2", 1)) { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse( + BSON("ok" << 0 << "mismatch" << true), Milliseconds(8)))); + } else { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(ErrorCodes::NoSuchKey, "No response")); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); - ASSERT_REASON_CONTAINS(status, "Our set name did not match"); - ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); - ASSERT_REASON_CONTAINS(status, "h2:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); - } - - TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToInsufficientVoters) { - // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are voters, - // and of the voters, only "h1" responds. As a result, quorum check fails. - // "h5" also responds, but because it cannot vote, is irrelevant for the reconfig - // quorum check. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 2 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1" << "votes" << 0) << - BSON("_id" << 5 << "host" << "h5:1" << "votes" << 0)))); - const int myConfigIndex = 3; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h5", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), - Milliseconds(8)))); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(ErrorCodes::NoSuchKey, "No response")); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status); + ASSERT_REASON_CONTAINS(status, "Our set name did not match"); + ASSERT_NOT_REASON_CONTAINS(status, "h1:1"); + ASSERT_REASON_CONTAINS(status, "h2:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h3:1"); +} + +TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToInsufficientVoters) { + // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are voters, + // and of the voters, only "h1" responds. As a result, quorum check fails. + // "h5" also responds, but because it cannot vote, is irrelevant for the reconfig + // quorum check. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1" + << "votes" << 0) + << BSON("_id" << 5 << "host" + << "h5:1" + << "votes" << 0)))); + const int myConfigIndex = 3; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h5", 1)) { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); + } else { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(ErrorCodes::NoSuchKey, "No response")); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); - ASSERT_REASON_CONTAINS(status, "not enough voting nodes responded; required 2 but only"); - ASSERT_REASON_CONTAINS(status, "h1:1"); - ASSERT_REASON_CONTAINS(status, "h2:1 failed with"); - ASSERT_REASON_CONTAINS(status, "h3:1 failed with"); - ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); - ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); } - - TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToNoElectableNodeResponding) { - // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are electable, - // and none of them respond. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 2 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1" << "priority" << 0) << - BSON("_id" << 5 << "host" << "h5:1" << "priority" << 0)))); - const int myConfigIndex = 3; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h5", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), - Milliseconds(8)))); - } - else { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(ErrorCodes::NoSuchKey, "No response")); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); + ASSERT_REASON_CONTAINS(status, "not enough voting nodes responded; required 2 but only"); + ASSERT_REASON_CONTAINS(status, "h1:1"); + ASSERT_REASON_CONTAINS(status, "h2:1 failed with"); + ASSERT_REASON_CONTAINS(status, "h3:1 failed with"); + ASSERT_NOT_REASON_CONTAINS(status, "h4:1"); + ASSERT_NOT_REASON_CONTAINS(status, "h5:1"); +} + +TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToNoElectableNodeResponding) { + // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are electable, + // and none of them respond. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1" + << "priority" << 0) + << BSON("_id" << 5 << "host" + << "h5:1" + << "priority" << 0)))); + const int myConfigIndex = 3; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h5", 1)) { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); + } else { + _net->scheduleResponse(noi, + startDate + Milliseconds(10), + ResponseStatus(ErrorCodes::NoSuchKey, "No response")); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - Status status = waitForQuorumCheck(); - ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); - ASSERT_REASON_CONTAINS(status, "no electable nodes responded"); } - - TEST_F(CheckQuorumForReconfig, QuorumCheckSucceedsWithAsSoonAsPossible) { - // In this test, "we" are host "h4". Only "h1", "h2" and "h3" can vote. - // This test should succeed as soon as h1 and h2 respond, so we block - // h3 and h5 from responding or timing out until the test completes. - - const ReplicaSetConfig rsConfig = assertMakeRSConfig( - BSON("_id" << "rs0" << - "version" << 2 << - "members" << BSON_ARRAY( - BSON("_id" << 1 << "host" << "h1:1") << - BSON("_id" << 2 << "host" << "h2:1") << - BSON("_id" << 3 << "host" << "h3:1") << - BSON("_id" << 4 << "host" << "h4:1" << "votes" << 0) << - BSON("_id" << 5 << "host" << "h5:1" << "votes" << 0)))); - const int myConfigIndex = 3; - const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); - - startQuorumCheck(rsConfig, myConfigIndex); - const Date_t startDate = _net->now(); - const int numCommandsExpected = rsConfig.getNumMembers() - 1; - unordered_set<HostAndPort> seenHosts; - _net->enterNetwork(); - for (int i = 0; i < numCommandsExpected; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - ASSERT_EQUALS("admin", request.dbname); - ASSERT_EQUALS(hbRequest, request.cmdObj); - ASSERT(seenHosts.insert(request.target).second) << - "Already saw " << request.target.toString(); - if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h2", 1)) { - _net->scheduleResponse(noi, - startDate + Milliseconds(10), - ResponseStatus(RemoteCommandResponse( - BSON("ok" << 1), - Milliseconds(8)))); - } - else { - _net->blackHole(noi); - } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + Status status = waitForQuorumCheck(); + ASSERT_EQUALS(ErrorCodes::NodeNotFound, status); + ASSERT_REASON_CONTAINS(status, "no electable nodes responded"); +} + +TEST_F(CheckQuorumForReconfig, QuorumCheckSucceedsWithAsSoonAsPossible) { + // In this test, "we" are host "h4". Only "h1", "h2" and "h3" can vote. + // This test should succeed as soon as h1 and h2 respond, so we block + // h3 and h5 from responding or timing out until the test completes. + + const ReplicaSetConfig rsConfig = + assertMakeRSConfig(BSON("_id" + << "rs0" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") << BSON("_id" << 3 << "host" + << "h3:1") + << BSON("_id" << 4 << "host" + << "h4:1" + << "votes" << 0) + << BSON("_id" << 5 << "host" + << "h5:1" + << "votes" << 0)))); + const int myConfigIndex = 3; + const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex); + + startQuorumCheck(rsConfig, myConfigIndex); + const Date_t startDate = _net->now(); + const int numCommandsExpected = rsConfig.getNumMembers() - 1; + unordered_set<HostAndPort> seenHosts; + _net->enterNetwork(); + for (int i = 0; i < numCommandsExpected; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + ASSERT_EQUALS("admin", request.dbname); + ASSERT_EQUALS(hbRequest, request.cmdObj); + ASSERT(seenHosts.insert(request.target).second) << "Already saw " + << request.target.toString(); + if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h2", 1)) { + _net->scheduleResponse( + noi, + startDate + Milliseconds(10), + ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8)))); + } else { + _net->blackHole(noi); } - _net->runUntil(startDate + Milliseconds(10)); - _net->exitNetwork(); - ASSERT_OK(waitForQuorumCheck()); } + _net->runUntil(startDate + Milliseconds(10)); + _net->exitNetwork(); + ASSERT_OK(waitForQuorumCheck()); +} } // namespace } // namespace repl |