summaryrefslogtreecommitdiff
path: root/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'src/mongo/db/repl/check_quorum_for_config_change_test.cpp')
-rw-r--r--src/mongo/db/repl/check_quorum_for_config_change_test.cpp1440
1 files changed, 727 insertions, 713 deletions
diff --git a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
index 8f0e01ec00b..7b3d869b47b 100644
--- a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
+++ b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
@@ -45,756 +45,770 @@
#include "mongo/unittest/unittest.h"
#include "mongo/util/net/hostandport.h"
-#define ASSERT_REASON_CONTAINS(STATUS, PATTERN) do { \
- const mongo::Status s_ = (STATUS); \
- ASSERT_FALSE(s_.reason().find(PATTERN) == std::string::npos) << \
- #STATUS ".reason() == " << s_.reason(); \
+#define ASSERT_REASON_CONTAINS(STATUS, PATTERN) \
+ do { \
+ const mongo::Status s_ = (STATUS); \
+ ASSERT_FALSE(s_.reason().find(PATTERN) == std::string::npos) \
+ << #STATUS ".reason() == " << s_.reason(); \
} while (false)
-#define ASSERT_NOT_REASON_CONTAINS(STATUS, PATTERN) do { \
- const mongo::Status s_ = (STATUS); \
- ASSERT_TRUE(s_.reason().find(PATTERN) == std::string::npos) << \
- #STATUS ".reason() == " << s_.reason(); \
+#define ASSERT_NOT_REASON_CONTAINS(STATUS, PATTERN) \
+ do { \
+ const mongo::Status s_ = (STATUS); \
+ ASSERT_TRUE(s_.reason().find(PATTERN) == std::string::npos) \
+ << #STATUS ".reason() == " << s_.reason(); \
} while (false)
namespace mongo {
namespace repl {
namespace {
- using executor::NetworkInterfaceMock;
-
- class CheckQuorumTest : public mongo::unittest::Test {
- protected:
- CheckQuorumTest();
-
- void startQuorumCheck(const ReplicaSetConfig& config, int myIndex);
- Status waitForQuorumCheck();
- bool isQuorumCheckDone();
-
- NetworkInterfaceMock* _net;
- StorageInterfaceMock* _storage;
- std::unique_ptr<ReplicationExecutor> _executor;
-
- private:
- void setUp();
- void tearDown();
-
- void _runQuorumCheck(const ReplicaSetConfig& config, int myIndex);
- virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) = 0;
-
- std::unique_ptr<stdx::thread> _executorThread;
- std::unique_ptr<stdx::thread> _quorumCheckThread;
- Status _quorumCheckStatus;
- stdx::mutex _mutex;
- bool _isQuorumCheckDone;
- };
-
- CheckQuorumTest::CheckQuorumTest() :
- _quorumCheckStatus(ErrorCodes::InternalError, "Not executed") {
- }
-
- void CheckQuorumTest::setUp() {
- _net = new NetworkInterfaceMock;
- _storage = new StorageInterfaceMock;
- _executor.reset(new ReplicationExecutor(_net, _storage, 1 /* prng */ ));
- _executorThread.reset(new stdx::thread(stdx::bind(&ReplicationExecutor::run,
- _executor.get())));
- }
-
- void CheckQuorumTest::tearDown() {
- _executor->shutdown();
- _executorThread->join();
- }
-
- void CheckQuorumTest::startQuorumCheck(const ReplicaSetConfig& config, int myIndex) {
- ASSERT_FALSE(_quorumCheckThread);
- _isQuorumCheckDone = false;
- _quorumCheckThread.reset(new stdx::thread(stdx::bind(&CheckQuorumTest::_runQuorumCheck,
- this,
- config,
- myIndex)));
- }
-
- Status CheckQuorumTest::waitForQuorumCheck() {
- ASSERT_TRUE(_quorumCheckThread);
- _quorumCheckThread->join();
- return _quorumCheckStatus;
+using executor::NetworkInterfaceMock;
+
+class CheckQuorumTest : public mongo::unittest::Test {
+protected:
+ CheckQuorumTest();
+
+ void startQuorumCheck(const ReplicaSetConfig& config, int myIndex);
+ Status waitForQuorumCheck();
+ bool isQuorumCheckDone();
+
+ NetworkInterfaceMock* _net;
+ StorageInterfaceMock* _storage;
+ std::unique_ptr<ReplicationExecutor> _executor;
+
+private:
+ void setUp();
+ void tearDown();
+
+ void _runQuorumCheck(const ReplicaSetConfig& config, int myIndex);
+ virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) = 0;
+
+ std::unique_ptr<stdx::thread> _executorThread;
+ std::unique_ptr<stdx::thread> _quorumCheckThread;
+ Status _quorumCheckStatus;
+ stdx::mutex _mutex;
+ bool _isQuorumCheckDone;
+};
+
+CheckQuorumTest::CheckQuorumTest()
+ : _quorumCheckStatus(ErrorCodes::InternalError, "Not executed") {}
+
+void CheckQuorumTest::setUp() {
+ _net = new NetworkInterfaceMock;
+ _storage = new StorageInterfaceMock;
+ _executor.reset(new ReplicationExecutor(_net, _storage, 1 /* prng */));
+ _executorThread.reset(new stdx::thread(stdx::bind(&ReplicationExecutor::run, _executor.get())));
+}
+
+void CheckQuorumTest::tearDown() {
+ _executor->shutdown();
+ _executorThread->join();
+}
+
+void CheckQuorumTest::startQuorumCheck(const ReplicaSetConfig& config, int myIndex) {
+ ASSERT_FALSE(_quorumCheckThread);
+ _isQuorumCheckDone = false;
+ _quorumCheckThread.reset(
+ new stdx::thread(stdx::bind(&CheckQuorumTest::_runQuorumCheck, this, config, myIndex)));
+}
+
+Status CheckQuorumTest::waitForQuorumCheck() {
+ ASSERT_TRUE(_quorumCheckThread);
+ _quorumCheckThread->join();
+ return _quorumCheckStatus;
+}
+
+bool CheckQuorumTest::isQuorumCheckDone() {
+ stdx::lock_guard<stdx::mutex> lk(_mutex);
+ return _isQuorumCheckDone;
+}
+
+void CheckQuorumTest::_runQuorumCheck(const ReplicaSetConfig& config, int myIndex) {
+ _quorumCheckStatus = _runQuorumCheckImpl(config, myIndex);
+ stdx::lock_guard<stdx::mutex> lk(_mutex);
+ _isQuorumCheckDone = true;
+}
+
+class CheckQuorumForInitiate : public CheckQuorumTest {
+private:
+ virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) {
+ return checkQuorumForInitiate(_executor.get(), config, myIndex);
}
+};
- bool CheckQuorumTest::isQuorumCheckDone() {
- stdx::lock_guard<stdx::mutex> lk(_mutex);
- return _isQuorumCheckDone;
+class CheckQuorumForReconfig : public CheckQuorumTest {
+protected:
+ virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) {
+ return checkQuorumForReconfig(_executor.get(), config, myIndex);
}
-
- void CheckQuorumTest::_runQuorumCheck(const ReplicaSetConfig& config, int myIndex) {
- _quorumCheckStatus = _runQuorumCheckImpl(config, myIndex);
- stdx::lock_guard<stdx::mutex> lk(_mutex);
- _isQuorumCheckDone = true;
- }
-
- class CheckQuorumForInitiate : public CheckQuorumTest {
- private:
- virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) {
- return checkQuorumForInitiate(_executor.get(), config, myIndex);
- }
- };
-
- class CheckQuorumForReconfig : public CheckQuorumTest {
- protected:
- virtual Status _runQuorumCheckImpl(const ReplicaSetConfig& config, int myIndex) {
- return checkQuorumForReconfig(_executor.get(), config, myIndex);
- }
- };
-
- ReplicaSetConfig assertMakeRSConfig(const BSONObj& configBson) {
- ReplicaSetConfig config;
- ASSERT_OK(config.initialize(configBson));
- ASSERT_OK(config.validate());
- return config;
+};
+
+ReplicaSetConfig assertMakeRSConfig(const BSONObj& configBson) {
+ ReplicaSetConfig config;
+ ASSERT_OK(config.initialize(configBson));
+ ASSERT_OK(config.validate());
+ return config;
+}
+
+TEST_F(CheckQuorumForInitiate, ValidSingleNodeSet) {
+ ReplicaSetConfig config = assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1"))));
+ startQuorumCheck(config, 0);
+ ASSERT_OK(waitForQuorumCheck());
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckCanceledByShutdown) {
+ _executor->shutdown();
+ ReplicaSetConfig config = assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1"))));
+ startQuorumCheck(config, 0);
+ ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, waitForQuorumCheck());
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSeveralDownNodes) {
+ // In this test, "we" are host "h3:1". All other nodes time out on
+ // their heartbeat request, and so the quorum check for initiate
+ // will fail because some members were unavailable.
+ ReplicaSetConfig config =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ startQuorumCheck(config, 2);
+ _net->enterNetwork();
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = config.getNumMembers() - 1;
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ _net->scheduleResponse(_net->getNextReadyRequest(),
+ startDate + Milliseconds(10),
+ ResponseStatus(ErrorCodes::NoSuchKey, "No reply"));
}
-
- TEST_F(CheckQuorumForInitiate, ValidSingleNodeSet) {
- ReplicaSetConfig config = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1"))));
- startQuorumCheck(config, 0);
- ASSERT_OK(waitForQuorumCheck());
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ ASSERT_EQUALS(startDate + Milliseconds(10), _net->now());
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
+ ASSERT_REASON_CONTAINS(
+ status, "replSetInitiate quorum check failed because not all proposed set members");
+ ASSERT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_REASON_CONTAINS(status, "h5:1");
+}
+
+const BSONObj makeHeartbeatRequest(const ReplicaSetConfig& rsConfig, int myConfigIndex) {
+ const MemberConfig& myConfig = rsConfig.getMemberAt(myConfigIndex);
+ ReplSetHeartbeatArgs hbArgs;
+ hbArgs.setSetName(rsConfig.getReplSetName());
+ hbArgs.setProtocolVersion(1);
+ hbArgs.setConfigVersion(rsConfig.getConfigVersion());
+ hbArgs.setCheckEmpty(rsConfig.getConfigVersion() == 1);
+ hbArgs.setSenderHost(myConfig.getHostAndPort());
+ hbArgs.setSenderId(myConfig.getId());
+ return hbArgs.toBSON();
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckSuccessForFiveNodes) {
+ // In this test, "we" are host "h3:1". All nodes respond successfully to their heartbeat
+ // requests, and the quorum check succeeds.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
}
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckCanceledByShutdown) {
- _executor->shutdown();
- ReplicaSetConfig config = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1"))));
- startQuorumCheck(config, 0);
- ASSERT_EQUALS(ErrorCodes::ShutdownInProgress, waitForQuorumCheck());
- }
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSeveralDownNodes) {
- // In this test, "we" are host "h3:1". All other nodes time out on
- // their heartbeat request, and so the quorum check for initiate
- // will fail because some members were unavailable.
- ReplicaSetConfig config = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- startQuorumCheck(config, 2);
- _net->enterNetwork();
- const Date_t startDate = _net->now();
- const int numCommandsExpected = config.getNumMembers() - 1;
- for (int i = 0; i < numCommandsExpected; ++i) {
- _net->scheduleResponse(_net->getNextReadyRequest(),
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ ASSERT_OK(waitForQuorumCheck());
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToOneDownNode) {
+ // In this test, "we" are host "h3:1". All nodes except "h2:1" respond
+ // successfully to their heartbeat requests, but quorum check fails because
+ // all nodes must be available for initiate. This is so even though "h2"
+ // is neither voting nor electable.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1"
+ << "priority" << 0 << "votes" << 0)
+ << BSON("_id" << 3 << "host"
+ << "h3:1") << BSON("_id" << 4 << "host"
+ << "h4:1")
+ << BSON("_id" << 5 << "host"
+ << "h5:1") << BSON("_id" << 6 << "host"
+ << "h6:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h2", 1)) {
+ _net->scheduleResponse(noi,
startDate + Milliseconds(10),
- ResponseStatus(ErrorCodes::NoSuchKey, "No reply"));
+ ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
+ } else {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- ASSERT_EQUALS(startDate + Milliseconds(10), _net->now());
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
- ASSERT_REASON_CONTAINS(
- status, "replSetInitiate quorum check failed because not all proposed set members");
- ASSERT_REASON_CONTAINS(status, "h1:1");
- ASSERT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_REASON_CONTAINS(status, "h4:1");
- ASSERT_REASON_CONTAINS(status, "h5:1");
}
-
- const BSONObj makeHeartbeatRequest(const ReplicaSetConfig& rsConfig, int myConfigIndex) {
- const MemberConfig& myConfig = rsConfig.getMemberAt(myConfigIndex);
- ReplSetHeartbeatArgs hbArgs;
- hbArgs.setSetName(rsConfig.getReplSetName());
- hbArgs.setProtocolVersion(1);
- hbArgs.setConfigVersion(rsConfig.getConfigVersion());
- hbArgs.setCheckEmpty(rsConfig.getConfigVersion() == 1);
- hbArgs.setSenderHost(myConfig.getHostAndPort());
- hbArgs.setSenderId(myConfig.getId());
- return hbArgs.toBSON();
- }
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckSuccessForFiveNodes) {
- // In this test, "we" are host "h3:1". All nodes respond successfully to their heartbeat
- // requests, and the quorum check succeeds.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
+ ASSERT_REASON_CONTAINS(
+ status, "replSetInitiate quorum check failed because not all proposed set members");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h6:1");
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSetNameMismatch) {
+ // In this test, "we" are host "h3:1". All nodes respond
+ // successfully to their heartbeat requests, but quorum check fails because
+ // "h4" declares that the requested replica set name was not what it expected.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h4", 1)) {
_net->scheduleResponse(noi,
startDate + Milliseconds(10),
ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1), Milliseconds(8))));
+ BSON("ok" << 0 << "mismatch" << true), Milliseconds(8))));
+ } else {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- ASSERT_OK(waitForQuorumCheck());
}
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToOneDownNode) {
- // In this test, "we" are host "h3:1". All nodes except "h2:1" respond
- // successfully to their heartbeat requests, but quorum check fails because
- // all nodes must be available for initiate. This is so even though "h2"
- // is neither voting nor electable.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1" <<
- "priority" << 0 << "votes" << 0) <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1") <<
- BSON("_id" << 6 << "host" << "h6:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h2", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1), Milliseconds(8))));
- }
- }
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
- ASSERT_REASON_CONTAINS(
- status, "replSetInitiate quorum check failed because not all proposed set members");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h6:1");
- }
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToSetNameMismatch) {
- // In this test, "we" are host "h3:1". All nodes respond
- // successfully to their heartbeat requests, but quorum check fails because
- // "h4" declares that the requested replica set name was not what it expected.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h4", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 0 << "mismatch" << true),
- Milliseconds(8))));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1), Milliseconds(8))));
- }
- }
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
- ASSERT_REASON_CONTAINS(
- status, "Our set name did not match");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_REASON_CONTAINS(status, "h4:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
- }
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNode) {
- // In this test, "we" are host "h3:1". All nodes respond
- // successfully to their heartbeat requests, but quorum check fails because
- // "h5" declares that it is already initialized.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h5", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 0 <<
- "set" << "rs0" <<
- "v" << 1),
- Milliseconds(8))));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1), Milliseconds(8))));
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
+ ASSERT_REASON_CONTAINS(status, "Our set name did not match");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNode) {
+ // In this test, "we" are host "h3:1". All nodes respond
+ // successfully to their heartbeat requests, but quorum check fails because
+ // "h5" declares that it is already initialized.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h5", 1)) {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set"
+ << "rs0"
+ << "v" << 1),
+ Milliseconds(8))));
+ } else {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
- ASSERT_REASON_CONTAINS(
- status, "Our config version of");
- ASSERT_REASON_CONTAINS(
- status, "is no larger than the version");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
- ASSERT_REASON_CONTAINS(status, "h5:1");
}
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNodeOnlyOneRespondent) {
- // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes,
- // and quorum check fails because "h5" declares that it is already initialized.
- //
- // Compare to QuorumCheckFailedDueToInitializedNode, above.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h5", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 0 <<
- "set" << "rs0" <<
- "v" << 1),
- Milliseconds(8))));
- }
- else {
- _net->blackHole(noi);
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
+ ASSERT_REASON_CONTAINS(status, "Our config version of");
+ ASSERT_REASON_CONTAINS(status, "is no larger than the version");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_REASON_CONTAINS(status, "h5:1");
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToInitializedNodeOnlyOneRespondent) {
+ // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes,
+ // and quorum check fails because "h5" declares that it is already initialized.
+ //
+ // Compare to QuorumCheckFailedDueToInitializedNode, above.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h5", 1)) {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set"
+ << "rs0"
+ << "v" << 1),
+ Milliseconds(8))));
+ } else {
+ _net->blackHole(noi);
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
- ASSERT_REASON_CONTAINS(
- status, "Our config version of");
- ASSERT_REASON_CONTAINS(
- status, "is no larger than the version");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
- ASSERT_REASON_CONTAINS(status, "h5:1");
}
-
- TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToNodeWithData) {
- // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes,
- // and quorum check fails because "h5" declares that it has data already.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 1 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1") <<
- BSON("_id" << 5 << "host" << "h5:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- ReplSetHeartbeatResponse hbResp;
- hbResp.setConfigVersion(0);
- hbResp.noteHasData();
- if (request.target == HostAndPort("h5", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- hbResp.toBSON(false),
- Milliseconds(8))));
- }
- else {
- _net->blackHole(noi);
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
+ ASSERT_REASON_CONTAINS(status, "Our config version of");
+ ASSERT_REASON_CONTAINS(status, "is no larger than the version");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_REASON_CONTAINS(status, "h5:1");
+}
+
+TEST_F(CheckQuorumForInitiate, QuorumCheckFailedDueToNodeWithData) {
+ // In this test, "we" are host "h3:1". Only node "h5" responds before the test completes,
+ // and quorum check fails because "h5" declares that it has data already.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 1 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1") << BSON("_id" << 5 << "host"
+ << "h5:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ ReplSetHeartbeatResponse hbResp;
+ hbResp.setConfigVersion(0);
+ hbResp.noteHasData();
+ if (request.target == HostAndPort("h5", 1)) {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(hbResp.toBSON(false), Milliseconds(8))));
+ } else {
+ _net->blackHole(noi);
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::CannotInitializeNodeWithData, status);
- ASSERT_REASON_CONTAINS(
- status, "has data already");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
- ASSERT_REASON_CONTAINS(status, "h5:1");
}
- TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToHigherConfigVersion) {
- // In this test, "we" are host "h3:1". The request to "h2" does not arrive before the end
- // of the test, and the request to "h1" comes back indicating a higher config version.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 2 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h1", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 0 <<
- "set" << "rs0" <<
- "v" << 5),
- Milliseconds(8))));
- }
- else {
- _net->blackHole(noi);
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::CannotInitializeNodeWithData, status);
+ ASSERT_REASON_CONTAINS(status, "has data already");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_REASON_CONTAINS(status, "h5:1");
+}
+TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToHigherConfigVersion) {
+ // In this test, "we" are host "h3:1". The request to "h2" does not arrive before the end
+ // of the test, and the request to "h1" comes back indicating a higher config version.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 2 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h1", 1)) {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 0 << "set"
+ << "rs0"
+ << "v" << 5),
+ Milliseconds(8))));
+ } else {
+ _net->blackHole(noi);
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
- ASSERT_REASON_CONTAINS(
- status, "Our config version of");
- ASSERT_REASON_CONTAINS(
- status, "is no larger than the version");
- ASSERT_REASON_CONTAINS(status, "h1:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
}
-
- TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToIncompatibleSetName) {
- // In this test, "we" are host "h3:1". The request to "h1" times out,
- // and the request to "h2" comes back indicating an incompatible set name.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 2 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1"))));
- const int myConfigIndex = 2;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h2", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 0 << "mismatch" << true),
- Milliseconds(8))));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
+ ASSERT_REASON_CONTAINS(status, "Our config version of");
+ ASSERT_REASON_CONTAINS(status, "is no larger than the version");
+ ASSERT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+}
+
+TEST_F(CheckQuorumForReconfig, QuorumCheckVetoedDueToIncompatibleSetName) {
+ // In this test, "we" are host "h3:1". The request to "h1" times out,
+ // and the request to "h2" comes back indicating an incompatible set name.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 2 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1"))));
+ const int myConfigIndex = 2;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h2", 1)) {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(
+ BSON("ok" << 0 << "mismatch" << true), Milliseconds(8))));
+ } else {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
- ASSERT_REASON_CONTAINS(status, "Our set name did not match");
- ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
- ASSERT_REASON_CONTAINS(status, "h2:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
-
}
-
- TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToInsufficientVoters) {
- // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are voters,
- // and of the voters, only "h1" responds. As a result, quorum check fails.
- // "h5" also responds, but because it cannot vote, is irrelevant for the reconfig
- // quorum check.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 2 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1" << "votes" << 0) <<
- BSON("_id" << 5 << "host" << "h5:1" << "votes" << 0))));
- const int myConfigIndex = 3;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h5", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1),
- Milliseconds(8))));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NewReplicaSetConfigurationIncompatible, status);
+ ASSERT_REASON_CONTAINS(status, "Our set name did not match");
+ ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_REASON_CONTAINS(status, "h2:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+}
+
+TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToInsufficientVoters) {
+ // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are voters,
+ // and of the voters, only "h1" responds. As a result, quorum check fails.
+ // "h5" also responds, but because it cannot vote, is irrelevant for the reconfig
+ // quorum check.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 2 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1"
+ << "votes" << 0)
+ << BSON("_id" << 5 << "host"
+ << "h5:1"
+ << "votes" << 0))));
+ const int myConfigIndex = 3;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h5", 1)) {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
+ } else {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
- ASSERT_REASON_CONTAINS(status, "not enough voting nodes responded; required 2 but only");
- ASSERT_REASON_CONTAINS(status, "h1:1");
- ASSERT_REASON_CONTAINS(status, "h2:1 failed with");
- ASSERT_REASON_CONTAINS(status, "h3:1 failed with");
- ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
- ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
}
-
- TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToNoElectableNodeResponding) {
- // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are electable,
- // and none of them respond.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 2 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1" << "priority" << 0) <<
- BSON("_id" << 5 << "host" << "h5:1" << "priority" << 0))));
- const int myConfigIndex = 3;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h5", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1),
- Milliseconds(8))));
- }
- else {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
+ ASSERT_REASON_CONTAINS(status, "not enough voting nodes responded; required 2 but only");
+ ASSERT_REASON_CONTAINS(status, "h1:1");
+ ASSERT_REASON_CONTAINS(status, "h2:1 failed with");
+ ASSERT_REASON_CONTAINS(status, "h3:1 failed with");
+ ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
+ ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
+}
+
+TEST_F(CheckQuorumForReconfig, QuorumCheckFailsDueToNoElectableNodeResponding) {
+ // In this test, "we" are host "h4". Only "h1", "h2" and "h3" are electable,
+ // and none of them respond.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 2 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1"
+ << "priority" << 0)
+ << BSON("_id" << 5 << "host"
+ << "h5:1"
+ << "priority" << 0))));
+ const int myConfigIndex = 3;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h5", 1)) {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
+ } else {
+ _net->scheduleResponse(noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(ErrorCodes::NoSuchKey, "No response"));
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- Status status = waitForQuorumCheck();
- ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
- ASSERT_REASON_CONTAINS(status, "no electable nodes responded");
}
-
- TEST_F(CheckQuorumForReconfig, QuorumCheckSucceedsWithAsSoonAsPossible) {
- // In this test, "we" are host "h4". Only "h1", "h2" and "h3" can vote.
- // This test should succeed as soon as h1 and h2 respond, so we block
- // h3 and h5 from responding or timing out until the test completes.
-
- const ReplicaSetConfig rsConfig = assertMakeRSConfig(
- BSON("_id" << "rs0" <<
- "version" << 2 <<
- "members" << BSON_ARRAY(
- BSON("_id" << 1 << "host" << "h1:1") <<
- BSON("_id" << 2 << "host" << "h2:1") <<
- BSON("_id" << 3 << "host" << "h3:1") <<
- BSON("_id" << 4 << "host" << "h4:1" << "votes" << 0) <<
- BSON("_id" << 5 << "host" << "h5:1" << "votes" << 0))));
- const int myConfigIndex = 3;
- const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
-
- startQuorumCheck(rsConfig, myConfigIndex);
- const Date_t startDate = _net->now();
- const int numCommandsExpected = rsConfig.getNumMembers() - 1;
- unordered_set<HostAndPort> seenHosts;
- _net->enterNetwork();
- for (int i = 0; i < numCommandsExpected; ++i) {
- const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
- const RemoteCommandRequest& request = noi->getRequest();
- ASSERT_EQUALS("admin", request.dbname);
- ASSERT_EQUALS(hbRequest, request.cmdObj);
- ASSERT(seenHosts.insert(request.target).second) <<
- "Already saw " << request.target.toString();
- if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h2", 1)) {
- _net->scheduleResponse(noi,
- startDate + Milliseconds(10),
- ResponseStatus(RemoteCommandResponse(
- BSON("ok" << 1),
- Milliseconds(8))));
- }
- else {
- _net->blackHole(noi);
- }
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ Status status = waitForQuorumCheck();
+ ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
+ ASSERT_REASON_CONTAINS(status, "no electable nodes responded");
+}
+
+TEST_F(CheckQuorumForReconfig, QuorumCheckSucceedsWithAsSoonAsPossible) {
+ // In this test, "we" are host "h4". Only "h1", "h2" and "h3" can vote.
+ // This test should succeed as soon as h1 and h2 respond, so we block
+ // h3 and h5 from responding or timing out until the test completes.
+
+ const ReplicaSetConfig rsConfig =
+ assertMakeRSConfig(BSON("_id"
+ << "rs0"
+ << "version" << 2 << "members"
+ << BSON_ARRAY(BSON("_id" << 1 << "host"
+ << "h1:1")
+ << BSON("_id" << 2 << "host"
+ << "h2:1") << BSON("_id" << 3 << "host"
+ << "h3:1")
+ << BSON("_id" << 4 << "host"
+ << "h4:1"
+ << "votes" << 0)
+ << BSON("_id" << 5 << "host"
+ << "h5:1"
+ << "votes" << 0))));
+ const int myConfigIndex = 3;
+ const BSONObj hbRequest = makeHeartbeatRequest(rsConfig, myConfigIndex);
+
+ startQuorumCheck(rsConfig, myConfigIndex);
+ const Date_t startDate = _net->now();
+ const int numCommandsExpected = rsConfig.getNumMembers() - 1;
+ unordered_set<HostAndPort> seenHosts;
+ _net->enterNetwork();
+ for (int i = 0; i < numCommandsExpected; ++i) {
+ const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+ const RemoteCommandRequest& request = noi->getRequest();
+ ASSERT_EQUALS("admin", request.dbname);
+ ASSERT_EQUALS(hbRequest, request.cmdObj);
+ ASSERT(seenHosts.insert(request.target).second) << "Already saw "
+ << request.target.toString();
+ if (request.target == HostAndPort("h1", 1) || request.target == HostAndPort("h2", 1)) {
+ _net->scheduleResponse(
+ noi,
+ startDate + Milliseconds(10),
+ ResponseStatus(RemoteCommandResponse(BSON("ok" << 1), Milliseconds(8))));
+ } else {
+ _net->blackHole(noi);
}
- _net->runUntil(startDate + Milliseconds(10));
- _net->exitNetwork();
- ASSERT_OK(waitForQuorumCheck());
}
+ _net->runUntil(startDate + Milliseconds(10));
+ _net->exitNetwork();
+ ASSERT_OK(waitForQuorumCheck());
+}
} // namespace
} // namespace repl
View Lorry Controller Status