diff options
-rw-r--r-- | jstests/slowNightly/repl_monitor_stress.js | 123 | ||||
-rw-r--r-- | src/mongo/dbtests/mock/mock_replica_set.cpp | 190 | ||||
-rw-r--r-- | src/mongo/dbtests/mock/mock_replica_set.h | 34 | ||||
-rw-r--r-- | src/mongo/dbtests/mock_replica_set_test.cpp | 107 | ||||
-rw-r--r-- | src/mongo/dbtests/replica_set_monitor_test.cpp | 56 |
5 files changed, 297 insertions, 213 deletions
diff --git a/jstests/slowNightly/repl_monitor_stress.js b/jstests/slowNightly/repl_monitor_stress.js deleted file mode 100644 index 005a8eb9572..00000000000 --- a/jstests/slowNightly/repl_monitor_stress.js +++ /dev/null @@ -1,123 +0,0 @@ -/** - * Stress test for ReplicaSetMonitor. Basically test removing a node - * from the set with different nodes as the current primary and makes - * sure ReplSetMonitor doesn't crash. - */ - -var NODE_COUNT = 5; -var st = new ShardingTest({ shards: { rs0: { nodes: NODE_COUNT, oplogSize: 10 }}, - separateConfig: true }); -var replTest = st.rs0; -var mongos = st.s; - -// prevent the balancer from talking to the shards, which can trigger a replica set -// view refresh -st.stopBalancer(); - -mongos.getDB('test').user.find().explain(); - -// Iterate from the last node first as this is the index that is most prone to error -for (var node = NODE_COUNT - 1; node >= 0; node--) { - jsTest.log('Iteration to remove node ' + node + '/' + (NODE_COUNT - 1)); - var connPoolStats = mongos.getDB('admin').runCommand({ connPoolStats: 1 }); - var targetHostName = connPoolStats['replicaSets'][replTest.name].hosts[node].addr; - - var priConn = replTest.getPrimary(); - var origConfDoc = priConn.getDB('local').system.replset.findOne(); - var confDoc = {}; - - // Do a deep copy of the config doc - Object.extend(confDoc, origConfDoc, true); - - // Force target to become primary - for (var idx = 0; idx < confDoc.members.length; idx++) { - if (confDoc.members[idx].host == targetHostName) { - confDoc.members[idx].priority = 100; - } - else { - confDoc.members[idx].priority = 1; - } - } - - confDoc.version++; - - jsTest.log('Changing conf to ' + tojson(confDoc)); - - try { - priConn.getDB('admin').adminCommand({ replSetReconfig: confDoc }); - } catch (x) { - print('Expected exception because of reconfig' + x); - } - - assert.soon(function() { - var connPoolStats = mongos.getDB('admin').runCommand({ connPoolStats: 1 }); - var replView = connPoolStats.replicaSets[replTest.name]; - print('Current replView: ' + tojson(replView)); - return replView.master == node; - }, 'timed out waiting for node ' + node + ' to become master', 60000); - - // Remove first node from set - confDoc.members.shift(); - confDoc.version++; - - jsTest.log('Removing node, new conf: ' + tojson(confDoc)); - - assert.soon(function() { - try { - replTest.getPrimary().getDB('admin').adminCommand({ replSetReconfig: confDoc }); - var newConf = replTest.getPrimary().getDB('local').system.replset.findOne(); - print('Current config after attempting to remove: ' + tojson(newConf)); - return newConf.members.length == confDoc.members.length; - } catch (x) { - print('Expected exception because of reconfig: ' + x); - return false; - } - }, 'timed out trying to remove node from config'); - - var waitNodeCount = function(count) { - var connPoolStats = mongos.getDB('admin').runCommand('connPoolStats'); - var replView = connPoolStats.replicaSets[replTest.name].hosts; - print('current replView: ' + tojson(replView)); - - return replView.length == count; - }; - - assert.soon(function() { - /* TODO: SERVER-5175 - * 1. Trigger/Add sleep right before _master gets updated to the current value - * 2. Trigger/Add sleep right before acquiring the lock inside getMaster. Sleep - * should be long enough to time with ReplicaSetMonitorWatcher refresh and - * not too long to let sleep at #1 expire. - */ - try { - mongos.getDB('test').user.find().explain(); - } catch (x) { - print('query error, try again: ' + x); - } - - return waitNodeCount(NODE_COUNT - 1); - }, 'timed out waiting for node to be removed', 60000); - - jsTest.log(tojson(mongos.getDB('admin').runCommand('connPoolStats'))); - - origConfDoc.version = confDoc.version + 1; - jsTest.log('Put node back, new conf: ' + tojson(origConfDoc)); - - try { - replTest.getPrimary().getDB('admin').adminCommand({ replSetReconfig: origConfDoc }); - } catch (x) { - print('Expected exception because of replSetReconfig: ' + x); - } - - replTest.awaitSecondaryNodes(); - - // Make sure mongos view of replica set is in steady state before proceeding - assert.soon(function() { return waitNodeCount(NODE_COUNT); }, - 'timed out waiting for node to get back to set', 60000); -} - -// Make sure that mongos did not crash -assert(mongos.getDB('admin').runCommand({ serverStatus: 1 }).ok); - -st.stop(); - diff --git a/src/mongo/dbtests/mock/mock_replica_set.cpp b/src/mongo/dbtests/mock/mock_replica_set.cpp index 331a0bc7331..c266aed0a83 100644 --- a/src/mongo/dbtests/mock/mock_replica_set.cpp +++ b/src/mongo/dbtests/mock/mock_replica_set.cpp @@ -22,16 +22,10 @@ #include <sstream> -using mongo::BSONObjBuilder; - -using std::map; -using std::string; -using std::vector; - namespace mongo { MockReplicaSet::MockReplicaSet(const string& setName, size_t nodes): _setName(setName) { - std::vector<mongo::ReplSetConfig::MemberCfg> replConfig; + ReplConfigMap replConfig; for (size_t n = 0; n < nodes; n++) { std::stringstream str; @@ -41,25 +35,22 @@ namespace mongo { if (n == 0) { _primaryHost = hostName; } - else { - _secondaryHosts.push_back(hostName); - } MockRemoteDBServer* mockServer = new MockRemoteDBServer(hostName); _nodeMap[hostName] = mockServer; MockConnRegistry::get()->addServer(mockServer); - mongo::ReplSetConfig::MemberCfg config; - config.h = mongo::HostAndPort(hostName); - replConfig.push_back(config); + ReplSetConfig::MemberCfg config; + config.h = HostAndPort(hostName); + replConfig.insert(std::make_pair(hostName, config)); } setConfig(replConfig); } MockReplicaSet::~MockReplicaSet() { - for (std::map<string, MockRemoteDBServer*>::iterator iter = _nodeMap.begin(); + for (ReplNodeMap::iterator iter = _nodeMap.begin(); iter != _nodeMap.end(); ++iter) { MockConnRegistry::get()->removeServer(iter->second->getServerAddress()); delete iter->second; @@ -75,7 +66,7 @@ namespace mongo { str << _setName; str << "/"; - std::map<string, MockRemoteDBServer*>::const_iterator iter = _nodeMap.begin(); + ReplNodeMap::const_iterator iter = _nodeMap.begin(); while (iter != _nodeMap.end()) { str << iter->second->getServerAddress(); ++iter; @@ -88,12 +79,12 @@ namespace mongo { return str.str(); } - vector<mongo::HostAndPort> MockReplicaSet::getHosts() const { - vector<mongo::HostAndPort> list; + vector<HostAndPort> MockReplicaSet::getHosts() const { + vector<HostAndPort> list; - for (std::map<string, MockRemoteDBServer*>::const_iterator iter = _nodeMap.begin(); + for (ReplNodeMap::const_iterator iter = _nodeMap.begin(); iter != _nodeMap.end(); ++iter) { - list.push_back(mongo::HostAndPort(iter->second->getServerAddress())); + list.push_back(HostAndPort(iter->second->getServerAddress())); } return list; @@ -103,27 +94,49 @@ namespace mongo { return _primaryHost; } - const vector<string>& MockReplicaSet::getSecondaries() const { - return _secondaryHosts; + void MockReplicaSet::setPrimary(const string& hostAndPort) { + ReplConfigMap::const_iterator iter = _replConfig.find(hostAndPort); + fassert(16578, iter != _replConfig.end()); + + const ReplSetConfig::MemberCfg& config = iter->second; + fassert(16579, !config.hidden && config.priority > 0 && !config.arbiterOnly); + + _primaryHost = hostAndPort; + + mockIsMasterCmd(); + mockReplSetGetStatusCmd(); + } + + vector<string> MockReplicaSet::getSecondaries() const { + vector<string> secondaries; + + for (ReplConfigMap::const_iterator iter = _replConfig.begin(); + iter != _replConfig.end(); ++iter) { + if (iter->first != _primaryHost) { + secondaries.push_back(iter->first); + } + } + + return secondaries; } - MockRemoteDBServer* MockReplicaSet::getNode(const string& hostName) { - return mapFindWithDefault(_nodeMap, hostName, static_cast<MockRemoteDBServer*>(NULL)); + MockRemoteDBServer* MockReplicaSet::getNode(const string& hostAndPort) { + return mapFindWithDefault(_nodeMap, hostAndPort, static_cast<MockRemoteDBServer*>(NULL)); } - const vector<mongo::ReplSetConfig::MemberCfg>& MockReplicaSet::getReplConfig() const { + MockReplicaSet::ReplConfigMap MockReplicaSet::getReplConfig() const { return _replConfig; } - void MockReplicaSet::setConfig(const vector<mongo::ReplSetConfig::MemberCfg>& newConfig) { + void MockReplicaSet::setConfig(const MockReplicaSet::ReplConfigMap& newConfig) { _replConfig = newConfig; mockIsMasterCmd(); mockReplSetGetStatusCmd(); } - void MockReplicaSet::kill(const string& hostName) { - verify(_nodeMap.count(hostName) == 1); - _nodeMap[hostName]->shutdown(); + void MockReplicaSet::kill(const string& hostAndPort) { + verify(_nodeMap.count(hostAndPort) == 1); + _nodeMap[hostAndPort]->shutdown(); } void MockReplicaSet::kill(const vector<string>& hostList) { @@ -133,89 +146,106 @@ namespace mongo { } } - void MockReplicaSet::restore(const string& hostName) { - verify(_nodeMap.count(hostName) == 1); - _nodeMap[hostName]->reboot(); + void MockReplicaSet::restore(const string& hostAndPort) { + verify(_nodeMap.count(hostAndPort) == 1); + _nodeMap[hostAndPort]->reboot(); } void MockReplicaSet::mockIsMasterCmd() { // Copied from ReplSetImpl::_fillIsMaster - for (vector<mongo::ReplSetConfig::MemberCfg>::iterator iter = _replConfig.begin(); - iter != _replConfig.end(); ++iter) { - const string hostName(iter->h.toString(true)); + for (ReplNodeMap::iterator nodeIter = _nodeMap.begin(); + nodeIter != _nodeMap.end(); ++nodeIter) { + const string& hostAndPort = nodeIter->first; + BSONObjBuilder builder; builder.append("setName", _setName); - const bool isPrimary = hostName == getPrimary(); - builder.append("ismaster", isPrimary); - builder.append("secondary", !isPrimary); + ReplConfigMap::const_iterator configIter = _replConfig.find(hostAndPort); + if (configIter == _replConfig.end()) { + builder.append("ismaster", false); + builder.append("secondary", false); - { - // TODO: add passives & arbiters vector<string> hostList; - hostList.push_back(getPrimary()); - for (vector<string>::const_iterator secIter = getSecondaries().begin(); - secIter != getSecondaries().end(); ++secIter) { - hostList.push_back(*secIter); - } - builder.append("hosts", hostList); } + else { + const bool isPrimary = hostAndPort == getPrimary(); + builder.append("ismaster", isPrimary); + builder.append("secondary", !isPrimary); + + { + // TODO: add passives & arbiters + vector<string> hostList; + hostList.push_back(getPrimary()); + + const vector<string> secondaries = getSecondaries(); + for (vector<string>::const_iterator secIter = secondaries.begin(); + secIter != secondaries.end(); ++secIter) { + hostList.push_back(*secIter); + } + + builder.append("hosts", hostList); + } - builder.append("primary", getPrimary()); - - if (iter->arbiterOnly) { - builder.append("arbiterOnly", true); - } + builder.append("primary", getPrimary()); - if (iter->priority == 0 && !iter->arbiterOnly) { - builder.append("passive", true); - } + const ReplSetConfig::MemberCfg& replConfig = configIter->second; + if (replConfig.arbiterOnly) { + builder.append("arbiterOnly", true); + } - if (iter->slaveDelay) { - builder.append("slaveDelay", iter->slaveDelay); - } + if (replConfig.priority == 0 && !replConfig.arbiterOnly) { + builder.append("passive", true); + } - if (iter->hidden) { - builder.append("hidden", true); - } + if (replConfig.slaveDelay) { + builder.append("slaveDelay", replConfig.slaveDelay); + } - if (!iter->buildIndexes) { - builder.append("buildIndexes", false); - } + if (replConfig.hidden) { + builder.append("hidden", true); + } - if(!iter->tags.empty()) { - BSONObjBuilder tagBuilder; - for(map<string, string>::const_iterator tagIter = iter->tags.begin(); - tagIter != iter->tags.end(); tagIter++) { - tagBuilder.append(tagIter->first, tagIter->second); + if (!replConfig.buildIndexes) { + builder.append("buildIndexes", false); } - builder.append("tags", tagBuilder.done()); + if(!replConfig.tags.empty()) { + BSONObjBuilder tagBuilder; + for(map<string, string>::const_iterator tagIter = replConfig.tags.begin(); + tagIter != replConfig.tags.end(); tagIter++) { + tagBuilder.append(tagIter->first, tagIter->second); + } + + builder.append("tags", tagBuilder.done()); + } } - builder.append("me", hostName); + builder.append("me", hostAndPort); builder.append("ok", true); - getNode(hostName)->setCommandReply("ismaster", builder.done()); + nodeIter->second->setCommandReply("ismaster", builder.done()); } } - int MockReplicaSet::getState(const std::string& host) const { - if (host == getPrimary()) { - return static_cast<int>(mongo::MemberState::RS_PRIMARY); + int MockReplicaSet::getState(const std::string& hostAndPort) const { + if (_replConfig.count(hostAndPort) < 1) { + return static_cast<int>(MemberState::RS_SHUNNED); + } + else if (hostAndPort == getPrimary()) { + return static_cast<int>(MemberState::RS_PRIMARY); } else { - return static_cast<int>(mongo::MemberState::RS_SECONDARY); + return static_cast<int>(MemberState::RS_SECONDARY); } } void MockReplicaSet::mockReplSetGetStatusCmd() { // Copied from ReplSetImpl::_summarizeStatus - for (std::map<string, MockRemoteDBServer*>::iterator nodeIter = _nodeMap.begin(); - nodeIter != _nodeMap.end(); ++nodeIter) { + for (ReplNodeMap::iterator nodeIter = _nodeMap.begin(); + nodeIter != _nodeMap.end(); ++nodeIter) { MockRemoteDBServer* node = nodeIter->second; - vector<mongo::BSONObj> hostsField; + vector<BSONObj> hostsField; BSONObjBuilder fullStatBuilder; @@ -231,9 +261,9 @@ namespace mongo { hostsField.push_back(selfStatBuilder.obj()); } - for (std::map<string, MockRemoteDBServer*>::iterator hostNodeIter = _nodeMap.begin(); - hostNodeIter != _nodeMap.end(); ++hostNodeIter) { - MockRemoteDBServer* hostNode = hostNodeIter->second; + for (ReplConfigMap::const_iterator replConfIter = _replConfig.begin(); + replConfIter != _replConfig.end(); ++replConfIter) { + MockRemoteDBServer* hostNode = getNode(replConfIter->first); if (hostNode == node) { continue; diff --git a/src/mongo/dbtests/mock/mock_replica_set.h b/src/mongo/dbtests/mock/mock_replica_set.h index a628c61c747..baf55db5570 100644 --- a/src/mongo/dbtests/mock/mock_replica_set.h +++ b/src/mongo/dbtests/mock/mock_replica_set.h @@ -27,10 +27,18 @@ namespace mongo { * This is a helper class for managing a replica set consisting of * MockRemoteDBServer instances. * + * Note: Be sure to call ScopedDbConnection::clearPool() after every test + * when doing tests that involves the ReplicaSetMonitor. This is because + * it uses ScopedDbConnection which means you can have a residue connections + * that was created from previous tests and can cause a seg fault if the + * MockRemoteDBServer instances were already destroyed. + * * Warning: Not thread-safe */ class MockReplicaSet { public: + typedef std::map<std::string, ReplSetConfig::MemberCfg> ReplConfigMap; + /** * Creates a mock replica set and automatically mocks the isMaster * and replSetGetStatus commands based on the default replica set @@ -48,30 +56,35 @@ namespace mongo { std::string getSetName() const; std::string getConnectionString() const; - std::vector<mongo::HostAndPort> getHosts() const; - const std::vector<mongo::ReplSetConfig::MemberCfg>& getReplConfig() const; + std::vector<HostAndPort> getHosts() const; + ReplConfigMap getReplConfig() const; std::string getPrimary() const; - const std::vector<std::string>& getSecondaries() const; + std::vector<std::string> getSecondaries() const; /** * Sets the configuration for this replica sets. This also has a side effect * of mocking the ismaster and replSetGetStatus command responses based on * the new config. + * + * Note: does not automatically select a new primary. Can be done manually by + * calling setPrimary. */ - void setConfig(const std::vector<mongo::ReplSetConfig::MemberCfg>& newConfig); + void setConfig(const ReplConfigMap& newConfig); + + void setPrimary(const std::string& hostAndPort); /** * @return pointer to the mocked remote server with the given hostName. * NULL if host doesn't exists. */ - MockRemoteDBServer* getNode(const std::string& hostName); + MockRemoteDBServer* getNode(const std::string& hostAndPort); /** * Kills a node belonging to this set. * * @param hostName the name of the replica node to kill. */ - void kill(const std::string& hostName); + void kill(const std::string& hostAndPort); /** * Kills a set of host belonging to this set. @@ -88,6 +101,8 @@ namespace mongo { void restore(const std::string& hostName); private: + typedef std::map<std::string, MockRemoteDBServer*> ReplNodeMap; + /** * Mocks the ismaster command based on the information on the current * replica set configuration. @@ -103,13 +118,12 @@ namespace mongo { /** * @return the replica set state of the given host */ - int getState(const std::string& host) const; + int getState(const std::string& hostAndPort) const; const std::string _setName; - std::map<std::string, MockRemoteDBServer*> _nodeMap; - std::vector<mongo::ReplSetConfig::MemberCfg> _replConfig; + ReplNodeMap _nodeMap; + ReplConfigMap _replConfig; std::string _primaryHost; - std::vector<std::string> _secondaryHosts; }; } diff --git a/src/mongo/dbtests/mock_replica_set_test.cpp b/src/mongo/dbtests/mock_replica_set_test.cpp index d82c7ff8d30..1a59ddb4718 100644 --- a/src/mongo/dbtests/mock_replica_set_test.cpp +++ b/src/mongo/dbtests/mock_replica_set_test.cpp @@ -243,6 +243,113 @@ namespace mongo_test { ASSERT(expectedMembers == memberList); } + TEST(MockReplicaSetTest, IsMasterReconfigNodeRemoved) { + MockReplicaSet replSet("n", 3); + + MockReplicaSet::ReplConfigMap config = replSet.getReplConfig(); + const string hostToRemove("$n1:27017"); + config.erase(hostToRemove); + replSet.setConfig(config); + + { + // Check isMaster for node still in set + BSONObj cmdResponse; + MockRemoteDBServer* node = replSet.getNode("$n0:27017"); + const MockRemoteDBServer::InstanceID id = node->getInstanceID(); + bool ok = node->runCommand(id, "foo.bar", BSON("ismaster" << 1), cmdResponse); + ASSERT(ok); + + ASSERT(cmdResponse["ismaster"].trueValue()); + ASSERT(!cmdResponse["secondary"].trueValue()); + ASSERT_EQUALS("$n0:27017", cmdResponse["me"].str()); + ASSERT_EQUALS("$n0:27017", cmdResponse["primary"].str()); + ASSERT_EQUALS("n", cmdResponse["setName"].str()); + + set<string> expectedHosts; + expectedHosts.insert("$n0:27017"); + expectedHosts.insert("$n2:27017"); + + set<string> hostList; + BSONObjIterator iter(cmdResponse["hosts"].embeddedObject()); + while (iter.more()) { + hostList.insert(iter.next().str()); + } + + ASSERT(expectedHosts == hostList); + ASSERT(hostList.count(hostToRemove) == 0); + } + + { + // Check isMaster for node still not in set anymore + BSONObj cmdResponse; + MockRemoteDBServer* node = replSet.getNode(hostToRemove); + const MockRemoteDBServer::InstanceID id = node->getInstanceID(); + bool ok = node->runCommand(id, "foo.bar", BSON("ismaster" << 1), cmdResponse); + ASSERT(ok); + + ASSERT(!cmdResponse["ismaster"].trueValue()); + ASSERT(!cmdResponse["secondary"].trueValue()); + ASSERT_EQUALS(hostToRemove, cmdResponse["me"].str()); + ASSERT_EQUALS("n", cmdResponse["setName"].str()); + } + } + + TEST(MockReplicaSetTest, replSetGetStatusReconfigNodeRemoved) { + MockReplicaSet replSet("n", 3); + + MockReplicaSet::ReplConfigMap config = replSet.getReplConfig(); + const string hostToRemove("$n1:27017"); + config.erase(hostToRemove); + replSet.setConfig(config); + + { + // Check replSetGetStatus for node still in set + BSONObj cmdResponse; + MockRemoteDBServer* node = replSet.getNode("$n2:27017"); + const MockRemoteDBServer::InstanceID id = node->getInstanceID(); + bool ok = node->runCommand(id, "foo.bar", BSON("replSetGetStatus" << 1), cmdResponse); + ASSERT(ok); + + ASSERT_EQUALS("n", cmdResponse["set"].str()); + ASSERT_EQUALS(2, cmdResponse["myState"].numberInt()); + + set<string> memberList; + BSONObjIterator iter(cmdResponse["members"].embeddedObject()); + while (iter.more()) { + BSONElement member(iter.next()); + memberList.insert(member["name"].str()); + + if (member["self"].trueValue()) { + ASSERT_EQUALS(2, member["state"].numberInt()); + ASSERT_EQUALS("$n2:27017", member["name"].str()); + } + else if (member["name"].str() == "$n0:27017") { + ASSERT_EQUALS(1, member["state"].numberInt()); + } + else { + ASSERT_EQUALS(2, member["state"].numberInt()); + } + } + + set<string> expectedMembers; + expectedMembers.insert("$n0:27017"); + expectedMembers.insert("$n2:27017"); + ASSERT(expectedMembers == memberList); + } + + { + // Check replSetGetStatus for node still not in set anymore + BSONObj cmdResponse; + MockRemoteDBServer* node = replSet.getNode(hostToRemove); + const MockRemoteDBServer::InstanceID id = node->getInstanceID(); + bool ok = node->runCommand(id, "foo.bar", BSON("replSetGetStatus" << 1), cmdResponse); + ASSERT(ok); + + ASSERT_EQUALS("n", cmdResponse["set"].str()); + ASSERT_EQUALS(10, cmdResponse["myState"].numberInt()); + } + } + TEST(MockReplicaSetTest, KillNode) { MockReplicaSet replSet("n", 3); const string priHostName(replSet.getPrimary()); diff --git a/src/mongo/dbtests/replica_set_monitor_test.cpp b/src/mongo/dbtests/replica_set_monitor_test.cpp index db909a5c205..57ef4756d49 100644 --- a/src/mongo/dbtests/replica_set_monitor_test.cpp +++ b/src/mongo/dbtests/replica_set_monitor_test.cpp @@ -19,6 +19,7 @@ * ReplicaSetMonitor::selectNode and TagSet */ +#include "mongo/client/connpool.h" #include "mongo/client/dbclientinterface.h" #include "mongo/client/dbclient_rs.h" #include "mongo/dbtests/mock/mock_conn_registry.h" @@ -32,14 +33,17 @@ using std::string; using boost::scoped_ptr; using mongo::BSONObj; +using mongo::BSONObjBuilder; using mongo::BSONArray; using mongo::BSONArrayBuilder; +using mongo::BSONElement; using mongo::ConnectionString; using mongo::HostAndPort; using mongo::MockReplicaSet; using mongo::ReadPreference; using mongo::ReplicaSetMonitor; using mongo::ReplicaSetMonitorPtr; +using mongo::ScopedDbConnection; using mongo::TagSet; namespace mongo_test { @@ -1501,6 +1505,8 @@ namespace mongo_test { void tearDown() { ConnectionString::setConnectionHook(_originalConnectionHook); ReplicaSetMonitor::remove(_replSet->getSetName(), true); + _replSet.reset(); + mongo::ScopedDbConnection::clearPool(); } MockReplicaSet* getReplSet() { @@ -1532,4 +1538,54 @@ namespace mongo_test { // Trigger calls to Node::getConnWithRefresh monitor->check(true); } + + // Stress test case for a node that is previously a primary being removed from the set. + // This test goes through configurations with different positions for the primary node + // in the host list returned from the isMaster command. The test here is to make sure + // that the ReplicaSetMonitor will not crash under these situations. + TEST(ReplicaSetMonitorTest, PrimaryRemovedFromSetStress) { + const size_t NODE_COUNT = 5; + MockReplicaSet replSet("test", NODE_COUNT); + ConnectionString::ConnectionHook* originalConnHook = + ConnectionString::getConnectionHook(); + ConnectionString::setConnectionHook(mongo::MockConnRegistry::get()->getConnStrHook()); + + const string replSetName(replSet.getSetName()); + vector<HostAndPort> seedList; + seedList.push_back(HostAndPort(replSet.getPrimary())); + ReplicaSetMonitor::createIfNeeded(replSetName, seedList); + + const MockReplicaSet::ReplConfigMap origConfig = replSet.getReplConfig(); + mongo::ReplicaSetMonitorPtr replMonitor = ReplicaSetMonitor::get(replSetName); + + for (size_t idxToRemove = 0; idxToRemove < NODE_COUNT; idxToRemove++) { + MockReplicaSet::ReplConfigMap newConfig = origConfig; + + replSet.setConfig(origConfig); + replMonitor->check(true); // Make sure the monitor sees the change + + string hostToRemove; + { + BSONObjBuilder monitorStateBuilder; + replMonitor->appendInfo(monitorStateBuilder); + BSONObj monitorState = monitorStateBuilder.done(); + + BSONElement hostsElem = monitorState["hosts"]; + BSONElement addrElem = hostsElem[mongo::str::stream() << idxToRemove]["addr"]; + hostToRemove = addrElem.String(); + } + + replSet.setPrimary(hostToRemove); + replMonitor->check(true); // Make sure the monitor sees the new primary + + newConfig.erase(hostToRemove); + replSet.setConfig(newConfig); + replSet.setPrimary(newConfig.begin()->first); + replMonitor->check(true); // Force refresh -> should not crash + } + + ReplicaSetMonitor::remove(replSet.getSetName(), true); + ConnectionString::setConnectionHook(originalConnHook); + mongo::ScopedDbConnection::clearPool(); + } } |