diff options
Diffstat (limited to 'src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp')
| -rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp | 688 |
1 files changed, 346 insertions, 342 deletions
diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp index a0e4149de24..b187ccd74c2 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_test.cpp @@ -48,367 +48,371 @@ namespace mongo { namespace repl { namespace { - using executor::NetworkInterfaceMock; - - class ReplCoordElectTest : public ReplCoordTest { - protected: - void simulateEnoughHeartbeatsForElectability(); - void simulateFreshEnoughForElectability(); - }; - - void ReplCoordElectTest::simulateEnoughHeartbeatsForElectability() { - ReplicationCoordinatorImpl* replCoord = getReplCoord(); - ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - ReplSetHeartbeatArgs hbArgs; - if (hbArgs.initialize(request.cmdObj).isOK()) { - ReplSetHeartbeatResponse hbResp; - hbResp.setSetName(rsConfig.getReplSetName()); - hbResp.setState(MemberState::RS_SECONDARY); - hbResp.setConfigVersion(rsConfig.getConfigVersion()); - BSONObjBuilder respObj; - respObj << "ok" << 1; - hbResp.addToBSON(&respObj, false); - net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj())); - } - else { - error() << "Black holing unexpected request to " << request.target << ": " << - request.cmdObj; - net->blackHole(noi); - } - net->runReadyNetworkOperations(); +using executor::NetworkInterfaceMock; + +class ReplCoordElectTest : public ReplCoordTest { +protected: + void simulateEnoughHeartbeatsForElectability(); + void simulateFreshEnoughForElectability(); +}; + +void ReplCoordElectTest::simulateEnoughHeartbeatsForElectability() { + ReplicationCoordinatorImpl* replCoord = getReplCoord(); + ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + log() << request.target.toString() << " processing " << request.cmdObj; + ReplSetHeartbeatArgs hbArgs; + if (hbArgs.initialize(request.cmdObj).isOK()) { + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName(rsConfig.getReplSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + BSONObjBuilder respObj; + respObj << "ok" << 1; + hbResp.addToBSON(&respObj, false); + net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj())); + } else { + error() << "Black holing unexpected request to " << request.target << ": " + << request.cmdObj; + net->blackHole(noi); } - net->exitNetwork(); + net->runReadyNetworkOperations(); } + net->exitNetwork(); +} - void ReplCoordElectTest::simulateFreshEnoughForElectability() { - ReplicationCoordinatorImpl* replCoord = getReplCoord(); - ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - if (request.cmdObj.firstElement().fieldNameStringData() == "replSetFresh") { - net->scheduleResponse(noi, net->now(), makeResponseStatus( - BSON("ok" << 1 << - "fresher" << false << - "opTime" << Date_t::fromMillisSinceEpoch( - Timestamp(0, 0).asLL()) << - "veto" << false))); - } - else { - error() << "Black holing unexpected request to " << request.target << ": " << - request.cmdObj; - net->blackHole(noi); - } - net->runReadyNetworkOperations(); +void ReplCoordElectTest::simulateFreshEnoughForElectability() { + ReplicationCoordinatorImpl* replCoord = getReplCoord(); + ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + for (int i = 0; i < rsConfig.getNumMembers() - 1; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + log() << request.target.toString() << " processing " << request.cmdObj; + if (request.cmdObj.firstElement().fieldNameStringData() == "replSetFresh") { + net->scheduleResponse( + noi, + net->now(), + makeResponseStatus(BSON("ok" << 1 << "fresher" << false << "opTime" + << Date_t::fromMillisSinceEpoch(Timestamp(0, 0).asLL()) + << "veto" << false))); + } else { + error() << "Black holing unexpected request to " << request.target << ": " + << request.cmdObj; + net->blackHole(noi); } - net->exitNetwork(); + net->runReadyNetworkOperations(); } + net->exitNetwork(); +} - TEST_F(ReplCoordElectTest, ElectTooSoon) { - logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(3)); - // Election never starts because we haven't set a lastOpTimeApplied value yet, via a - // heartbeat. - startCapturingLogMessages(); - assertStartSuccess( - BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") << - BSON("_id" << 2 << "host" << "node2:12345"))), - HostAndPort("node1", 12345)); - ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - simulateEnoughHeartbeatsForElectability(); - stopCapturingLogMessages(); - ASSERT_EQUALS(1, countLogLinesContaining("node has no applied oplog entries")); - } +TEST_F(ReplCoordElectTest, ElectTooSoon) { + logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(3)); + // Election never starts because we haven't set a lastOpTimeApplied value yet, via a + // heartbeat. + startCapturingLogMessages(); + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345"))), + HostAndPort("node1", 12345)); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + simulateEnoughHeartbeatsForElectability(); + stopCapturingLogMessages(); + ASSERT_EQUALS(1, countLogLinesContaining("node has no applied oplog entries")); +} - /** - * This test checks that an election can happen when only one node is up, and it has the - * vote(s) to win. - */ - TEST_F(ReplCoordElectTest, ElectTwoNodesWithOneZeroVoter) { - OperationContextReplMock txn; - assertStartSuccess( - BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") << - BSON("_id" << 2 << "host" << "node2:12345" << - "votes" << 0 << "hidden" << true << - "priority" << 0))), - HostAndPort("node1", 12345)); - - getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - - ASSERT(getReplCoord()->getMemberState().secondary()) << - getReplCoord()->getMemberState().toString(); - - getReplCoord()->setMyLastOptime(OpTime(Timestamp(10,0), 0)); - - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - // blackhole heartbeat - net->scheduleResponse(noi, - net->now(), - ResponseStatus(ErrorCodes::OperationFailed, "timeout")); - net->runReadyNetworkOperations(); - // blackhole freshness - const NetworkInterfaceMock::NetworkOperationIterator noi2 = net->getNextReadyRequest(); - net->scheduleResponse(noi2, - net->now(), - ResponseStatus(ErrorCodes::OperationFailed, "timeout")); - net->runReadyNetworkOperations(); - net->exitNetwork(); - - ASSERT(getReplCoord()->getMemberState().primary()) << - getReplCoord()->getMemberState().toString(); - ASSERT(getReplCoord()->isWaitingForApplierToDrain()); - - // Since we're still in drain mode, expect that we report ismaster: false, issecondary:true. - IsMasterResponse imResponse; - getReplCoord()->fillIsMasterForReplSet(&imResponse); - ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString(); - ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString(); - getReplCoord()->signalDrainComplete(&txn); - getReplCoord()->fillIsMasterForReplSet(&imResponse); - ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString(); - ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString(); - } +/** + * This test checks that an election can happen when only one node is up, and it has the + * vote(s) to win. + */ +TEST_F(ReplCoordElectTest, ElectTwoNodesWithOneZeroVoter) { + OperationContextReplMock txn; + assertStartSuccess( + BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345" + << "votes" << 0 << "hidden" << true << "priority" << 0))), + HostAndPort("node1", 12345)); + + getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); + + ASSERT(getReplCoord()->getMemberState().secondary()) + << getReplCoord()->getMemberState().toString(); + + getReplCoord()->setMyLastOptime(OpTime(Timestamp(10, 0), 0)); + + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + // blackhole heartbeat + net->scheduleResponse(noi, net->now(), ResponseStatus(ErrorCodes::OperationFailed, "timeout")); + net->runReadyNetworkOperations(); + // blackhole freshness + const NetworkInterfaceMock::NetworkOperationIterator noi2 = net->getNextReadyRequest(); + net->scheduleResponse(noi2, net->now(), ResponseStatus(ErrorCodes::OperationFailed, "timeout")); + net->runReadyNetworkOperations(); + net->exitNetwork(); + + ASSERT(getReplCoord()->getMemberState().primary()) + << getReplCoord()->getMemberState().toString(); + ASSERT(getReplCoord()->isWaitingForApplierToDrain()); + + // Since we're still in drain mode, expect that we report ismaster: false, issecondary:true. + IsMasterResponse imResponse; + getReplCoord()->fillIsMasterForReplSet(&imResponse); + ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString(); + ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString(); + getReplCoord()->signalDrainComplete(&txn); + getReplCoord()->fillIsMasterForReplSet(&imResponse); + ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString(); + ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString(); +} - TEST_F(ReplCoordElectTest, Elect1NodeSuccess) { - OperationContextReplMock txn; - startCapturingLogMessages(); - assertStartSuccess( - BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345"))), - HostAndPort("node1", 12345)); - - getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); - - ASSERT(getReplCoord()->getMemberState().primary()) << - getReplCoord()->getMemberState().toString(); - ASSERT(getReplCoord()->isWaitingForApplierToDrain()); - - // Since we're still in drain mode, expect that we report ismaster: false, issecondary:true. - IsMasterResponse imResponse; - getReplCoord()->fillIsMasterForReplSet(&imResponse); - ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString(); - ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString(); - getReplCoord()->signalDrainComplete(&txn); - getReplCoord()->fillIsMasterForReplSet(&imResponse); - ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString(); - ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString(); - } +TEST_F(ReplCoordElectTest, Elect1NodeSuccess) { + OperationContextReplMock txn; + startCapturingLogMessages(); + assertStartSuccess(BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345"))), + HostAndPort("node1", 12345)); + + getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY); + + ASSERT(getReplCoord()->getMemberState().primary()) + << getReplCoord()->getMemberState().toString(); + ASSERT(getReplCoord()->isWaitingForApplierToDrain()); + + // Since we're still in drain mode, expect that we report ismaster: false, issecondary:true. + IsMasterResponse imResponse; + getReplCoord()->fillIsMasterForReplSet(&imResponse); + ASSERT_FALSE(imResponse.isMaster()) << imResponse.toBSON().toString(); + ASSERT_TRUE(imResponse.isSecondary()) << imResponse.toBSON().toString(); + getReplCoord()->signalDrainComplete(&txn); + getReplCoord()->fillIsMasterForReplSet(&imResponse); + ASSERT_TRUE(imResponse.isMaster()) << imResponse.toBSON().toString(); + ASSERT_FALSE(imResponse.isSecondary()) << imResponse.toBSON().toString(); +} - TEST_F(ReplCoordElectTest, ElectManyNodesSuccess) { - BSONObj configObj = BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") - << BSON("_id" << 2 << "host" << "node2:12345") - << BSON("_id" << 3 << "host" << "node3:12345") - )); - assertStartSuccess(configObj, HostAndPort("node1", 12345)); - OperationContextNoop txn; - getReplCoord()->setMyLastOptime(OpTime(Timestamp (100, 1), 0)); - ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - startCapturingLogMessages(); - simulateSuccessfulElection(); - stopCapturingLogMessages(); - ASSERT_EQUALS(1, countLogLinesContaining("election succeeded")); - } +TEST_F(ReplCoordElectTest, ElectManyNodesSuccess) { + BSONObj configObj = BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345") + << BSON("_id" << 3 << "host" + << "node3:12345"))); + assertStartSuccess(configObj, HostAndPort("node1", 12345)); + OperationContextNoop txn; + getReplCoord()->setMyLastOptime(OpTime(Timestamp(100, 1), 0)); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + startCapturingLogMessages(); + simulateSuccessfulElection(); + stopCapturingLogMessages(); + ASSERT_EQUALS(1, countLogLinesContaining("election succeeded")); +} - TEST_F(ReplCoordElectTest, ElectNotEnoughVotes) { - // one responds with -10000 votes, and one doesn't respond, and we are not elected - startCapturingLogMessages(); - BSONObj configObj = BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") - << BSON("_id" << 2 << "host" << "node2:12345") - << BSON("_id" << 3 << "host" << "node3:12345") - )); - assertStartSuccess(configObj, HostAndPort("node1", 12345)); - ReplicaSetConfig config = assertMakeRSConfig(configObj); - - OperationContextNoop txn; - OpTime time1(Timestamp(100, 1), 0); - getReplCoord()->setMyLastOptime(time1); - ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - - simulateEnoughHeartbeatsForElectability(); - simulateFreshEnoughForElectability(); - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - while (net->hasReadyRequests()) { - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - if (request.target != HostAndPort("node2", 12345)) { - net->blackHole(noi); - } - else if (request.cmdObj.firstElement().fieldNameStringData() != "replSetElect") { - net->blackHole(noi); - } - else { - net->scheduleResponse( - noi, - net->now(), - makeResponseStatus(BSON("ok" << 1 << - "vote" << -10000 << - "round" << OID()))); - } - net->runReadyNetworkOperations(); +TEST_F(ReplCoordElectTest, ElectNotEnoughVotes) { + // one responds with -10000 votes, and one doesn't respond, and we are not elected + startCapturingLogMessages(); + BSONObj configObj = BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345") + << BSON("_id" << 3 << "host" + << "node3:12345"))); + assertStartSuccess(configObj, HostAndPort("node1", 12345)); + ReplicaSetConfig config = assertMakeRSConfig(configObj); + + OperationContextNoop txn; + OpTime time1(Timestamp(100, 1), 0); + getReplCoord()->setMyLastOptime(time1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + + simulateEnoughHeartbeatsForElectability(); + simulateFreshEnoughForElectability(); + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + while (net->hasReadyRequests()) { + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + log() << request.target.toString() << " processing " << request.cmdObj; + if (request.target != HostAndPort("node2", 12345)) { + net->blackHole(noi); + } else if (request.cmdObj.firstElement().fieldNameStringData() != "replSetElect") { + net->blackHole(noi); + } else { + net->scheduleResponse( + noi, + net->now(), + makeResponseStatus(BSON("ok" << 1 << "vote" << -10000 << "round" << OID()))); } - net->exitNetwork(); - stopCapturingLogMessages(); - ASSERT_EQUALS(1, - countLogLinesContaining("couldn't elect self, only received -9999 votes")); + net->runReadyNetworkOperations(); } + net->exitNetwork(); + stopCapturingLogMessages(); + ASSERT_EQUALS(1, countLogLinesContaining("couldn't elect self, only received -9999 votes")); +} - TEST_F(ReplCoordElectTest, ElectWrongTypeForVote) { - // one responds with a bad 'vote' field, and one doesn't respond, and we are not elected - startCapturingLogMessages(); - BSONObj configObj = BSON("_id" << "mySet" << - "version" << 1 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") - << BSON("_id" << 2 << "host" << "node2:12345") - << BSON("_id" << 3 << "host" << "node3:12345") - )); - assertStartSuccess(configObj, HostAndPort("node1", 12345)); - ReplicaSetConfig config = assertMakeRSConfig(configObj); - - OperationContextNoop txn; - OpTime time1(Timestamp(100, 1), 0); - getReplCoord()->setMyLastOptime(time1); - ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - - simulateEnoughHeartbeatsForElectability(); - simulateFreshEnoughForElectability(); - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - while (net->hasReadyRequests()) { - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - if (request.target != HostAndPort("node2", 12345)) { - net->blackHole(noi); - } - else if (request.cmdObj.firstElement().fieldNameStringData() != "replSetElect") { - net->blackHole(noi); - } - else { - net->scheduleResponse( - noi, - net->now(), - makeResponseStatus(BSON("ok" << 1 << - "vote" << "yea" << - "round" << OID()))); - } - net->runReadyNetworkOperations(); +TEST_F(ReplCoordElectTest, ElectWrongTypeForVote) { + // one responds with a bad 'vote' field, and one doesn't respond, and we are not elected + startCapturingLogMessages(); + BSONObj configObj = BSON("_id" + << "mySet" + << "version" << 1 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345") + << BSON("_id" << 3 << "host" + << "node3:12345"))); + assertStartSuccess(configObj, HostAndPort("node1", 12345)); + ReplicaSetConfig config = assertMakeRSConfig(configObj); + + OperationContextNoop txn; + OpTime time1(Timestamp(100, 1), 0); + getReplCoord()->setMyLastOptime(time1); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + + simulateEnoughHeartbeatsForElectability(); + simulateFreshEnoughForElectability(); + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + while (net->hasReadyRequests()) { + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + log() << request.target.toString() << " processing " << request.cmdObj; + if (request.target != HostAndPort("node2", 12345)) { + net->blackHole(noi); + } else if (request.cmdObj.firstElement().fieldNameStringData() != "replSetElect") { + net->blackHole(noi); + } else { + net->scheduleResponse(noi, + net->now(), + makeResponseStatus(BSON("ok" << 1 << "vote" + << "yea" + << "round" << OID()))); } - net->exitNetwork(); - stopCapturingLogMessages(); - ASSERT_EQUALS(1, - countLogLinesContaining("wrong type for vote argument in replSetElect command")); + net->runReadyNetworkOperations(); } + net->exitNetwork(); + stopCapturingLogMessages(); + ASSERT_EQUALS(1, + countLogLinesContaining("wrong type for vote argument in replSetElect command")); +} - TEST_F(ReplCoordElectTest, ElectionDuringHBReconfigFails) { - // start up, receive reconfig via heartbeat while at the same time, become candidate. - // candidate state should be cleared. - OperationContextNoop txn; - assertStartSuccess( - BSON("_id" << "mySet" << - "version" << 2 << - "members" << BSON_ARRAY(BSON("_id" << 1 << "host" << "node1:12345") << - BSON("_id" << 2 << "host" << "node2:12345") << - BSON("_id" << 3 << "host" << "node3:12345") << - BSON("_id" << 4 << "host" << "node4:12345") << - BSON("_id" << 5 << "host" << "node5:12345") )), - HostAndPort("node1", 12345)); - ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); - getReplCoord()->setMyLastOptime(OpTime(Timestamp(100,0), 0)); - - // set hbreconfig to hang while in progress - getExternalState()->setStoreLocalConfigDocumentToHang(true); - - // hb reconfig - NetworkInterfaceMock* net = getNet(); - net->enterNetwork(); - ReplSetHeartbeatResponse hbResp2; - ReplicaSetConfig config; - config.initialize(BSON("_id" << "mySet" << - "version" << 3 << - "members" << BSON_ARRAY(BSON("_id" << 1 << - "host" << "node1:12345") << - BSON("_id" << 2 << - "host" << "node2:12345")))); - hbResp2.setConfig(config); - hbResp2.setConfigVersion(3); - hbResp2.setSetName("mySet"); - hbResp2.setState(MemberState::RS_SECONDARY); - BSONObjBuilder respObj2; - respObj2 << "ok" << 1; - hbResp2.addToBSON(&respObj2, false); - net->runUntil(net->now() + Seconds(10)); // run until we've sent a heartbeat request - const NetworkInterfaceMock::NetworkOperationIterator noi2 = net->getNextReadyRequest(); - net->scheduleResponse(noi2, net->now(), makeResponseStatus(respObj2.obj())); - net->runReadyNetworkOperations(); - getNet()->exitNetwork(); - - // prepare candidacy - BSONObjBuilder result; - ReplicationCoordinator::ReplSetReconfigArgs args; - args.force = false; - args.newConfigObj = config.toBSON(); - ASSERT_EQUALS(ErrorCodes::ConfigurationInProgress, - getReplCoord()->processReplSetReconfig(&txn, args, &result)); - - logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(2)); - startCapturingLogMessages(); - - // receive sufficient heartbeats to trigger an election - ReplicationCoordinatorImpl* replCoord = getReplCoord(); - ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); - net->enterNetwork(); - for (int i = 0; i < 2; ++i) { - const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); - const RemoteCommandRequest& request = noi->getRequest(); - log() << request.target.toString() << " processing " << request.cmdObj; - ReplSetHeartbeatArgs hbArgs; - if (hbArgs.initialize(request.cmdObj).isOK()) { - ReplSetHeartbeatResponse hbResp; - hbResp.setSetName(rsConfig.getReplSetName()); - hbResp.setState(MemberState::RS_SECONDARY); - hbResp.setConfigVersion(rsConfig.getConfigVersion()); - BSONObjBuilder respObj; - respObj << "ok" << 1; - hbResp.addToBSON(&respObj, false); - net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj())); - } - else { - error() << "Black holing unexpected request to " << request.target << ": " << - request.cmdObj; - net->blackHole(noi); - } - net->runReadyNetworkOperations(); +TEST_F(ReplCoordElectTest, ElectionDuringHBReconfigFails) { + // start up, receive reconfig via heartbeat while at the same time, become candidate. + // candidate state should be cleared. + OperationContextNoop txn; + assertStartSuccess( + BSON("_id" + << "mySet" + << "version" << 2 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345") << BSON("_id" << 3 << "host" + << "node3:12345") + << BSON("_id" << 4 << "host" + << "node4:12345") << BSON("_id" << 5 << "host" + << "node5:12345"))), + HostAndPort("node1", 12345)); + ASSERT(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + getReplCoord()->setMyLastOptime(OpTime(Timestamp(100, 0), 0)); + + // set hbreconfig to hang while in progress + getExternalState()->setStoreLocalConfigDocumentToHang(true); + + // hb reconfig + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + ReplSetHeartbeatResponse hbResp2; + ReplicaSetConfig config; + config.initialize(BSON("_id" + << "mySet" + << "version" << 3 << "members" + << BSON_ARRAY(BSON("_id" << 1 << "host" + << "node1:12345") + << BSON("_id" << 2 << "host" + << "node2:12345")))); + hbResp2.setConfig(config); + hbResp2.setConfigVersion(3); + hbResp2.setSetName("mySet"); + hbResp2.setState(MemberState::RS_SECONDARY); + BSONObjBuilder respObj2; + respObj2 << "ok" << 1; + hbResp2.addToBSON(&respObj2, false); + net->runUntil(net->now() + Seconds(10)); // run until we've sent a heartbeat request + const NetworkInterfaceMock::NetworkOperationIterator noi2 = net->getNextReadyRequest(); + net->scheduleResponse(noi2, net->now(), makeResponseStatus(respObj2.obj())); + net->runReadyNetworkOperations(); + getNet()->exitNetwork(); + + // prepare candidacy + BSONObjBuilder result; + ReplicationCoordinator::ReplSetReconfigArgs args; + args.force = false; + args.newConfigObj = config.toBSON(); + ASSERT_EQUALS(ErrorCodes::ConfigurationInProgress, + getReplCoord()->processReplSetReconfig(&txn, args, &result)); + + logger::globalLogDomain()->setMinimumLoggedSeverity(logger::LogSeverity::Debug(2)); + startCapturingLogMessages(); + + // receive sufficient heartbeats to trigger an election + ReplicationCoordinatorImpl* replCoord = getReplCoord(); + ReplicaSetConfig rsConfig = replCoord->getReplicaSetConfig_forTest(); + net->enterNetwork(); + for (int i = 0; i < 2; ++i) { + const NetworkInterfaceMock::NetworkOperationIterator noi = net->getNextReadyRequest(); + const RemoteCommandRequest& request = noi->getRequest(); + log() << request.target.toString() << " processing " << request.cmdObj; + ReplSetHeartbeatArgs hbArgs; + if (hbArgs.initialize(request.cmdObj).isOK()) { + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName(rsConfig.getReplSetName()); + hbResp.setState(MemberState::RS_SECONDARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + BSONObjBuilder respObj; + respObj << "ok" << 1; + hbResp.addToBSON(&respObj, false); + net->scheduleResponse(noi, net->now(), makeResponseStatus(respObj.obj())); + } else { + error() << "Black holing unexpected request to " << request.target << ": " + << request.cmdObj; + net->blackHole(noi); } - - stopCapturingLogMessages(); - // ensure node does not stand for election - ASSERT_EQUALS(1, - countLogLinesContaining("Not standing for election; processing " - "a configuration change")); - getExternalState()->setStoreLocalConfigDocumentToHang(false); + net->runReadyNetworkOperations(); } + stopCapturingLogMessages(); + // ensure node does not stand for election + ASSERT_EQUALS(1, + countLogLinesContaining( + "Not standing for election; processing " + "a configuration change")); + getExternalState()->setStoreLocalConfigDocumentToHang(false); +} } } } |