diff options
author | William Schultz <william.schultz@mongodb.com> | 2020-02-05 23:07:46 -0500 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2020-02-06 04:23:41 +0000 |
commit | 36f668e510c99b47620c1985e65c72a106fadbbe (patch) | |
tree | ccafcb0189c300d93a92a7a7f3d0267d14f4ef32 | |
parent | 97f3babf78fcf7d10059e842d580380f03370e65 (diff) | |
download | mongo-36f668e510c99b47620c1985e65c72a106fadbbe.tar.gz |
SERVER-45082 Secondaries only learn replica set configs with a newer (version, term) pair
-rw-r--r-- | src/mongo/db/repl/repl_set_config.h | 7 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_heartbeat_args_v1.cpp | 25 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_heartbeat_args_v1.h | 21 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_heartbeat_response.cpp | 8 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_heartbeat_response.h | 14 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_heartbeat_response_test.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp | 9 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp | 302 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator.cpp | 19 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_v1_test.cpp | 259 |
11 files changed, 658 insertions, 19 deletions
diff --git a/src/mongo/db/repl/repl_set_config.h b/src/mongo/db/repl/repl_set_config.h index b82f0ed5e22..23d2614adef 100644 --- a/src/mongo/db/repl/repl_set_config.h +++ b/src/mongo/db/repl/repl_set_config.h @@ -202,6 +202,13 @@ public: } /** + * Gets the (version, term) pair of this configuration. + */ + ConfigVersionAndTerm getConfigVersionAndTerm() const { + return ConfigVersionAndTerm(_version, _term); + } + + /** * Gets the name (_id field value) of the replica set described by this configuration. */ const std::string& getReplSetName() const { diff --git a/src/mongo/db/repl/repl_set_heartbeat_args_v1.cpp b/src/mongo/db/repl/repl_set_heartbeat_args_v1.cpp index a7cc785995e..57f85aa58f9 100644 --- a/src/mongo/db/repl/repl_set_heartbeat_args_v1.cpp +++ b/src/mongo/db/repl/repl_set_heartbeat_args_v1.cpp @@ -43,6 +43,7 @@ namespace { const std::string kCheckEmptyFieldName = "checkEmpty"; const std::string kConfigVersionFieldName = "configVersion"; +const std::string kConfigTermFieldName = "configTerm"; const std::string kHeartbeatVersionFieldName = "hbv"; const std::string kSenderHostFieldName = "from"; const std::string kSenderIdFieldName = "fromId"; @@ -51,6 +52,7 @@ const std::string kTermFieldName = "term"; const std::string kLegalHeartbeatFieldNames[] = {kCheckEmptyFieldName, kConfigVersionFieldName, + kConfigTermFieldName, kHeartbeatVersionFieldName, kSenderHostFieldName, kSenderIdFieldName, @@ -73,6 +75,11 @@ Status ReplSetHeartbeatArgsV1::initialize(const BSONObj& argsObj) { if (!status.isOK()) return status; + status = bsonExtractIntegerFieldWithDefault( + argsObj, kConfigTermFieldName, OpTime::kUninitializedTerm, &_configTerm); + if (!status.isOK()) + return status; + long long tempHeartbeatVersion; status = bsonExtractIntegerField(argsObj, kHeartbeatVersionFieldName, &tempHeartbeatVersion); if (status.isOK()) { @@ -122,6 +129,10 @@ void ReplSetHeartbeatArgsV1::setConfigVersion(long long newVal) { _configVersion = newVal; } +void ReplSetHeartbeatArgsV1::setConfigTerm(long long newVal) { + _configTerm = newVal; +} + void ReplSetHeartbeatArgsV1::setHeartbeatVersion(long long newVal) { _heartbeatVersion = newVal; _hasHeartbeatVersion = true; @@ -148,19 +159,27 @@ void ReplSetHeartbeatArgsV1::setCheckEmpty() { _checkEmpty = true; } -BSONObj ReplSetHeartbeatArgsV1::toBSON() const { +BSONObj ReplSetHeartbeatArgsV1::toBSON(bool omitConfigTerm) const { invariant(isInitialized()); BSONObjBuilder builder; - addToBSON(&builder); + addToBSON(&builder, omitConfigTerm); return builder.obj(); } -void ReplSetHeartbeatArgsV1::addToBSON(BSONObjBuilder* builder) const { +void ReplSetHeartbeatArgsV1::addToBSON(BSONObjBuilder* builder, bool omitConfigTerm) const { builder->append(kSetNameFieldName, _setName); if (_checkEmpty) { builder->append(kCheckEmptyFieldName, _checkEmpty); } builder->appendIntOrLL(kConfigVersionFieldName, _configVersion); + // Only attach the term field if we are fully upgraded to 4.4, since 4.2 nodes won't be able to + // parse it. + if (serverGlobalParams.featureCompatibility.isVersionInitialized() && + serverGlobalParams.featureCompatibility.getVersion() == + ServerGlobalParams::FeatureCompatibility::Version::kFullyUpgradedTo44 && + !omitConfigTerm) { + builder->appendIntOrLL(kConfigTermFieldName, _configTerm); + } if (_hasHeartbeatVersion) { builder->appendIntOrLL(kHeartbeatVersionFieldName, _hasHeartbeatVersion); } diff --git a/src/mongo/db/repl/repl_set_heartbeat_args_v1.h b/src/mongo/db/repl/repl_set_heartbeat_args_v1.h index de89a29df7f..f183a22072a 100644 --- a/src/mongo/db/repl/repl_set_heartbeat_args_v1.h +++ b/src/mongo/db/repl/repl_set_heartbeat_args_v1.h @@ -31,6 +31,7 @@ #include <string> +#include "mongo/db/repl/repl_set_config.h" #include "mongo/util/net/hostandport.h" namespace mongo { @@ -64,6 +65,20 @@ public: } /** + * Gets the ReplSetConfig term number of the sender. + */ + long long getConfigTerm() const { + return _configTerm; + } + + /** + * Gets the ReplSetConfig (version, term) pair of the sender. + */ + ConfigVersionAndTerm getConfigVersionAndTerm() const { + return ConfigVersionAndTerm(_configVersion, _configTerm); + } + + /** * Gets the heartbeat version number of the sender. This field was added to ensure that * heartbeats sent from featureCompatibilityVersion 3.6 nodes to binary version 3.4 nodes fail. */ @@ -124,6 +139,7 @@ public: * The below methods set the value in the method name to 'newVal'. */ void setConfigVersion(long long newVal); + void setConfigTerm(long long newVal); void setHeartbeatVersion(long long newVal); void setSenderId(long long newVal); void setSenderHost(const HostAndPort& newVal); @@ -136,13 +152,14 @@ public: * Should only be called if the mandatory fields have been set. * Optional fields are only included if they have been set. */ - BSONObj toBSON() const; + BSONObj toBSON(bool omitConfigTerm = false) const; - void addToBSON(BSONObjBuilder* builder) const; + void addToBSON(BSONObjBuilder* builder, bool omitConfigTerm) const; private: // look at the body of the isInitialized() function to see which fields are mandatory long long _configVersion = -1; + long long _configTerm = OpTime::kUninitializedTerm; long long _heartbeatVersion = -1; long long _senderId = -1; long long _term = -1; diff --git a/src/mongo/db/repl/repl_set_heartbeat_response.cpp b/src/mongo/db/repl/repl_set_heartbeat_response.cpp index c43316acdc0..5905881ac3d 100644 --- a/src/mongo/db/repl/repl_set_heartbeat_response.cpp +++ b/src/mongo/db/repl/repl_set_heartbeat_response.cpp @@ -51,6 +51,7 @@ namespace { const std::string kConfigFieldName = "config"; const std::string kConfigVersionFieldName = "v"; +const std::string kConfigTermFieldName = "t"; const std::string kElectionTimeFieldName = "electionTime"; const std::string kMemberStateFieldName = "state"; const std::string kOkFieldName = "ok"; @@ -80,6 +81,7 @@ void ReplSetHeartbeatResponse::addToBSON(BSONObjBuilder* builder) const { } if (_configVersion != -1) { *builder << kConfigVersionFieldName << _configVersion; + *builder << kConfigTermFieldName << _configTerm; } if (!_setName.empty()) { *builder << kReplSetFieldName << _setName; @@ -218,6 +220,12 @@ Status ReplSetHeartbeatResponse::initialize(const BSONObj& doc, long long term) } _configVersion = configVersionElement.numberInt(); + // Allow a missing term field for backward compatibility. + const BSONElement configTermElement = doc[kConfigTermFieldName]; + if (!configTermElement.eoo() && configVersionElement.type() == NumberInt) { + _configTerm = configTermElement.numberInt(); + } + const BSONElement syncingToElement = doc[kSyncSourceFieldName]; if (syncingToElement.eoo()) { _syncingTo = HostAndPort(); diff --git a/src/mongo/db/repl/repl_set_heartbeat_response.h b/src/mongo/db/repl/repl_set_heartbeat_response.h index 0f18536e7f5..bb5c6d7bd68 100644 --- a/src/mongo/db/repl/repl_set_heartbeat_response.h +++ b/src/mongo/db/repl/repl_set_heartbeat_response.h @@ -88,6 +88,12 @@ public: int getConfigVersion() const { return _configVersion; } + int getConfigTerm() const { + return _configTerm; + } + ConfigVersionAndTerm getConfigVersionAndTerm() const { + return ConfigVersionAndTerm(_configVersion, _configTerm); + } bool hasConfig() const { return _configSet; } @@ -148,6 +154,13 @@ public: } /** + * Sets _configTerm to "configTerm". + */ + void setConfigTerm(int configTerm) { + _configTerm = configTerm; + } + + /** * Initializes _config with "config". */ void setConfig(const ReplSetConfig& config) { @@ -189,6 +202,7 @@ private: MemberState _state; int _configVersion = -1; + int _configTerm = OpTime::kUninitializedTerm; std::string _setName; HostAndPort _syncingTo; diff --git a/src/mongo/db/repl/repl_set_heartbeat_response_test.cpp b/src/mongo/db/repl/repl_set_heartbeat_response_test.cpp index 89d439af86e..774dd13c76b 100644 --- a/src/mongo/db/repl/repl_set_heartbeat_response_test.cpp +++ b/src/mongo/db/repl/repl_set_heartbeat_response_test.cpp @@ -70,6 +70,9 @@ TEST(ReplSetHeartbeatResponse, DefaultConstructThenSlowlyBuildToFullObj) { // set version hbResponse.setConfigVersion(1); ++fieldsSet; + // set config term. + hbResponse.setConfigTerm(1); + ++fieldsSet; // set setname hbResponse.setSetName("rs0"); ++fieldsSet; @@ -113,6 +116,7 @@ TEST(ReplSetHeartbeatResponse, DefaultConstructThenSlowlyBuildToFullObj) { ASSERT_EQUALS(fieldsSet, hbResponseObj.nFields()); ASSERT_EQUALS("rs0", hbResponseObj["set"].String()); ASSERT_EQUALS(1, hbResponseObj["v"].Number()); + ASSERT_EQUALS(1, hbResponseObj["t"].Number()); ASSERT_EQUALS(Timestamp(10, 0), hbResponseObj["electionTime"].timestamp()); ASSERT_EQUALS(Timestamp(0, 50), hbResponseObj["opTime"]["ts"].timestamp()); ASSERT_EQUALS(Timestamp(0, 10), hbResponseObj["durableOpTime"]["ts"].timestamp()); diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index 991dc848976..191c038b968 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -4120,13 +4120,20 @@ Status ReplicationCoordinatorImpl::processHeartbeatV1(const ReplSetHeartbeatArgs // a configuration that contains us. Chances are excellent that it will, since that // is the only reason for a remote node to send this node a heartbeat request. if (!senderHost.empty() && _seedList.insert(senderHost).second) { + log() << "Scheduling heartbeat to fetch a new config from: " << senderHost + << " since we are not a member of our current config."; _scheduleHeartbeatToTarget_inlock(senderHost, -1, now); } - } else if (result.isOK() && response->getConfigVersion() < args.getConfigVersion()) { + } else if (result.isOK() && + response->getConfigVersionAndTerm() < args.getConfigVersionAndTerm()) { // Schedule a heartbeat to the sender to fetch the new config. + // Only send this if the sender's config is newer. // We cannot cancel the enqueued heartbeat, but either this one or the enqueued heartbeat // will trigger reconfig, which cancels and reschedules all heartbeats. if (args.hasSender()) { + log() << "Scheduling heartbeat to fetch a newer config with term " + << args.getConfigTerm() << " and version " << args.getConfigVersion() + << " from member: " << senderHost; int senderIndex = _rsConfig.findMemberIndexByHostAndPort(senderHost); _scheduleHeartbeatToTarget_inlock(senderHost, senderIndex, now); } diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 88010f8dbec..0d5731711f4 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -103,7 +103,10 @@ void ReplicationCoordinatorImpl::_doMemberHeartbeat(executor::TaskExecutor::Call Milliseconds timeout(0); const std::pair<ReplSetHeartbeatArgsV1, Milliseconds> hbRequest = _topCoord->prepareHeartbeatRequestV1(now, _settings.ourSetName(), target); - heartbeatObj = hbRequest.first.toBSON(); + // Omit config term from all heartbeat requests sent by arbiter, since it cannot be parsed by + // 4.2 nodes and arbiters do not respect FCV setting. + bool omitConfigTerm = _getMemberState_inlock().arbiter(); + heartbeatObj = hbRequest.first.toBSON(omitConfigTerm); timeout = hbRequest.second; const RemoteCommandRequest request( @@ -464,7 +467,7 @@ void ReplicationCoordinatorImpl::_scheduleHeartbeatReconfig_inlock(const ReplSet } _setConfigState_inlock(kConfigHBReconfiguring); invariant(!_rsConfig.isInitialized() || - _rsConfig.getConfigVersion() < newConfig.getConfigVersion()); + _rsConfig.getConfigVersionAndTerm() < newConfig.getConfigVersionAndTerm()); if (auto electionFinishedEvent = _cancelElectionIfNeeded_inlock()) { LOG_FOR_HEARTBEATS(2) << "Rescheduling heartbeat reconfig to config with term " << newConfig.getConfigTerm() << ", version " @@ -654,7 +657,7 @@ void ReplicationCoordinatorImpl::_heartbeatReconfigFinish( invariant(_rsConfigState == kConfigHBReconfiguring); invariant(!_rsConfig.isInitialized() || - _rsConfig.getConfigVersion() < newConfig.getConfigVersion()); + _rsConfig.getConfigVersionAndTerm() < newConfig.getConfigVersionAndTerm()); if (!myIndex.isOK()) { switch (myIndex.getStatus().code()) { diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp index c26b8d114d2..5532d72f701 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat_v1_test.cpp @@ -72,6 +72,7 @@ ReplSetHeartbeatResponse ReplCoordHBV1Test::receiveHeartbeatFrom(const ReplSetCo const HostAndPort& source) { ReplSetHeartbeatArgsV1 hbArgs; hbArgs.setConfigVersion(rsConfig.getConfigVersion()); + hbArgs.setConfigTerm(rsConfig.getConfigTerm()); hbArgs.setSetName(rsConfig.getReplSetName()); hbArgs.setSenderHost(source); hbArgs.setSenderId(sourceId); @@ -152,6 +153,307 @@ TEST_F(ReplCoordHBV1Test, ASSERT_TRUE(getExternalState()->threadsStarted()); } +class ReplCoordHBV1ReconfigTest : public ReplCoordHBV1Test { +public: + void setUp() { + setMinimumLoggedSeverity(logger::LogSeverity::Debug(3)); + BSONObj configBson = BSON("_id" + << "mySet" + << "version" << initConfigVersion << "term" << initConfigTerm + << "members" << members << "protocolVersion" << 1); + ReplSetConfig rsConfig = assertMakeRSConfig(configBson); + assertStartSuccess(configBson, HostAndPort("h2", 1)); + ASSERT_OK(getReplCoord()->setFollowerMode(MemberState::RS_SECONDARY)); + + // Black hole initial heartbeat requests. + NetworkInterfaceMock* net = getNet(); + net->enterNetwork(); + net->blackHole(net->getNextReadyRequest()); + net->blackHole(net->getNextReadyRequest()); + net->exitNetwork(); + } + + BSONObj makeConfigObj(long long version, boost::optional<long long> term) { + BSONObjBuilder bob; + bob.appendElements(BSON("_id" + << "mySet" + << "version" << version << "members" << members << "protocolVersion" + << 1)); + if (term) { + bob.append("term", *term); + } + return bob.obj(); + } + + ReplSetConfig makeRSConfigWithVersionAndTerm(long long version, long long term) { + return assertMakeRSConfig(makeConfigObj(version, term)); + } + + int initConfigVersion = 2; + int initConfigTerm = 2; + long long UninitializedTerm = OpTime::kUninitializedTerm; + BSONArray members = BSON_ARRAY(BSON("_id" << 1 << "host" + << "h1:1") + << BSON("_id" << 2 << "host" + << "h2:1") + << BSON("_id" << 3 << "host" + << "h3:1")); +}; + + +TEST_F(ReplCoordHBV1ReconfigTest, + NodeSchedulesHeartbeatToFetchConfigIfItHearsAboutConfigWithNewerVersionAndWillInstallIt) { + // Config with newer version and same term. + ReplSetConfig rsConfig = + makeRSConfigWithVersionAndTerm((initConfigVersion + 1), initConfigTerm); + + // Receive a heartbeat request that tells us about a newer config. + receiveHeartbeatFrom(rsConfig, 1, HostAndPort("h1", 1)); + + getNet()->enterNetwork(); + ReplSetHeartbeatArgsV1 hbArgs; + auto noi = getNet()->getNextReadyRequest(); + const RemoteCommandRequest& hbrequest = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("h1", 1), hbrequest.target); + ASSERT_OK(hbArgs.initialize(hbrequest.cmdObj)); + ASSERT_EQUALS("mySet", hbArgs.getSetName()); + ASSERT_EQUALS(initConfigVersion, hbArgs.getConfigVersion()); + ASSERT_EQUALS(initConfigTerm, hbArgs.getConfigTerm()); + ASSERT_EQUALS(OpTime::kInitialTerm, hbArgs.getTerm()); + + // Construct the heartbeat response containing the newer config. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + hbResp.setConfigTerm(rsConfig.getConfigTerm()); + // The smallest valid optime in PV1. + OpTime opTime(Timestamp(), 0); + hbResp.setAppliedOpTimeAndWallTime({opTime, Date_t()}); + hbResp.setDurableOpTimeAndWallTime({opTime, Date_t()}); + BSONObjBuilder responseBuilder; + responseBuilder << "ok" << 1; + hbResp.addToBSON(&responseBuilder); + // Add the raw config object. + responseBuilder << "config" << makeConfigObj(initConfigVersion + 1, initConfigTerm); + auto origResObj = responseBuilder.obj(); + + // Schedule and deliver the heartbeat response. + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(origResObj)); + getNet()->runReadyNetworkOperations(); + + ASSERT_EQ(getReplCoord()->getConfig().getConfigVersion(), initConfigVersion + 1); + ASSERT_EQ(getReplCoord()->getConfig().getConfigTerm(), initConfigTerm); +} + +TEST_F(ReplCoordHBV1ReconfigTest, + NodeSchedulesHeartbeatToFetchConfigIfItHearsAboutConfigWithNewerTermAndWillInstallIt) { + // Config with newer term and same version. + ReplSetConfig rsConfig = makeRSConfigWithVersionAndTerm(initConfigVersion, initConfigTerm + 1); + + // Receive a heartbeat request that tells us about a newer config. + receiveHeartbeatFrom(rsConfig, 1, HostAndPort("h1", 1)); + + getNet()->enterNetwork(); + ReplSetHeartbeatArgsV1 hbArgs; + auto noi = getNet()->getNextReadyRequest(); + const RemoteCommandRequest& hbrequest = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("h1", 1), hbrequest.target); + ASSERT_OK(hbArgs.initialize(hbrequest.cmdObj)); + ASSERT_EQUALS("mySet", hbArgs.getSetName()); + ASSERT_EQUALS(initConfigVersion, hbArgs.getConfigVersion()); + ASSERT_EQUALS(initConfigTerm, hbArgs.getConfigTerm()); + ASSERT_EQUALS(OpTime::kInitialTerm, hbArgs.getTerm()); + + // Construct the heartbeat response containing the newer config. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + hbResp.setConfigTerm(rsConfig.getConfigTerm()); + // The smallest valid optime in PV1. + OpTime opTime(Timestamp(), 0); + hbResp.setAppliedOpTimeAndWallTime({opTime, Date_t()}); + hbResp.setDurableOpTimeAndWallTime({opTime, Date_t()}); + BSONObjBuilder responseBuilder; + responseBuilder << "ok" << 1; + hbResp.addToBSON(&responseBuilder); + // Add the raw config object. + responseBuilder << "config" << makeConfigObj(initConfigVersion, initConfigTerm + 1); + auto origResObj = responseBuilder.obj(); + + // Schedule and deliver the heartbeat response. + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(origResObj)); + getNet()->runReadyNetworkOperations(); + + ASSERT_EQ(getReplCoord()->getConfig().getConfigTerm(), initConfigTerm + 1); +} + +TEST_F(ReplCoordHBV1ReconfigTest, + NodeShouldntScheduleHeartbeatToFetchConfigIfItHearsAboutSameConfig) { + // Config with same term and same version. Shouldn't schedule any heartbeats. + receiveHeartbeatFrom(getReplCoord()->getReplicaSetConfig_forTest(), 1, HostAndPort("h1", 1)); + getNet()->enterNetwork(); + ASSERT_FALSE(getNet()->hasReadyRequests()); +} + +TEST_F( + ReplCoordHBV1ReconfigTest, + NodeSchedulesHeartbeatToFetchConfigIfItHearsAboutConfigWithNewerTermAndLowerVersionAndWillInstallIt) { + // Config with newer term and lower version. + ReplSetConfig rsConfig = + makeRSConfigWithVersionAndTerm((initConfigVersion - 1), (initConfigTerm + 1)); + + // Receive a heartbeat request that tells us about a newer config. + receiveHeartbeatFrom(rsConfig, 1, HostAndPort("h1", 1)); + + getNet()->enterNetwork(); + ReplSetHeartbeatArgsV1 hbArgs; + auto noi = getNet()->getNextReadyRequest(); + const RemoteCommandRequest& hbrequest = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("h1", 1), hbrequest.target); + ASSERT_OK(hbArgs.initialize(hbrequest.cmdObj)); + ASSERT_EQUALS("mySet", hbArgs.getSetName()); + ASSERT_EQUALS(initConfigVersion, hbArgs.getConfigVersion()); + ASSERT_EQUALS(initConfigTerm, hbArgs.getConfigTerm()); + ASSERT_EQUALS(OpTime::kInitialTerm, hbArgs.getTerm()); + + // Construct the heartbeat response containing the newer config. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + hbResp.setConfigTerm(rsConfig.getConfigTerm()); + // The smallest valid optime in PV1. + OpTime opTime(Timestamp(), 0); + hbResp.setAppliedOpTimeAndWallTime({opTime, Date_t()}); + hbResp.setDurableOpTimeAndWallTime({opTime, Date_t()}); + BSONObjBuilder responseBuilder; + responseBuilder << "ok" << 1; + hbResp.addToBSON(&responseBuilder); + // Add the raw config object. + responseBuilder << "config" << makeConfigObj((initConfigVersion - 1), (initConfigTerm + 1)); + auto origResObj = responseBuilder.obj(); + + // Schedule and deliver the heartbeat response. + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(origResObj)); + getNet()->runReadyNetworkOperations(); + + ASSERT_EQ(getReplCoord()->getConfig().getConfigVersion(), (initConfigVersion - 1)); + ASSERT_EQ(getReplCoord()->getConfig().getConfigTerm(), (initConfigTerm + 1)); +} + +TEST_F( + ReplCoordHBV1ReconfigTest, + NodeSchedulesHeartbeatToFetchConfigIfItHearsAboutConfigWithNewerVersionAndUninitializedTermAndWillInstallIt) { + // Config with version and uninitialized term. + ReplSetConfig rsConfig = + makeRSConfigWithVersionAndTerm(initConfigVersion + 1, UninitializedTerm); + + // Receive a heartbeat request that tells us about a newer config. + receiveHeartbeatFrom(rsConfig, 1, HostAndPort("h1", 1)); + + getNet()->enterNetwork(); + ReplSetHeartbeatArgsV1 hbArgs; + auto noi = getNet()->getNextReadyRequest(); + const RemoteCommandRequest& hbrequest = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("h1", 1), hbrequest.target); + ASSERT_OK(hbArgs.initialize(hbrequest.cmdObj)); + ASSERT_EQUALS("mySet", hbArgs.getSetName()); + ASSERT_EQUALS(initConfigVersion, hbArgs.getConfigVersion()); + ASSERT_EQUALS(initConfigTerm, hbArgs.getConfigTerm()); + ASSERT_EQUALS(OpTime::kInitialTerm, hbArgs.getTerm()); + + // Construct the heartbeat response containing the newer config. + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + hbResp.setConfigTerm(rsConfig.getConfigTerm()); + // The smallest valid optime in PV1. + OpTime opTime(Timestamp(), 0); + hbResp.setAppliedOpTimeAndWallTime({opTime, Date_t()}); + hbResp.setDurableOpTimeAndWallTime({opTime, Date_t()}); + BSONObjBuilder responseBuilder; + responseBuilder << "ok" << 1; + hbResp.addToBSON(&responseBuilder); + // Add the raw config object. + responseBuilder << "config" << makeConfigObj((initConfigVersion + 1), UninitializedTerm); + auto origResObj = responseBuilder.obj(); + + // Schedule and deliver the heartbeat response. + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(origResObj)); + getNet()->runReadyNetworkOperations(); + + ASSERT_EQ(getReplCoord()->getConfig().getConfigVersion(), (initConfigVersion + 1)); + ASSERT_EQ(getReplCoord()->getConfig().getConfigTerm(), UninitializedTerm); +} + +TEST_F(ReplCoordHBV1ReconfigTest, + NodeSchedulesHeartbeatToFetchNewerConfigAndInstallsConfigWithNoTermField) { + // Config with newer version. + ReplSetConfig rsConfig = + makeRSConfigWithVersionAndTerm(initConfigVersion + 1, UninitializedTerm); + + // Receive a heartbeat request that tells us about a newer config. + receiveHeartbeatFrom(rsConfig, 1, HostAndPort("h1", 1)); + + getNet()->enterNetwork(); + ReplSetHeartbeatArgsV1 hbArgs; + auto noi = getNet()->getNextReadyRequest(); + const RemoteCommandRequest& hbrequest = noi->getRequest(); + ASSERT_EQUALS(HostAndPort("h1", 1), hbrequest.target); + ASSERT_OK(hbArgs.initialize(hbrequest.cmdObj)); + ASSERT_EQUALS("mySet", hbArgs.getSetName()); + ASSERT_EQUALS(initConfigVersion, hbArgs.getConfigVersion()); + ASSERT_EQUALS(initConfigTerm, hbArgs.getConfigTerm()); + ASSERT_EQUALS(OpTime::kInitialTerm, hbArgs.getTerm()); + + ReplSetHeartbeatResponse hbResp; + hbResp.setSetName("mySet"); + hbResp.setState(MemberState::RS_PRIMARY); + hbResp.setConfigVersion(rsConfig.getConfigVersion()); + hbResp.setConfig(rsConfig); + // The smallest valid optime in PV1. + OpTime opTime(Timestamp(), 0); + hbResp.setAppliedOpTimeAndWallTime({opTime, Date_t()}); + hbResp.setDurableOpTimeAndWallTime({opTime, Date_t()}); + BSONObjBuilder responseBuilder; + responseBuilder << "ok" << 1; + hbResp.addToBSON(&responseBuilder); + auto origResObj = responseBuilder.obj(); + + // Construct a heartbeat response object that omits the top-level 't' field and the 'term' field + // from the config object. This simulates the case of receiving a heartbeat response from a 4.2 + // node. + BSONObjBuilder finalRes; + for (auto field : origResObj.getFieldNames<std::set<std::string>>()) { + if (field == "t") { + continue; + } else if (field == "config") { + finalRes.append("config", makeConfigObj(initConfigVersion + 1, boost::none /* term */)); + } else { + finalRes.append(origResObj[field]); + } + } + + // Make sure the response has no term fields. + auto finalResObj = finalRes.obj(); + ASSERT_FALSE(finalResObj.hasField("t")); + ASSERT_TRUE(finalResObj.hasField("config")); + ASSERT_TRUE(finalResObj["config"].isABSONObj()); + ASSERT_FALSE(finalResObj.getObjectField("config").hasField("term")); + + // Schedule and deliver the heartbeat response. + getNet()->scheduleResponse(noi, getNet()->now(), makeResponseStatus(finalResObj)); + getNet()->runReadyNetworkOperations(); + + // We should have installed the newer config, even though it had no term attached. + auto myConfig = getReplCoord()->getConfig(); + ASSERT_EQ(myConfig.getConfigVersion(), initConfigVersion + 1); + ASSERT_EQ(myConfig.getConfigTerm(), UninitializedTerm); +} + TEST_F(ReplCoordHBV1Test, AwaitIsMasterReturnsResponseOnReconfigViaHeartbeat) { init(); assertStartSuccess(BSON("_id" diff --git a/src/mongo/db/repl/topology_coordinator.cpp b/src/mongo/db/repl/topology_coordinator.cpp index 770440e7c1c..2248dc2e450 100644 --- a/src/mongo/db/repl/topology_coordinator.cpp +++ b/src/mongo/db/repl/topology_coordinator.cpp @@ -626,9 +626,11 @@ Status TopologyCoordinator::prepareHeartbeatResponseV1(Date_t now, } const long long v = _rsConfig.getConfigVersion(); + const long long t = _rsConfig.getConfigTerm(); response->setConfigVersion(v); - // Deliver new config if caller's version is older than ours - if (v > args.getConfigVersion()) { + response->setConfigTerm(t); + // Deliver new config if caller's config is older than ours + if (_rsConfig.getConfigVersionAndTerm() > args.getConfigVersionAndTerm()) { response->setConfig(_rsConfig); } @@ -676,6 +678,7 @@ std::pair<ReplSetHeartbeatArgsV1, Milliseconds> TopologyCoordinator::prepareHear if (_rsConfig.isInitialized()) { hbArgs.setSetName(_rsConfig.getReplSetName()); hbArgs.setConfigVersion(_rsConfig.getConfigVersion()); + hbArgs.setConfigTerm(_rsConfig.getConfigTerm()); if (_selfIndex >= 0) { const MemberConfig& me = _selfConfig(); hbArgs.setSenderId(me.getId().getData()); @@ -745,17 +748,19 @@ HeartbeatResponseAction TopologyCoordinator::processHeartbeatResponse( } if (hbResponse.isOK() && hbResponse.getValue().hasConfig()) { - const long long currentConfigVersion = - _rsConfig.isInitialized() ? _rsConfig.getConfigVersion() : -2; + // -2 is for uninitialized config. + const ConfigVersionAndTerm currentConfigVersionAndTerm = _rsConfig.isInitialized() + ? _rsConfig.getConfigVersionAndTerm() + : ConfigVersionAndTerm(-2, OpTime::kUninitializedTerm); const ReplSetConfig& newConfig = hbResponse.getValue().getConfig(); - if (newConfig.getConfigVersion() > currentConfigVersion) { + if (newConfig.getConfigVersionAndTerm() > currentConfigVersionAndTerm) { HeartbeatResponseAction nextAction = HeartbeatResponseAction::makeReconfigAction(); nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); return nextAction; } else { // Could be we got the newer version before we got the response, or the - // target erroneously sent us one, even through it isn't newer. - if (newConfig.getConfigVersion() < currentConfigVersion) { + // target erroneously sent us one, even though it isn't newer. + if (newConfig.getConfigVersionAndTerm() < currentConfigVersionAndTerm) { LOG(1) << "Config version from heartbeat was older than ours."; } else { LOG(2) << "Config from heartbeat response was same as ours."; diff --git a/src/mongo/db/repl/topology_coordinator_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_v1_test.cpp index 4720e7b01b4..6756e50e246 100644 --- a/src/mongo/db/repl/topology_coordinator_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_v1_test.cpp @@ -1863,7 +1863,7 @@ public: TopoCoordTest::setUp(); updateConfig(BSON("_id" << "rs0" - << "version" << 1 << "members" + << "version" << initConfigVersion << "term" << initConfigTerm << "members" << BSON_ARRAY(BSON("_id" << 10 << "host" << "hself") << BSON("_id" << 20 << "host" @@ -1880,6 +1880,9 @@ public: Status* result) { *result = getTopoCoord().prepareHeartbeatResponseV1(now()++, args, "rs0", response); } + + int initConfigVersion = 1; + int initConfigTerm = 1; }; TEST_F(PrepareHeartbeatResponseV1Test, @@ -2004,7 +2007,52 @@ TEST_F(PrepareHeartbeatResponseV1Test, PopulateHeartbeatResponseWithFullConfigWhenHeartbeatRequestHasAnOldConfigVersion) { // set up args with a config version lower than ours ReplSetHeartbeatArgsV1 args; - args.setConfigVersion(0); + args.setConfigVersion(initConfigVersion - 1); + args.setConfigTerm(initConfigTerm); + args.setSetName("rs0"); + args.setSenderId(20); + ReplSetHeartbeatResponse response; + Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); + + // prepare response and check the results + prepareHeartbeatResponseV1(args, &response, &result); + ASSERT_OK(result); + ASSERT_TRUE(response.hasConfig()); + ASSERT_EQUALS("rs0", response.getReplicaSetName()); + ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s); + ASSERT_EQUALS(OpTime(), response.getDurableOpTime()); + ASSERT_EQUALS(0, response.getTerm()); + ASSERT_EQUALS(1, response.getConfigVersion()); +} + +TEST_F(PrepareHeartbeatResponseV1Test, + PopulateHeartbeatResponseWithFullConfigWhenHeartbeatRequestHasAnOldConfigTerm) { + // set up args with a config version lower than ours + ReplSetHeartbeatArgsV1 args; + args.setConfigVersion(initConfigVersion); + args.setConfigTerm(initConfigTerm - 1); + args.setSetName("rs0"); + args.setSenderId(20); + ReplSetHeartbeatResponse response; + Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); + + // prepare response and check the results + prepareHeartbeatResponseV1(args, &response, &result); + ASSERT_OK(result); + ASSERT_TRUE(response.hasConfig()); + ASSERT_EQUALS("rs0", response.getReplicaSetName()); + ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s); + ASSERT_EQUALS(OpTime(), response.getDurableOpTime()); + ASSERT_EQUALS(0, response.getTerm()); + ASSERT_EQUALS(1, response.getConfigVersion()); +} + +TEST_F(PrepareHeartbeatResponseV1Test, + PopulateHeartbeatResponseWithFullConfigWhenHeartbeatRequestHasAnOlderTermButNewerVersion) { + // set up args with a config version lower than ours + ReplSetHeartbeatArgsV1 args; + args.setConfigVersion(initConfigVersion + 1); + args.setConfigTerm(initConfigTerm - 1); args.setSetName("rs0"); args.setSenderId(20); ReplSetHeartbeatResponse response; @@ -2026,6 +2074,30 @@ TEST_F(PrepareHeartbeatResponseV1Test, // set up args with a config version higher than ours ReplSetHeartbeatArgsV1 args; args.setConfigVersion(10); + args.setConfigTerm(1); + args.setSetName("rs0"); + args.setSenderId(20); + ReplSetHeartbeatResponse response; + Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); + + // prepare response and check the results + prepareHeartbeatResponseV1(args, &response, &result); + ASSERT_OK(result); + ASSERT_FALSE(response.hasConfig()); + ASSERT_EQUALS("rs0", response.getReplicaSetName()); + ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s); + ASSERT_EQUALS(OpTime(), response.getDurableOpTime()); + ASSERT_EQUALS(0, response.getTerm()); + ASSERT_EQUALS(1, response.getConfigVersion()); +} + +TEST_F(PrepareHeartbeatResponseV1Test, + PopulateFullHeartbeatResponseWhenHeartbeatRequestHasANewerConfigTerm) { + // set up args with a config term higher than ours + ReplSetHeartbeatArgsV1 args; + args.setConfigVersion(1); + args.setConfigTerm(10); + args.setTerm(10); args.setSetName("rs0"); args.setSenderId(20); ReplSetHeartbeatResponse response; @@ -3170,7 +3242,7 @@ public: TopoCoordTest::setUp(); updateConfig(BSON("_id" << "rs0" - << "version" << 5 << "members" + << "version" << 5 << "term" << 1 << "members" << BSON_ARRAY(BSON("_id" << 0 << "host" << "host1:27017") << BSON("_id" << 1 << "host" @@ -3517,6 +3589,187 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenMemberNotInConfig) HostAndPort("host4"), replMetadata, makeOplogQueryMetadata(), now())); } +class HeartbeatResponseReconfigTestV1 : public TopoCoordTest { +public: + virtual void setUp() { + TopoCoordTest::setUp(); + updateConfig(makeRSConfigWithVersionAndTerm(initConfigVersion, initConfigTerm), 0); + setSelfMemberState(MemberState::RS_SECONDARY); + } + + BSONObj makeRSConfigWithVersionAndTerm(long long version, long long term) { + return BSON("_id" + << "rs0" + << "version" << version << "term" << term << "members" + << BSON_ARRAY(BSON("_id" << 0 << "host" + << "host1:27017") + << BSON("_id" << 1 << "host" + << "host2:27017") + << BSON("_id" << 2 << "host" + << "host3:27017")) + << "protocolVersion" << 1 << "settings" << BSON("heartbeatTimeoutSecs" << 5)); + } + + long long initConfigVersion = 2; + long long initConfigTerm = 2; +}; + +TEST_F(HeartbeatResponseReconfigTestV1, NodeAcceptsConfigIfVersionInHeartbeatResponseIsNewer) { + ReplSetConfig config; + long long version = initConfigVersion + 1; + long long term = initConfigTerm; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 1) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host2")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host2"), hbResponse); + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::Reconfig); +} + +TEST_F(HeartbeatResponseReconfigTestV1, NodeAcceptsConfigIfTermInHeartbeatResponseIsNewer) { + ReplSetConfig config; + long long version = initConfigVersion; + long long term = initConfigTerm + 1; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 1) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host2")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host2"), hbResponse); + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::Reconfig); +} + +TEST_F(HeartbeatResponseReconfigTestV1, + NodeAcceptsConfigIfVersionInHeartbeatResponseIfNewerAndTermUninitialized) { + ReplSetConfig config; + long long version = initConfigVersion + 1; + long long term = OpTime::kUninitializedTerm; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 1) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host2")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host2"), hbResponse); + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::Reconfig); +} + +TEST_F(HeartbeatResponseReconfigTestV1, NodeRejectsConfigInHeartbeatResponseIfVersionIsOlder) { + // Older config version, same term. + ReplSetConfig config; + long long version = (initConfigVersion - 1); + long long term = initConfigTerm; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 1) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host2")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host2"), hbResponse); + // Don't reconfig to an older config. + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::NoAction); +} + +TEST_F(HeartbeatResponseReconfigTestV1, NodeRejectsConfigInHeartbeatResponseIfConfigIsTheSame) { + ReplSetConfig config; + long long version = initConfigVersion; + long long term = initConfigTerm; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 0) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host3")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host3"), hbResponse); + // Don't reconfig to the same config. + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::NoAction); +} + +TEST_F(HeartbeatResponseReconfigTestV1, NodeRejectsConfigInHeartbeatResponseIfTermIsOlder) { + // Older config term, same version + ReplSetConfig config; + long long version = initConfigVersion; + long long term = initConfigTerm - 1; + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 0) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host3")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host3"), hbResponse); + // Don't reconfig to an older config. + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::NoAction); +} + +TEST_F(HeartbeatResponseReconfigTestV1, + NodeRejectsConfigInHeartbeatResponseIfNewerVersionButOlderTerm) { + // Newer version but older term. + ReplSetConfig config; + long long version = (initConfigVersion + 1); + long long term = (initConfigTerm - 1); + config.initialize(makeRSConfigWithVersionAndTerm(version, term)).transitional_ignore(); + + ReplSetHeartbeatResponse hb; + hb.initialize(BSON("ok" << 1 << "v" << 1 << "state" << MemberState::RS_SECONDARY), 0) + .transitional_ignore(); + hb.setConfig(config); + hb.setConfigVersion(version); + hb.setConfigTerm(term); + StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + + getTopoCoord().prepareHeartbeatRequestV1(now(), "rs0", HostAndPort("host3")); + now() += Milliseconds(1); + HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( + now(), Milliseconds(1), HostAndPort("host3"), hbResponse); + // Don't reconfig to an older config. + ASSERT_EQ(action.getAction(), HeartbeatResponseAction::NoAction); +} + // TODO(dannenberg) figure out what this is trying to test.. TEST_F(HeartbeatResponseTestV1, ReconfigNodeRemovedBetweenHeartbeatRequestAndRepsonse) { OpTime election = OpTime(Timestamp(14, 0), 0); |