diff options
author | Matthew Russotto <matthew.russotto@10gen.com> | 2017-05-15 14:33:22 -0400 |
---|---|---|
committer | Matthew Russotto <matthew.russotto@10gen.com> | 2017-05-15 15:26:27 -0400 |
commit | c88c4809c2440d286ed0fc29e1e8d684f015e563 (patch) | |
tree | 71962294640d0ddecacda316e8e1eda2323c9333 | |
parent | b69aed9d10ef66de42880fd379b0a593419b6e47 (diff) | |
download | mongo-c88c4809c2440d286ed0fc29e1e8d684f015e563.tar.gz |
SERVER-26990 Unify tracking of secondary state between replication and topology coordinators
This fixes a bug in the 6adc71f6cf069803f9c1288aef88ffe0d21c6ffe which caused crashes when a
sync source change was requested of a node not in the configuration. It also fixes a dependency
problem affecting the shared library build.
-rw-r--r-- | src/mongo/db/repl/SConscript | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/heartbeat_response_action.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/heartbeat_response_action.h | 14 | ||||
-rw-r--r-- | src/mongo/db/repl/member_heartbeat_data.cpp | 56 | ||||
-rw-r--r-- | src/mongo/db/repl/member_heartbeat_data.h | 137 | ||||
-rw-r--r-- | src/mongo/db/repl/repl_set_html_summary.cpp | 4 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.cpp | 559 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl.h | 137 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_elect.cpp | 5 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp | 10 | ||||
-rw-r--r-- | src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp | 103 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator.h | 164 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_impl.cpp | 594 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_impl.h | 103 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_impl_test.cpp | 929 | ||||
-rw-r--r-- | src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp | 1020 |
16 files changed, 1670 insertions, 2173 deletions
diff --git a/src/mongo/db/repl/SConscript b/src/mongo/db/repl/SConscript index 335cd86b51b..40ef50f2e82 100644 --- a/src/mongo/db/repl/SConscript +++ b/src/mongo/db/repl/SConscript @@ -521,16 +521,18 @@ env.CppUnitTest( env.Library('topology_coordinator', [ 'heartbeat_response_action.cpp', + 'member_heartbeat_data.cpp', 'topology_coordinator.cpp', ], LIBDEPS=[ 'repl_coordinator_interface', + 'replica_set_messages', + 'rslog', '$BUILD_DIR/mongo/rpc/metadata', ]) env.Library('topology_coordinator_impl', [ - 'member_heartbeat_data.cpp', 'topology_coordinator_impl.cpp', ], LIBDEPS=[ diff --git a/src/mongo/db/repl/heartbeat_response_action.cpp b/src/mongo/db/repl/heartbeat_response_action.cpp index 97bfbd0c29a..7e787d42c96 100644 --- a/src/mongo/db/repl/heartbeat_response_action.cpp +++ b/src/mongo/db/repl/heartbeat_response_action.cpp @@ -75,5 +75,9 @@ void HeartbeatResponseAction::setNextHeartbeatStartDate(Date_t when) { _nextHeartbeatStartDate = when; } +void HeartbeatResponseAction::setAdvancedOpTime(bool advanced) { + _advancedOpTime = advanced; +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/heartbeat_response_action.h b/src/mongo/db/repl/heartbeat_response_action.h index 23c35d9c884..bb009600e5a 100644 --- a/src/mongo/db/repl/heartbeat_response_action.h +++ b/src/mongo/db/repl/heartbeat_response_action.h @@ -101,6 +101,11 @@ public: void setNextHeartbeatStartDate(Date_t when); /** + * Sets whether or not the heartbeat response advanced the member's opTime. + */ + void setAdvancedOpTime(bool advanced); + + /** * Gets the action type of this action. */ Action getAction() const { @@ -123,10 +128,19 @@ public: return _primaryIndex; } + /* + * Returns true if the heartbeat response resulting in our conception of the + * member's optime moving forward, so we need to recalculate lastCommittedOpTime. + */ + bool getAdvancedOpTime() const { + return _advancedOpTime; + } + private: Action _action; int _primaryIndex; Date_t _nextHeartbeatStartDate; + bool _advancedOpTime = false; }; } // namespace repl diff --git a/src/mongo/db/repl/member_heartbeat_data.cpp b/src/mongo/db/repl/member_heartbeat_data.cpp index 1b9b9ea3f13..7b553a1e682 100644 --- a/src/mongo/db/repl/member_heartbeat_data.cpp +++ b/src/mongo/db/repl/member_heartbeat_data.cpp @@ -39,13 +39,14 @@ namespace mongo { namespace repl { -MemberHeartbeatData::MemberHeartbeatData() : _health(-1), _authIssue(false) { +MemberHeartbeatData::MemberHeartbeatData() + : _health(-1), _authIssue(false), _configIndex(-1), _isSelf(false) { _lastResponse.setState(MemberState::RS_UNKNOWN); _lastResponse.setElectionTime(Timestamp()); _lastResponse.setAppliedOpTime(OpTime()); } -void MemberHeartbeatData::setUpValues(Date_t now, +bool MemberHeartbeatData::setUpValues(Date_t now, const HostAndPort& host, ReplSetHeartbeatResponse&& hbResponse) { _health = 1; @@ -54,6 +55,8 @@ void MemberHeartbeatData::setUpValues(Date_t now, } _authIssue = false; _lastHeartbeat = now; + _lastUpdate = now; + _lastUpdateStale = false; _updatedSinceRestart = true; if (!hbResponse.hasState()) { @@ -71,7 +74,11 @@ void MemberHeartbeatData::setUpValues(Date_t now, << hbResponse.getState().toString() << rsLog; } + bool opTimeAdvanced = advanceLastAppliedOpTime(hbResponse.getAppliedOpTime(), now); + auto durableOpTime = hbResponse.hasDurableOpTime() ? hbResponse.getDurableOpTime() : OpTime(); + opTimeAdvanced = advanceLastDurableOpTime(durableOpTime, now) || opTimeAdvanced; _lastResponse = std::move(hbResponse); + return opTimeAdvanced; } void MemberHeartbeatData::setDownValues(Date_t now, const std::string& heartbeatMessage) { @@ -87,6 +94,9 @@ void MemberHeartbeatData::setDownValues(Date_t now, const std::string& heartbeat _lastResponse.setAppliedOpTime(OpTime()); _lastResponse.setHbMsg(heartbeatMessage); _lastResponse.setSyncingTo(HostAndPort()); + + // The _lastAppliedOpTime/_lastDurableOpTime fields don't get cleared merely by missing a + // heartbeat. } void MemberHeartbeatData::setAuthIssue(Date_t now) { @@ -104,5 +114,47 @@ void MemberHeartbeatData::setAuthIssue(Date_t now) { _lastResponse.setSyncingTo(HostAndPort()); } +void MemberHeartbeatData::setLastAppliedOpTime(OpTime opTime, Date_t now) { + _lastUpdate = now; + _lastUpdateStale = false; + _lastAppliedOpTime = opTime; +} + +void MemberHeartbeatData::setLastDurableOpTime(OpTime opTime, Date_t now) { + _lastUpdate = now; + _lastUpdateStale = false; + if (_lastAppliedOpTime < opTime) { + // TODO(russotto): We think this should never happen, rollback or no rollback. Make this an + // invariant and see what happens. + log() << "Durable progress (" << opTime << ") is ahead of the applied progress (" + << _lastAppliedOpTime << ". This is likely due to a " + "rollback." + << " memberid: " << _memberId << " rid: " << _rid << " host " + << _hostAndPort.toString() << " previous durable progress: " << _lastDurableOpTime; + } else { + _lastDurableOpTime = opTime; + } +} + +bool MemberHeartbeatData::advanceLastAppliedOpTime(OpTime opTime, Date_t now) { + _lastUpdate = now; + _lastUpdateStale = false; + if (_lastAppliedOpTime < opTime) { + setLastAppliedOpTime(opTime, now); + return true; + } + return false; +} + +bool MemberHeartbeatData::advanceLastDurableOpTime(OpTime opTime, Date_t now) { + _lastUpdate = now; + _lastUpdateStale = false; + if (_lastDurableOpTime < opTime) { + setLastDurableOpTime(opTime, now); + return true; + } + return false; +} + } // namespace repl } // namespace mongo diff --git a/src/mongo/db/repl/member_heartbeat_data.h b/src/mongo/db/repl/member_heartbeat_data.h index f67a0a87757..4122ff86ec2 100644 --- a/src/mongo/db/repl/member_heartbeat_data.h +++ b/src/mongo/db/repl/member_heartbeat_data.h @@ -68,10 +68,10 @@ public: const HostAndPort& getSyncSource() const { return _lastResponse.getSyncingTo(); } - OpTime getAppliedOpTime() const { + OpTime getHeartbeatAppliedOpTime() const { return _lastResponse.getAppliedOpTime(); } - OpTime getDurableOpTime() const { + OpTime getHeartbeatDurableOpTime() const { return _lastResponse.hasDurableOpTime() ? _lastResponse.getDurableOpTime() : OpTime(); } int getConfigVersion() const { @@ -105,10 +105,49 @@ public: return _health != 0; } + OpTime getLastAppliedOpTime() const { + return _lastAppliedOpTime; + } + + OpTime getLastDurableOpTime() const { + return _lastDurableOpTime; + } + + // When was the last time this data was updated via any means? + Date_t getLastUpdate() const { + return _lastUpdate; + } + // Was the last update stale as of the last check? + bool lastUpdateStale() const { + return _lastUpdateStale; + } + + // Index of this member in the replica set config member list. + int getConfigIndex() const { + return _configIndex; + } + + int getMemberId() const { + return _memberId; + } + + OID getRid() const { + return _rid; + } + + bool isSelf() const { + return _isSelf; + } + + HostAndPort getHostAndPort() const { + return _hostAndPort; + } + /** * Sets values in this object from the results of a successful heartbeat command. + * Returns whether or not the optimes advanced as a result of this heartbeat response. */ - void setUpValues(Date_t now, const HostAndPort& host, ReplSetHeartbeatResponse&& hbResponse); + bool setUpValues(Date_t now, const HostAndPort& host, ReplSetHeartbeatResponse&& hbResponse); /** * Sets values in this object from the results of a erroring/failed heartbeat command. @@ -134,6 +173,66 @@ public: return _updatedSinceRestart; } + /** + * Sets the last applied op time (not the heartbeat applied op time) and updates the + * lastUpdate time. + */ + void setLastAppliedOpTime(OpTime opTime, Date_t now); + + /** + * Sets the last durable op time (not the heartbeat durable op time) + */ + void setLastDurableOpTime(OpTime opTime, Date_t now); + + /** + * Sets the last applied op time (not the heartbeat applied op time) iff the new optime is + * later than the current optime, and updates the lastUpdate time. Returns true if the + * optime was advanced. + */ + bool advanceLastAppliedOpTime(OpTime opTime, Date_t now); + + /** + * Sets the last durable op time (not the heartbeat applied op time) iff the new optime is + * later than the current optime, and updates the lastUpdate time. Returns true if the + * optime was advanced. + */ + bool advanceLastDurableOpTime(OpTime opTime, Date_t now); + + /* + * Indicates that this data is stale, based on _lastUpdateTime. + */ + void markLastUpdateStale() { + _lastUpdateStale = true; + } + + /* + * Updates the _lastUpdateTime and clears staleness without changing anything else. + */ + void updateLiveness(Date_t now) { + _lastUpdate = now; + _lastUpdateStale = false; + } + + void setConfigIndex(int configIndex) { + _configIndex = configIndex; + } + + void setIsSelf(bool isSelf) { + _isSelf = isSelf; + } + + void setHostAndPort(HostAndPort hostAndPort) { + _hostAndPort = hostAndPort; + } + + void setMemberId(int memberId) { + _memberId = memberId; + } + + void setRid(OID rid) { + _rid = rid; + } + private: // -1 = not checked yet, 0 = member is down/unreachable, 1 = member is up int _health; @@ -153,6 +252,38 @@ private: // Have we received heartbeats since the last restart? bool _updatedSinceRestart = false; + + // Last time we got any information about this member, whether heartbeat + // or replSetUpdatePosition. + Date_t _lastUpdate; + + // Set when lastUpdate time exceeds the election timeout. Implies that the member is down + // on the primary, but not the secondaries. + bool _lastUpdateStale = false; + + // Last known OpTime that the replica has applied and journaled to. + OpTime _lastDurableOpTime; + + // Last known OpTime that the replica has applied, whether journaled or unjournaled. + OpTime _lastAppliedOpTime; + + // TODO(russotto): Since memberHeartbeatData is kept in config order, _configIndex + // and _isSelf may not be necessary. + // Index of this member in the replica set configuration. + int _configIndex; + + // Is this the data for this member? + bool _isSelf; + + // This member's RID, used only in master/slave replication. + OID _rid; + + // This member's member ID. memberId and hostAndPort duplicate information in the + // configuration for replica sets, but are required to be here for master/slave replication. + int _memberId = -1; + + // Client address of this member. + HostAndPort _hostAndPort; }; } // namespace repl diff --git a/src/mongo/db/repl/repl_set_html_summary.cpp b/src/mongo/db/repl/repl_set_html_summary.cpp index 14c2ff81b7d..d59a7c31649 100644 --- a/src/mongo/db/repl/repl_set_html_summary.cpp +++ b/src/mongo/db/repl/repl_set_html_summary.cpp @@ -187,7 +187,7 @@ const std::string ReplSetHtmlSummary::toHtmlString() const { // TODO(dannenberg): change timestamp to optime in V1 memberTable << td(memberHB.getLastHeartbeat() == Date_t() ? "?" - : memberHB.getAppliedOpTime().toString()); + : memberHB.getHeartbeatAppliedOpTime().toString()); } memberTable << _tr(); } @@ -201,7 +201,7 @@ const std::string ReplSetHtmlSummary::toHtmlString() const { const MemberConfig& selfConfig = _config.getMemberAt(_selfIndex); if (_primaryIndex >= 0 && _primaryIndex != _selfIndex && !selfConfig.isArbiter()) { - int lag = _hbData[_primaryIndex].getAppliedOpTime().getTimestamp().getSecs() - + int lag = _hbData[_primaryIndex].getHeartbeatAppliedOpTime().getTimestamp().getSecs() - _selfOptime.getTimestamp().getSecs(); s << tr("Lag: ", str::stream() << lag << " secs"); } diff --git a/src/mongo/db/repl/replication_coordinator_impl.cpp b/src/mongo/db/repl/replication_coordinator_impl.cpp index b4138baa290..7591746797d 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl.cpp @@ -135,25 +135,6 @@ BSONObj incrementConfigVersionByRandom(BSONObj config) { const Seconds kNoopWriterPeriod(10); } // namespace -BSONObj ReplicationCoordinatorImpl::SlaveInfo::toBSON() const { - BSONObjBuilder bo; - bo.append("id", memberId); - bo.append("rid", rid); - bo.append("host", hostAndPort.toString()); - bo.append("lastDurableOpTime", lastDurableOpTime.toBSON()); - bo.append("lastAppliedOpTime", lastAppliedOpTime.toBSON()); - if (self) - bo.append("self", true); - if (down) - bo.append("down", true); - bo.append("lastUpdated", lastUpdate); - return bo.obj(); -} - -std::string ReplicationCoordinatorImpl::SlaveInfo::toString() const { - return toBSON().toString(); -} - ReplicationCoordinatorImpl::Waiter::Waiter(OpTime _opTime, const WriteConcernOptions* _writeConcern) : opTime(std::move(_opTime)), writeConcern(_writeConcern) {} @@ -331,11 +312,6 @@ ReplicationCoordinatorImpl::ReplicationCoordinatorImpl( return; } - // Make sure there is always an entry in _slaveInfo for ourself. - SlaveInfo selfInfo; - selfInfo.self = true; - _slaveInfo.push_back(selfInfo); - _externalState->setupNoopWriter(kNoopWriterPeriod); } @@ -681,7 +657,7 @@ void ReplicationCoordinatorImpl::startup(OperationContext* opCtx) { fassert(18822, !_inShutdown); _setConfigState_inlock(kConfigStartingUp); _myRID = rid; - _slaveInfo[_getMyIndexInSlaveInfo_inlock()].rid = rid; + _topCoord->getMyMemberHeartbeatData()->setRid(rid); } if (!_settings.usingReplSets()) { @@ -917,7 +893,7 @@ void ReplicationCoordinatorImpl::signalDrainComplete(OperationContext* opCtx, lk.unlock(); OpTime firstOpTime = _externalState->onTransitionToPrimary(opCtx, isV1ElectionProtocol()); lk.lock(); - _setFirstOpTimeOfMyTerm_inlock(firstOpTime); + _topCoord->setFirstOpTimeOfMyTerm(firstOpTime); // Must calculate the commit level again because firstOpTimeOfMyTerm wasn't set when we logged // our election in onTransitionToPrimary(), above. @@ -946,127 +922,6 @@ void ReplicationCoordinatorImpl::signalUpstreamUpdater() { _externalState->forwardSlaveProgress(); } -ReplicationCoordinatorImpl::SlaveInfo* ReplicationCoordinatorImpl::_findSlaveInfoByMemberID_inlock( - int memberId) { - for (SlaveInfoVector::iterator it = _slaveInfo.begin(); it != _slaveInfo.end(); ++it) { - if (it->memberId == memberId) { - return &(*it); - } - } - return NULL; -} - -ReplicationCoordinatorImpl::SlaveInfo* ReplicationCoordinatorImpl::_findSlaveInfoByRID_inlock( - const OID& rid) { - for (SlaveInfoVector::iterator it = _slaveInfo.begin(); it != _slaveInfo.end(); ++it) { - if (it->rid == rid) { - return &(*it); - } - } - return NULL; -} - -void ReplicationCoordinatorImpl::_addSlaveInfo_inlock(const SlaveInfo& slaveInfo) { - invariant(getReplicationMode() == modeMasterSlave); - _slaveInfo.push_back(slaveInfo); - - _updateLastCommittedOpTime_inlock(); - // Wake up any threads waiting for replication that now have their replication - // check satisfied - _wakeReadyWaiters_inlock(); -} - -void ReplicationCoordinatorImpl::_updateSlaveInfoAppliedOpTime_inlock(SlaveInfo* slaveInfo, - const OpTime& opTime) { - slaveInfo->lastAppliedOpTime = opTime; - slaveInfo->lastUpdate = _replExecutor->now(); - slaveInfo->down = false; - - _updateLastCommittedOpTime_inlock(); - // Wake up any threads waiting for replication that now have their replication - // check satisfied - _wakeReadyWaiters_inlock(); -} - -void ReplicationCoordinatorImpl::_updateSlaveInfoDurableOpTime_inlock(SlaveInfo* slaveInfo, - const OpTime& opTime) { - // lastAppliedOpTime cannot be behind lastDurableOpTime. - if (slaveInfo->lastAppliedOpTime < opTime) { - log() << "Durable progress (" << opTime << ") is ahead of the applied progress (" - << slaveInfo->lastAppliedOpTime << ". This is likely due to a " - "rollback. slaveInfo: " - << slaveInfo->toString(); - return; - } - slaveInfo->lastDurableOpTime = opTime; - slaveInfo->lastUpdate = _replExecutor->now(); - slaveInfo->down = false; - - _updateLastCommittedOpTime_inlock(); - // Wake up any threads waiting for replication that now have their replication - // check satisfied - _wakeReadyWaiters_inlock(); -} - -void ReplicationCoordinatorImpl::_updateSlaveInfoFromConfig_inlock() { - invariant(_settings.usingReplSets()); - - SlaveInfoVector oldSlaveInfos; - _slaveInfo.swap(oldSlaveInfos); - - if (_selfIndex == -1) { - // If we aren't in the config then the only data we care about is for ourself - for (SlaveInfoVector::const_iterator it = oldSlaveInfos.begin(); it != oldSlaveInfos.end(); - ++it) { - if (it->self) { - SlaveInfo slaveInfo = *it; - slaveInfo.memberId = -1; - _slaveInfo.push_back(slaveInfo); - return; - } - } - invariant(false); // There should always have been an entry for ourself - } - - for (int i = 0; i < _rsConfig.getNumMembers(); ++i) { - const MemberConfig& memberConfig = _rsConfig.getMemberAt(i); - int memberId = memberConfig.getId(); - const HostAndPort& memberHostAndPort = memberConfig.getHostAndPort(); - - SlaveInfo slaveInfo; - - // Check if the node existed with the same member ID and hostname in the old data - for (SlaveInfoVector::const_iterator it = oldSlaveInfos.begin(); it != oldSlaveInfos.end(); - ++it) { - if ((it->memberId == memberId && it->hostAndPort == memberHostAndPort) || - (i == _selfIndex && it->self)) { - slaveInfo = *it; - } - } - - // Make sure you have the most up-to-date info for member ID and hostAndPort. - slaveInfo.memberId = memberId; - slaveInfo.hostAndPort = memberHostAndPort; - _slaveInfo.push_back(slaveInfo); - } - invariant(static_cast<int>(_slaveInfo.size()) == _rsConfig.getNumMembers()); -} - -size_t ReplicationCoordinatorImpl::_getMyIndexInSlaveInfo_inlock() const { - if (getReplicationMode() == modeMasterSlave) { - // Self data always lives in the first entry in _slaveInfo for master/slave - return 0; - } else { - invariant(_settings.usingReplSets()); - if (_selfIndex == -1) { - invariant(_slaveInfo.size() == 1); - return 0; - } else { - return _selfIndex; - } - } -} - Status ReplicationCoordinatorImpl::setLastOptimeForSlave(const OID& rid, const Timestamp& ts) { stdx::unique_lock<stdx::mutex> lock(_mutex); massert(28576, @@ -1077,17 +932,14 @@ Status ReplicationCoordinatorImpl::setLastOptimeForSlave(const OID& rid, const T // term == -1 for master-slave OpTime opTime(ts, OpTime::kUninitializedTerm); - SlaveInfo* slaveInfo = _findSlaveInfoByRID_inlock(rid); - if (slaveInfo) { - if (slaveInfo->lastAppliedOpTime < opTime) { - _updateSlaveInfoAppliedOpTime_inlock(slaveInfo, opTime); - } + MemberHeartbeatData* memberHeartbeatData = _topCoord->findMemberHeartbeatDataByRid(rid); + if (memberHeartbeatData) { + memberHeartbeatData->advanceLastAppliedOpTime(opTime, _replExecutor->now()); } else { - SlaveInfo newSlaveInfo; - newSlaveInfo.rid = rid; - newSlaveInfo.lastAppliedOpTime = opTime; - _addSlaveInfo_inlock(newSlaveInfo); + auto* memberHeartbeatData = _topCoord->addSlaveMemberData(rid); + memberHeartbeatData->setLastAppliedOpTime(opTime, _replExecutor->now()); } + _updateLastCommittedOpTime_inlock(); return Status::OK(); } @@ -1156,27 +1008,20 @@ void ReplicationCoordinatorImpl::_reportUpstream_inlock(stdx::unique_lock<stdx:: void ReplicationCoordinatorImpl::_setMyLastAppliedOpTime_inlock(const OpTime& opTime, bool isRollbackAllowed) { - SlaveInfo* mySlaveInfo = &_slaveInfo[_getMyIndexInSlaveInfo_inlock()]; - invariant(isRollbackAllowed || mySlaveInfo->lastAppliedOpTime <= opTime); - _updateSlaveInfoAppliedOpTime_inlock(mySlaveInfo, opTime); - + auto* myMemberHeartbeatData = _topCoord->getMyMemberHeartbeatData(); + invariant(isRollbackAllowed || myMemberHeartbeatData->getLastAppliedOpTime() <= opTime); + myMemberHeartbeatData->setLastAppliedOpTime(opTime, _replExecutor->now()); + _updateLastCommittedOpTime_inlock(); _opTimeWaiterList.signalAndRemoveIf_inlock( [opTime](Waiter* waiter) { return waiter->opTime <= opTime; }); } void ReplicationCoordinatorImpl::_setMyLastDurableOpTime_inlock(const OpTime& opTime, bool isRollbackAllowed) { - SlaveInfo* mySlaveInfo = &_slaveInfo[_getMyIndexInSlaveInfo_inlock()]; - invariant(isRollbackAllowed || mySlaveInfo->lastDurableOpTime <= opTime); - // lastAppliedOpTime cannot be behind lastDurableOpTime. - if (mySlaveInfo->lastAppliedOpTime < opTime) { - log() << "My durable progress (" << opTime << ") is ahead of my applied progress (" - << mySlaveInfo->lastAppliedOpTime << ". This is likely due to a " - "rollback. slaveInfo: " - << mySlaveInfo->toString(); - return; - } - _updateSlaveInfoDurableOpTime_inlock(mySlaveInfo, opTime); + auto* myMemberHeartbeatData = _topCoord->getMyMemberHeartbeatData(); + invariant(isRollbackAllowed || myMemberHeartbeatData->getLastDurableOpTime() <= opTime); + myMemberHeartbeatData->setLastDurableOpTime(opTime, _replExecutor->now()); + _updateLastCommittedOpTime_inlock(); } OpTime ReplicationCoordinatorImpl::getMyLastAppliedOpTime() const { @@ -1343,11 +1188,11 @@ Status ReplicationCoordinatorImpl::_waitUntilOpTimeForReadDeprecated( } OpTime ReplicationCoordinatorImpl::_getMyLastAppliedOpTime_inlock() const { - return _slaveInfo[_getMyIndexInSlaveInfo_inlock()].lastAppliedOpTime; + return _topCoord->getMyLastAppliedOpTime(); } OpTime ReplicationCoordinatorImpl::_getMyLastDurableOpTime_inlock() const { - return _slaveInfo[_getMyIndexInSlaveInfo_inlock()].lastDurableOpTime; + return _topCoord->getMyLastDurableOpTime(); } Status ReplicationCoordinatorImpl::setLastDurableOptime_forTest(long long cfgVer, @@ -1402,7 +1247,6 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock( << " in config with version " << args.cfgver << " has durably reached optime: " << args.ts; - SlaveInfo* slaveInfo = NULL; if (args.cfgver != _rsConfig.getConfigVersion()) { std::string errmsg = str::stream() << "Received replSetUpdatePosition for node with memberId " << args.memberId @@ -1413,8 +1257,8 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock( return Status(ErrorCodes::InvalidReplicaSetConfig, errmsg); } - slaveInfo = _findSlaveInfoByMemberID_inlock(args.memberId); - if (!slaveInfo) { + auto* memberHeartbeatData = _topCoord->findMemberHeartbeatDataByMemberId(args.memberId); + if (!memberHeartbeatData) { invariant(!_rsConfig.findMemberByID(args.memberId)); std::string errmsg = str::stream() @@ -1424,25 +1268,22 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock( return Status(ErrorCodes::NodeNotFound, errmsg); } - invariant(args.memberId == slaveInfo->memberId); + invariant(args.memberId == memberHeartbeatData->getMemberId()); - LOG(3) << "Node with memberID " << args.memberId << " has durably applied operationss through " - << slaveInfo->lastDurableOpTime << " and has applied operations through " - << slaveInfo->lastAppliedOpTime << "; updating to new durable operation with timestamp " - << args.ts; + LOG(3) << "Node with memberID " << args.memberId << " has durably applied operations through " + << memberHeartbeatData->getLastDurableOpTime() << " and has applied operations through " + << memberHeartbeatData->getLastAppliedOpTime() + << "; updating to new durable operation with timestamp " << args.ts; - // Only update remote optimes if they increase. - if (slaveInfo->lastAppliedOpTime < args.ts) { - _updateSlaveInfoAppliedOpTime_inlock(slaveInfo, args.ts); - } - if (slaveInfo->lastDurableOpTime < args.ts) { - _updateSlaveInfoDurableOpTime_inlock(slaveInfo, args.ts); - } + auto now(_replExecutor->now()); + bool advancedOpTime = memberHeartbeatData->advanceLastAppliedOpTime(args.ts, now); + advancedOpTime = memberHeartbeatData->advanceLastDurableOpTime(args.ts, now) || advancedOpTime; + // Only update committed optime if the remote optimes increased. + if (advancedOpTime) { + _updateLastCommittedOpTime_inlock(); + } - // Update liveness for this node. - slaveInfo->lastUpdate = _replExecutor->now(); - slaveInfo->down = false; _cancelAndRescheduleLivenessUpdate_inlock(args.memberId); return Status::OK(); } @@ -1474,7 +1315,6 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock(const UpdatePositionArg << " has reached optime: " << args.appliedOpTime << " and is durable through: " << args.durableOpTime; - SlaveInfo* slaveInfo = NULL; if (args.cfgver != _rsConfig.getConfigVersion()) { std::string errmsg = str::stream() << "Received replSetUpdatePosition for node with memberId " << args.memberId @@ -1485,8 +1325,8 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock(const UpdatePositionArg return Status(ErrorCodes::InvalidReplicaSetConfig, errmsg); } - slaveInfo = _findSlaveInfoByMemberID_inlock(args.memberId); - if (!slaveInfo) { + auto* memberHeartbeatData = _topCoord->findMemberHeartbeatDataByMemberId(args.memberId); + if (!memberHeartbeatData) { invariant(!_rsConfig.findMemberByID(args.memberId)); std::string errmsg = str::stream() @@ -1496,25 +1336,24 @@ Status ReplicationCoordinatorImpl::_setLastOptime_inlock(const UpdatePositionArg return Status(ErrorCodes::NodeNotFound, errmsg); } - invariant(args.memberId == slaveInfo->memberId); + invariant(args.memberId == memberHeartbeatData->getMemberId()); LOG(3) << "Node with memberID " << args.memberId << " currently has optime " - << slaveInfo->lastAppliedOpTime << " durable through " << slaveInfo->lastDurableOpTime - << "; updating to optime " << args.appliedOpTime << " and durable through " - << args.durableOpTime; + << memberHeartbeatData->getLastAppliedOpTime() << " durable through " + << memberHeartbeatData->getLastDurableOpTime() << "; updating to optime " + << args.appliedOpTime << " and durable through " << args.durableOpTime; - // Only update remote optimes if they increase. - if (slaveInfo->lastAppliedOpTime < args.appliedOpTime) { - _updateSlaveInfoAppliedOpTime_inlock(slaveInfo, args.appliedOpTime); - } - if (slaveInfo->lastDurableOpTime < args.durableOpTime) { - _updateSlaveInfoDurableOpTime_inlock(slaveInfo, args.durableOpTime); + auto now(_replExecutor->now()); + bool advancedOpTime = memberHeartbeatData->advanceLastAppliedOpTime(args.appliedOpTime, now); + advancedOpTime = + memberHeartbeatData->advanceLastDurableOpTime(args.durableOpTime, now) || advancedOpTime; + + // Only update committed optime if the remote optimes increased. + if (advancedOpTime) { + _updateLastCommittedOpTime_inlock(); } - // Update liveness for this node. - slaveInfo->lastUpdate = _replExecutor->now(); - slaveInfo->down = false; _cancelAndRescheduleLivenessUpdate_inlock(args.memberId); return Status::OK(); } @@ -1531,7 +1370,8 @@ bool ReplicationCoordinatorImpl::_doneWaitingForReplication_inlock( const bool useDurableOpTime = writeConcern.syncMode == WriteConcernOptions::SyncMode::JOURNAL; if (writeConcern.wMode.empty()) { - return _haveNumNodesReachedOpTime_inlock(opTime, writeConcern.wNumNodes, useDurableOpTime); + return _topCoord->haveNumNodesReachedOpTime( + opTime, writeConcern.wNumNodes, useDurableOpTime); } StringData patternName; @@ -1564,53 +1404,7 @@ bool ReplicationCoordinatorImpl::_doneWaitingForReplication_inlock( if (!tagPattern.isOK()) { return true; } - return _haveTaggedNodesReachedOpTime_inlock(opTime, tagPattern.getValue(), useDurableOpTime); -} - -bool ReplicationCoordinatorImpl::_haveNumNodesReachedOpTime_inlock(const OpTime& targetOpTime, - int numNodes, - bool durablyWritten) { - // Replication progress that is for some reason ahead of us should not allow us to - // satisfy a write concern if we aren't caught up ourselves. - OpTime myOpTime = - durablyWritten ? _getMyLastDurableOpTime_inlock() : _getMyLastAppliedOpTime_inlock(); - if (myOpTime < targetOpTime) { - return false; - } - - for (SlaveInfoVector::iterator it = _slaveInfo.begin(); it != _slaveInfo.end(); ++it) { - const OpTime& slaveTime = durablyWritten ? it->lastDurableOpTime : it->lastAppliedOpTime; - if (slaveTime >= targetOpTime) { - --numNodes; - } - - if (numNodes <= 0) { - return true; - } - } - return false; -} - -bool ReplicationCoordinatorImpl::_haveTaggedNodesReachedOpTime_inlock( - const OpTime& opTime, const ReplSetTagPattern& tagPattern, bool durablyWritten) { - ReplSetTagMatch matcher(tagPattern); - for (SlaveInfoVector::iterator it = _slaveInfo.begin(); it != _slaveInfo.end(); ++it) { - const OpTime& slaveTime = durablyWritten ? it->lastDurableOpTime : it->lastAppliedOpTime; - if (slaveTime >= opTime) { - // This node has reached the desired optime, now we need to check if it is a part - // of the tagPattern. - const MemberConfig* memberConfig = _rsConfig.findMemberByID(it->memberId); - invariant(memberConfig); - for (MemberConfig::TagIterator it = memberConfig->tagsBegin(); - it != memberConfig->tagsEnd(); - ++it) { - if (matcher.update(*it)) { - return true; - } - } - } - } - return false; + return _topCoord->haveTaggedNodesReachedOpTime(opTime, tagPattern.getValue(), useDurableOpTime); } ReplicationCoordinator::StatusAndDuration ReplicationCoordinatorImpl::awaitReplication( @@ -1736,7 +1530,7 @@ Status ReplicationCoordinatorImpl::_awaitReplication_inlock( if (Command::testCommandsEnabled) { // log state of replica set on timeout to help with diagnosis. BSONObjBuilder progress; - _appendSlaveInfoData_inlock(&progress); + _topCoord->fillMemberData(&progress); log() << "Replication for failed WC: " << writeConcern.toBSON() << ", waitInfo: " << waiter << ", opID: " << opCtx->getOpID() << ", progress: " << progress.done(); @@ -1835,7 +1629,7 @@ bool ReplicationCoordinatorImpl::_tryToStepDown_inlock(const Date_t waitUntil, OpTime lastApplied = _getMyLastAppliedOpTime_inlock(); if (forceNow) { - return _topCoord->stepDown(stepDownUntil, forceNow, lastApplied); + return _topCoord->stepDown(stepDownUntil, forceNow); } auto tagStatus = _rsConfig.findCustomWriteMode(ReplSetConfig::kMajorityWriteConcernModeName); @@ -1843,8 +1637,8 @@ bool ReplicationCoordinatorImpl::_tryToStepDown_inlock(const Date_t waitUntil, // Check if a majority of nodes have reached the last applied optime // and there exist an electable node that has my last applied optime. - if (_haveTaggedNodesReachedOpTime_inlock(lastApplied, tagStatus.getValue(), false) && - _topCoord->stepDown(stepDownUntil, forceNow, lastApplied)) { + if (_topCoord->haveTaggedNodesReachedOpTime(lastApplied, tagStatus.getValue(), false) && + _topCoord->stepDown(stepDownUntil, forceNow)) { return true; } @@ -2060,57 +1854,9 @@ Status ReplicationCoordinatorImpl::resyncData(OperationContext* opCtx, bool wait StatusWith<BSONObj> ReplicationCoordinatorImpl::prepareReplSetUpdatePositionCommand( ReplicationCoordinator::ReplSetUpdatePositionCommandStyle commandStyle) const { - BSONObjBuilder cmdBuilder; - { - stdx::lock_guard<stdx::mutex> lock(_mutex); - invariant(_rsConfig.isInitialized()); - // Do not send updates if we have been removed from the config. - if (_selfIndex == -1) { - return Status(ErrorCodes::NodeNotFound, - "This node is not in the current replset configuration."); - } - cmdBuilder.append(UpdatePositionArgs::kCommandFieldName, 1); - // Create an array containing objects each live member connected to us and for ourself. - BSONArrayBuilder arrayBuilder(cmdBuilder.subarrayStart("optimes")); - for (const auto& slaveInfo : _slaveInfo) { - if (slaveInfo.lastAppliedOpTime.isNull()) { - // Don't include info on members we haven't heard from yet. - continue; - } - // Don't include members we think are down. - if (!slaveInfo.self && slaveInfo.down) { - continue; - } - - BSONObjBuilder entry(arrayBuilder.subobjStart()); - switch (commandStyle) { - case ReplSetUpdatePositionCommandStyle::kNewStyle: - slaveInfo.lastDurableOpTime.append(&entry, - UpdatePositionArgs::kDurableOpTimeFieldName); - slaveInfo.lastAppliedOpTime.append(&entry, - UpdatePositionArgs::kAppliedOpTimeFieldName); - break; - case ReplSetUpdatePositionCommandStyle::kOldStyle: - entry.append("_id", slaveInfo.rid); - if (isV1ElectionProtocol()) { - slaveInfo.lastDurableOpTime.append(&entry, "optime"); - } else { - entry.append("optime", slaveInfo.lastDurableOpTime.getTimestamp()); - } - break; - } - entry.append(UpdatePositionArgs::kMemberIdFieldName, slaveInfo.memberId); - entry.append(UpdatePositionArgs::kConfigVersionFieldName, _rsConfig.getConfigVersion()); - } - arrayBuilder.done(); - } - - // Add metadata to command. Old style parsing logic will reject the metadata. - if (commandStyle == ReplSetUpdatePositionCommandStyle::kNewStyle) { - stdx::lock_guard<stdx::mutex> lock(_mutex); - _prepareReplSetMetadata_inlock(OpTime(), &cmdBuilder); - } - return cmdBuilder.obj(); + stdx::lock_guard<stdx::mutex> lock(_mutex); + return _topCoord->prepareReplSetUpdatePositionCommand( + commandStyle, _getCurrentCommittedSnapshotOpTime_inlock()); } Status ReplicationCoordinatorImpl::processReplSetGetStatus( @@ -2130,9 +1876,6 @@ Status ReplicationCoordinatorImpl::processReplSetGetStatus( TopologyCoordinator::ReplSetStatusArgs{ _replExecutor->now(), static_cast<unsigned>(time(0) - serverGlobalParams.started), - _getMyLastAppliedOpTime_inlock(), - _getMyLastDurableOpTime_inlock(), - _lastCommittedOpTime, _getCurrentCommittedSnapshotOpTime_inlock(), initialSyncProgress}, response, @@ -2162,34 +1905,7 @@ void ReplicationCoordinatorImpl::fillIsMasterForReplSet(IsMasterResponse* respon void ReplicationCoordinatorImpl::appendSlaveInfoData(BSONObjBuilder* result) { stdx::lock_guard<stdx::mutex> lock(_mutex); - _appendSlaveInfoData_inlock(result); -} - -void ReplicationCoordinatorImpl::_appendSlaveInfoData_inlock(BSONObjBuilder* result) { - BSONArrayBuilder replicationProgress(result->subarrayStart("replicationProgress")); - { - for (SlaveInfoVector::const_iterator itr = _slaveInfo.begin(); itr != _slaveInfo.end(); - ++itr) { - BSONObjBuilder entry(replicationProgress.subobjStart()); - entry.append("rid", itr->rid); - if (isV1ElectionProtocol()) { - BSONObjBuilder opTime(entry.subobjStart("optime")); - opTime.append("ts", itr->lastDurableOpTime.getTimestamp()); - opTime.append("term", itr->lastDurableOpTime.getTerm()); - opTime.done(); - } else { - entry.append("optime", itr->lastDurableOpTime.getTimestamp()); - } - entry.append("host", itr->hostAndPort.toString()); - if (getReplicationMode() == modeReplSet) { - if (_selfIndex == -1) { - continue; - } - invariant(itr->memberId >= 0); - entry.append("memberId", itr->memberId); - } - } - } + _topCoord->fillMemberData(result); } ReplSetConfig ReplicationCoordinatorImpl::getConfig() const { @@ -2278,8 +1994,7 @@ Status ReplicationCoordinatorImpl::processReplSetSyncFrom(OperationContext* opCt auto doResync = false; { stdx::lock_guard<stdx::mutex> lk(_mutex); - auto opTime = _getMyLastAppliedOpTime_inlock(); - _topCoord->prepareSyncFromResponse(target, opTime, resultObj, &result); + _topCoord->prepareSyncFromResponse(target, resultObj, &result); // If we are in the middle of an initial sync, do a resync. doResync = result.isOK() && _initialSyncer && _initialSyncer->isActive(); } @@ -2325,12 +2040,8 @@ Status ReplicationCoordinatorImpl::processHeartbeat(const ReplSetHeartbeatArgs& stdx::lock_guard<stdx::mutex> lk(_mutex); const Date_t now = _replExecutor->now(); - Status result = _topCoord->prepareHeartbeatResponse(now, - args, - _settings.ourSetName(), - _getMyLastAppliedOpTime_inlock(), - _getMyLastDurableOpTime_inlock(), - response); + Status result = + _topCoord->prepareHeartbeatResponse(now, args, _settings.ourSetName(), response); if ((result.isOK() || result == ErrorCodes::InvalidReplicaSetConfig) && _selfIndex < 0) { // If this node does not belong to the configuration it knows about, send heartbeats // back to any node that sends us a heartbeat, in case one of those remote nodes has @@ -2893,8 +2604,7 @@ Status ReplicationCoordinatorImpl::processReplSetFresh(const ReplSetFreshArgs& a BSONObjBuilder* resultObj) { stdx::lock_guard<stdx::mutex> lk(_mutex); Status result(ErrorCodes::InternalError, "didn't set status in prepareFreshResponse"); - _topCoord->prepareFreshResponse( - args, _replExecutor->now(), _getMyLastAppliedOpTime_inlock(), resultObj, &result); + _topCoord->prepareFreshResponse(args, _replExecutor->now(), resultObj, &result); return result; } @@ -2902,8 +2612,7 @@ Status ReplicationCoordinatorImpl::processReplSetElect(const ReplSetElectArgs& a BSONObjBuilder* responseObj) { stdx::lock_guard<stdx::mutex> lk(_mutex); Status result = Status(ErrorCodes::InternalError, "status not set by callback"); - _topCoord->prepareElectResponse( - args, _replExecutor->now(), _getMyLastAppliedOpTime_inlock(), responseObj, &result); + _topCoord->prepareElectResponse(args, _replExecutor->now(), responseObj, &result); return result; } @@ -2914,9 +2623,7 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig_inlock(const ReplSetConfig& newC _cancelHeartbeats_inlock(); _setConfigState_inlock(kConfigSteady); - // Must get this before changing our config. - OpTime myOptime = _getMyLastAppliedOpTime_inlock(); - _topCoord->updateConfig(newConfig, myIndex, _replExecutor->now(), myOptime); + _topCoord->updateConfig(newConfig, myIndex, _replExecutor->now()); const ReplSetConfig oldConfig = _rsConfig; _rsConfig = newConfig; _protVersion.store(_rsConfig.getProtocolVersion()); @@ -2933,7 +2640,6 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig_inlock(const ReplSetConfig& newC _cancelAndRescheduleElectionTimeout_inlock(); const PostMemberStateUpdateAction action = _updateMemberStateFromTopologyCoordinator_inlock(); - _updateSlaveInfoFromConfig_inlock(); if (_selfIndex >= 0) { // Don't send heartbeats if we're not in the config, if we get re-added one of the // nodes in the set will contact us. @@ -2959,7 +2665,6 @@ ReplicationCoordinatorImpl::_setCurrentRSConfig_inlock(const ReplSetConfig& newC } } - _wakeReadyWaiters_inlock(); return action; } @@ -3020,24 +2725,20 @@ Status ReplicationCoordinatorImpl::processHandshake(OperationContext* opCtx, const HandshakeArgs& handshake) { LOG(2) << "Received handshake " << handshake.toBSON(); - stdx::unique_lock<stdx::mutex> lock(_mutex); + stdx::lock_guard<stdx::mutex> lock(_mutex); if (getReplicationMode() != modeMasterSlave) { return Status(ErrorCodes::IllegalOperation, "The handshake command is only used for master/slave replication"); } - SlaveInfo* slaveInfo = _findSlaveInfoByRID_inlock(handshake.getRid()); - if (slaveInfo) { + auto* memberHeartbeatData = _topCoord->findMemberHeartbeatDataByRid(handshake.getRid()); + if (memberHeartbeatData) { return Status::OK(); // nothing to do } - SlaveInfo newSlaveInfo; - newSlaveInfo.rid = handshake.getRid(); - newSlaveInfo.memberId = -1; - newSlaveInfo.hostAndPort = _externalState->getClientHostAndPort(opCtx); - // Don't call _addSlaveInfo_inlock as that would wake sleepers unnecessarily. - _slaveInfo.push_back(newSlaveInfo); + memberHeartbeatData = _topCoord->addSlaveMemberData(handshake.getRid()); + memberHeartbeatData->setHostAndPort(_externalState->getClientHostAndPort(opCtx)); return Status::OK(); } @@ -3053,26 +2754,10 @@ bool ReplicationCoordinatorImpl::buildsIndexes() { std::vector<HostAndPort> ReplicationCoordinatorImpl::getHostsWrittenTo(const OpTime& op, bool durablyWritten) { - std::vector<HostAndPort> hosts; stdx::lock_guard<stdx::mutex> lk(_mutex); - for (size_t i = 0; i < _slaveInfo.size(); ++i) { - const SlaveInfo& slaveInfo = _slaveInfo[i]; - if (getReplicationMode() == modeMasterSlave && slaveInfo.rid == _getMyRID_inlock()) { - // Master-slave doesn't know the HostAndPort for itself at this point. - continue; - } - - if (durablyWritten) { - if (slaveInfo.lastDurableOpTime < op) { - continue; - } - } else if (slaveInfo.lastAppliedOpTime < op) { - continue; - } - - hosts.push_back(slaveInfo.hostAndPort); - } - return hosts; + /* skip self in master-slave mode because our own HostAndPort is unknown */ + const bool skipSelf = getReplicationMode() == modeMasterSlave; + return _topCoord->getHostsWrittenTo(op, durablyWritten, skipSelf); } std::vector<HostAndPort> ReplicationCoordinatorImpl::getOtherNodesInReplSet() const { @@ -3211,42 +2896,19 @@ bool ReplicationCoordinatorImpl::shouldChangeSyncSource( const rpc::ReplSetMetadata& replMetadata, boost::optional<rpc::OplogQueryMetadata> oqMetadata) { stdx::lock_guard<stdx::mutex> lock(_mutex); - return _topCoord->shouldChangeSyncSource(currentSource, - _getMyLastAppliedOpTime_inlock(), - replMetadata, - oqMetadata, - _replExecutor->now()); + return _topCoord->shouldChangeSyncSource( + currentSource, replMetadata, oqMetadata, _replExecutor->now()); } void ReplicationCoordinatorImpl::_updateLastCommittedOpTime_inlock() { - if (!_getMemberState_inlock().primary() || _topCoord->isStepDownPending()) { - return; + if (_topCoord->updateLastCommittedOpTime()) { + _updateCommitPoint_inlock(); } - - std::vector<OpTime> votingNodesOpTimes; - - // Whether we use the applied or durable OpTime for the commit point is decided here. - const bool useDurableOpTime = getWriteConcernMajorityShouldJournal_inlock(); - - for (const auto& sI : _slaveInfo) { - auto memberConfig = _rsConfig.findMemberByID(sI.memberId); - invariant(memberConfig); - if (memberConfig->isVoter()) { - const auto opTime = useDurableOpTime ? sI.lastDurableOpTime : sI.lastAppliedOpTime; - votingNodesOpTimes.push_back(opTime); - } - } - - invariant(votingNodesOpTimes.size() > 0); - if (votingNodesOpTimes.size() < static_cast<unsigned long>(_rsConfig.getWriteMajority())) { - return; - } - std::sort(votingNodesOpTimes.begin(), votingNodesOpTimes.end()); - - // need the majority to have this OpTime - OpTime committedOpTime = - votingNodesOpTimes[votingNodesOpTimes.size() - _rsConfig.getWriteMajority()]; - _advanceCommitPoint_inlock(committedOpTime); + // Wake up any threads waiting for replication that now have their replication + // check satisfied. We must do this regardless of whether we updated the lastCommittedOpTime, + // as lastCommittedOpTime may be based on durable optimes whereas some waiters may be + // waiting on applied (but not necessarily durable) optimes. + _wakeReadyWaiters_inlock(); } void ReplicationCoordinatorImpl::advanceCommitPoint(const OpTime& committedOpTime) { @@ -3255,28 +2917,17 @@ void ReplicationCoordinatorImpl::advanceCommitPoint(const OpTime& committedOpTim } void ReplicationCoordinatorImpl::_advanceCommitPoint_inlock(const OpTime& committedOpTime) { - if (committedOpTime == _lastCommittedOpTime) { - return; // Hasn't changed, so ignore it. - } else if (committedOpTime < _lastCommittedOpTime) { - LOG(1) << "Ignoring older committed snapshot optime: " << committedOpTime - << ", currentCommittedOpTime: " << _lastCommittedOpTime; - return; // This may have come from an out-of-order heartbeat. Ignore it. - } - - // This check is performed to ensure primaries do not commit an OpTime from a previous term. - if (_getMemberState_inlock().primary() && committedOpTime < _firstOpTimeOfMyTerm) { - LOG(1) << "Ignoring older committed snapshot from before I became primary, optime: " - << committedOpTime << ", firstOpTimeOfMyTerm: " << _firstOpTimeOfMyTerm; - return; - } + if (_topCoord->advanceLastCommittedOpTime(committedOpTime)) { + if (_getMemberState_inlock().arbiter()) { + _setMyLastAppliedOpTime_inlock(committedOpTime, false); + } - if (_getMemberState_inlock().arbiter()) { - _setMyLastAppliedOpTime_inlock(committedOpTime, false); + _updateCommitPoint_inlock(); } +} - LOG(2) << "Updating _lastCommittedOpTime to " << committedOpTime; - _lastCommittedOpTime = committedOpTime; - +void ReplicationCoordinatorImpl::_updateCommitPoint_inlock() { + auto committedOpTime = _topCoord->getLastCommittedOpTime(); _externalState->notifyOplogMetadataWaiters(); auto maxSnapshotForOpTime = SnapshotInfo{committedOpTime, SnapshotName::max()}; @@ -3302,13 +2953,9 @@ void ReplicationCoordinatorImpl::_advanceCommitPoint_inlock(const OpTime& commit } } -void ReplicationCoordinatorImpl::_setFirstOpTimeOfMyTerm_inlock(const OpTime& newOpTime) { - _firstOpTimeOfMyTerm = newOpTime; -} - OpTime ReplicationCoordinatorImpl::getLastCommittedOpTime() const { stdx::unique_lock<stdx::mutex> lk(_mutex); - return _lastCommittedOpTime; + return _topCoord->getLastCommittedOpTime(); } Status ReplicationCoordinatorImpl::processReplSetRequestVotes( @@ -3325,7 +2972,7 @@ Status ReplicationCoordinatorImpl::processReplSetRequestVotes( { stdx::lock_guard<stdx::mutex> lk(_mutex); - _topCoord->processReplSetRequestVotes(args, response, _getMyLastAppliedOpTime_inlock()); + _topCoord->processReplSetRequestVotes(args, response); } if (!args.isADryRun() && response->getVoteGranted()) { @@ -3374,16 +3021,13 @@ void ReplicationCoordinatorImpl::_prepareReplSetMetadata_inlock(const OpTime& la BSONObjBuilder* builder) const { OpTime lastVisibleOpTime = std::max(lastOpTimeFromClient, _getCurrentCommittedSnapshotOpTime_inlock()); - auto metadata = _topCoord->prepareReplSetMetadata(lastVisibleOpTime, _lastCommittedOpTime); + auto metadata = _topCoord->prepareReplSetMetadata(lastVisibleOpTime); metadata.writeToMetadata(builder); } void ReplicationCoordinatorImpl::_prepareOplogQueryMetadata_inlock(int rbid, BSONObjBuilder* builder) const { - OpTime lastAppliedOpTime = _getMyLastAppliedOpTime_inlock(); - auto metadata = - _topCoord->prepareOplogQueryMetadata(_lastCommittedOpTime, lastAppliedOpTime, rbid); - metadata.writeToMetadata(builder); + _topCoord->prepareOplogQueryMetadata(rbid).writeToMetadata(builder); } bool ReplicationCoordinatorImpl::isV1ElectionProtocol() const { @@ -3413,12 +3057,7 @@ Status ReplicationCoordinatorImpl::processHeartbeatV1(const ReplSetHeartbeatArgs auto senderHost(args.getSenderHost()); const Date_t now = _replExecutor->now(); - result = _topCoord->prepareHeartbeatResponseV1(now, - args, - _settings.ourSetName(), - _getMyLastAppliedOpTime_inlock(), - _getMyLastDurableOpTime_inlock(), - response); + result = _topCoord->prepareHeartbeatResponseV1(now, args, _settings.ourSetName(), response); if ((result.isOK() || result == ErrorCodes::InvalidReplicaSetConfig) && _selfIndex < 0) { // If this node does not belong to the configuration it knows about, send heartbeats @@ -3438,12 +3077,12 @@ Status ReplicationCoordinatorImpl::processHeartbeatV1(const ReplSetHeartbeatArgs } } else if (result.isOK()) { // Update liveness for sending node. - auto slaveInfo = _findSlaveInfoByMemberID_inlock(args.getSenderId()); - if (!slaveInfo) { + auto* memberHeartbeatData = + _topCoord->findMemberHeartbeatDataByMemberId(args.getSenderId()); + if (!memberHeartbeatData) { return result; } - slaveInfo->lastUpdate = _replExecutor->now(); - slaveInfo->down = false; + memberHeartbeatData->updateLiveness(_replExecutor->now()); } return result; } @@ -3574,7 +3213,7 @@ void ReplicationCoordinatorImpl::createSnapshot(OperationContext* opCtx, _externalState->createSnapshot(opCtx, name); auto snapshotInfo = SnapshotInfo{timeOfSnapshot, name}; - if (timeOfSnapshot <= _lastCommittedOpTime) { + if (timeOfSnapshot <= _topCoord->getLastCommittedOpTime()) { // This snapshot is ready to be marked as committed. invariant(_uncommittedSnapshots.empty()); _updateCommittedSnapshot_inlock(snapshotInfo); @@ -3596,7 +3235,7 @@ void ReplicationCoordinatorImpl::createSnapshot(OperationContext* opCtx, void ReplicationCoordinatorImpl::_updateCommittedSnapshot_inlock( SnapshotInfo newCommittedSnapshot) { invariant(!newCommittedSnapshot.opTime.isNull()); - invariant(newCommittedSnapshot.opTime <= _lastCommittedOpTime); + invariant(newCommittedSnapshot.opTime <= _topCoord->getLastCommittedOpTime()); if (_currentCommittedSnapshot) { invariant(newCommittedSnapshot.opTime >= _currentCommittedSnapshot->opTime); invariant(newCommittedSnapshot.name > _currentCommittedSnapshot->name); diff --git a/src/mongo/db/repl/replication_coordinator_impl.h b/src/mongo/db/repl/replication_coordinator_impl.h index a2a049cd75e..874d7ad901d 100644 --- a/src/mongo/db/repl/replication_coordinator_impl.h +++ b/src/mongo/db/repl/replication_coordinator_impl.h @@ -535,28 +535,6 @@ private: std::vector<WaiterType> _list; }; - // Struct that holds information about nodes in this replication group, mainly used for - // tracking replication progress for write concern satisfaction. - struct SlaveInfo { - // Our last known OpTime that this slave has applied and journaled to. - OpTime lastDurableOpTime; - // Our last known OpTime that this slave has applied, whether journaled or unjournaled. - OpTime lastAppliedOpTime; - HostAndPort hostAndPort; // Client address of the slave. - int memberId = - -1; // Id of the node in the replica set config, or -1 if we're not a replSet. - OID rid; // RID of the node. - bool self = false; // Whether this SlaveInfo stores the information about ourself - Date_t lastUpdate = - Date_t::max(); // The last time we heard from this node; used for liveness detection - bool down = false; // Indicator set when lastUpdate time exceeds the election timeout. - - BSONObj toBSON() const; - std::string toString() const; - }; - - typedef std::vector<SlaveInfo> SlaveInfoVector; - typedef std::vector<executor::TaskExecutor::CallbackHandle> HeartbeatHandles; // The state and logic of primary catchup. @@ -597,50 +575,6 @@ private: std::unique_ptr<CallbackWaiter> _waiter; }; - /** - * Appends a "replicationProgress" section with data for each member in set. - */ - void _appendSlaveInfoData_inlock(BSONObjBuilder* result); - - /** - * Looks up the SlaveInfo in _slaveInfo associated with the given RID and returns a pointer - * to it, or returns NULL if there is no SlaveInfo with the given RID. - */ - SlaveInfo* _findSlaveInfoByRID_inlock(const OID& rid); - - /** - * Looks up the SlaveInfo in _slaveInfo associated with the given member ID and returns a - * pointer to it, or returns NULL if there is no SlaveInfo with the given member ID. - */ - SlaveInfo* _findSlaveInfoByMemberID_inlock(int memberID); - - /** - * Adds the given SlaveInfo to _slaveInfo and wakes up any threads waiting for replication - * that now have their write concern satisfied. Only valid to call in master/slave setups. - */ - void _addSlaveInfo_inlock(const SlaveInfo& slaveInfo); - - /** - * Updates the durableOpTime field on the item in _slaveInfo pointed to by 'slaveInfo' with the - * given OpTime 'opTime' and wakes up any threads waiting for replication that now have their - * write concern satisfied. - */ - void _updateSlaveInfoDurableOpTime_inlock(SlaveInfo* slaveInfo, const OpTime& opTime); - - /** - * Updates the appliedOpTime field on the item in _slaveInfo pointed to by 'slaveInfo' with the - * given OpTime 'opTime' and wakes up any threads waiting for replication that now have their - * write concern satisfied. - */ - void _updateSlaveInfoAppliedOpTime_inlock(SlaveInfo* slaveInfo, const OpTime& opTime); - - /** - * Returns the index into _slaveInfo where data corresponding to ourself is stored. - * For more info on the rules about how we know where our entry is, see the comment for - * _slaveInfo. - */ - size_t _getMyIndexInSlaveInfo_inlock() const; - void _resetMyLastOpTimes_inlock(); /** @@ -664,14 +598,6 @@ private: Status _validateReadConcern(OperationContext* opCtx, const ReadConcernArgs& readConcern); /** - * Helper method that removes entries from _slaveInfo if they correspond to a node - * with a member ID that is not in the current replica set config. Will always leave an - * entry for ourself at the beginning of _slaveInfo, even if we aren't present in the - * config. - */ - void _updateSlaveInfoFromConfig_inlock(); - - /** * Helper to update our saved config, cancel any pending heartbeats, and kick off sending * new heartbeats based on the new config. * @@ -682,12 +608,6 @@ private: int myIndex); /** - * Updates the last committed OpTime to be "committedOpTime" if it is more recent than the - * current last committed OpTime. - */ - void _advanceCommitPoint_inlock(const OpTime& committedOpTime); - - /** * Helper to wake waiters in _replicationWaiterList that are doneWaitingForReplication. */ void _wakeReadyWaiters_inlock(); @@ -719,21 +639,6 @@ private: SnapshotName minSnapshot, const WriteConcernOptions& writeConcern); - /** - * Helper for _doneWaitingForReplication_inlock that takes an integer write concern. - * "durablyWritten" indicates whether the operation has to be durably applied. - */ - bool _haveNumNodesReachedOpTime_inlock(const OpTime& opTime, int numNodes, bool durablyWritten); - - /** - * Helper for _doneWaitingForReplication_inlock that takes a tag pattern representing a - * named write concern mode. - * "durablyWritten" indicates whether the operation has to be durably applied. - */ - bool _haveTaggedNodesReachedOpTime_inlock(const OpTime& opTime, - const ReplSetTagPattern& tagPattern, - bool durablyWritten); - Status _checkIfWriteConcernCanBeSatisfied_inlock(const WriteConcernOptions& writeConcern) const; /** @@ -1048,17 +953,28 @@ private: stdx::unique_lock<stdx::mutex> lock); /** - * Scan the SlaveInfoVector and determine the highest OplogEntry present on a majority of - * servers; set _lastCommittedOpTime to this new entry, if greater than the current entry. + * Updates the last committed OpTime to be "committedOpTime" if it is more recent than the + * current last committed OpTime. */ - void _updateLastCommittedOpTime_inlock(); + void _advanceCommitPoint_inlock(const OpTime& committedOpTime); + + /** + * Helper for advanceCommitPoint and updateLastCommittedOpTime. Notifies external waiters + * waiting on oplog metadata changes (not read or write concerns) of a change in + * lastCommittedOpTime and updates our committed snapshot. + */ + void _updateCommitPoint_inlock(); /** - * This is used to set a floor of "newOpTime" on the OpTimes we will consider committed. - * This prevents entries from before our election from counting as committed in our view, - * until our election (the "newOpTime" op) has been committed. + * Scan the memberHeartbeatData and determine the highest last applied or last + * durable optime present on a majority of servers; set _lastCommittedOpTime to this + * new entry. Wake any threads waiting for replication that now have their + * write concern satisfied. + * + * Whether the last applied or last durable op time is used depends on whether + * the config getWriteConcernMajorityShouldJournal is set. */ - void _setFirstOpTimeOfMyTerm_inlock(const OpTime& newOpTime); + void _updateLastCommittedOpTime_inlock(); /** * Callback that attempts to set the current term in topology coordinator and @@ -1302,16 +1218,6 @@ private: // Election ID of the last election that resulted in this node becoming primary. OID _electionId; // (M) - // Vector containing known information about each member (such as replication - // progress and member ID) in our replica set or each member replicating from - // us in a master-slave deployment. In master/slave, the first entry is - // guaranteed to correspond to ourself. In replica sets where we don't have a - // valid config or are in state REMOVED then the vector will be a single element - // just with info about ourself. In replica sets with a valid config the elements - // will be in the same order as the members in the replica set config, thus - // the entry for ourself will be at _thisMemberConfigIndex. - SlaveInfoVector _slaveInfo; // (M) - // Used to signal threads waiting for changes to _memberState. stdx::condition_variable _memberStateChange; // (M) @@ -1375,13 +1281,6 @@ private: // _canAcceptNonLocalWrites, its value is only meaningful on replica set secondaries. AtomicUInt32 _canServeNonLocalReads; // (S) - // OpTime of the latest committed operation. Matches the concurrency level of _slaveInfo. - OpTime _lastCommittedOpTime; // (M) - - // OpTime representing our transition to PRIMARY and the start of our term. - // _lastCommittedOpTime cannot be set to an earlier OpTime. - OpTime _firstOpTimeOfMyTerm; // (M) - // ReplicationProcess used to hold information related to the replication and application of // operations from the sync source. ReplicationProcess* const _replicationProcess; // (PS) diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp index be7d6ea79d7..322c84246e6 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect.cpp @@ -278,12 +278,11 @@ void ReplicationCoordinatorImpl::_recoverFromElectionTie( stdx::unique_lock<stdx::mutex> lk(_mutex); auto now = _replExecutor->now(); - auto lastOpApplied = _getMyLastAppliedOpTime_inlock(); - const auto status = _topCoord->checkShouldStandForElection(now, lastOpApplied); + const auto status = _topCoord->checkShouldStandForElection(now); if (!status.isOK()) { LOG(2) << "ReplicationCoordinatorImpl::_recoverFromElectionTie -- " << status.reason(); } else { - fassertStatusOK(28817, _topCoord->becomeCandidateIfElectable(now, lastOpApplied, false)); + fassertStatusOK(28817, _topCoord->becomeCandidateIfElectable(now, false)); _startElectSelf_inlock(); } } diff --git a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp index cf543852022..4d42efe75eb 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_elect_v1.cpp @@ -275,16 +275,10 @@ void ReplicationCoordinatorImpl::_onVoteRequestComplete(long long originalTerm) // Mark all nodes that responded to our vote request as up to avoid immediately // relinquishing primary. Date_t now = _replExecutor->now(); - const unordered_set<HostAndPort> liveNodes = _voteRequester->getResponders(); - for (auto& nodeInfo : _slaveInfo) { - if (liveNodes.count(nodeInfo.hostAndPort)) { - nodeInfo.down = false; - nodeInfo.lastUpdate = now; - } - } + _topCoord->resetMemberTimeouts(now, _voteRequester->getResponders()); // Prevent last committed optime from updating until we finish draining. - _setFirstOpTimeOfMyTerm_inlock( + _topCoord->setFirstOpTimeOfMyTerm( OpTime(Timestamp(std::numeric_limits<int>::max(), 0), std::numeric_limits<int>::max())); _voteRequester.reset(); diff --git a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp index 258b592d405..d7a1ab141b6 100644 --- a/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp +++ b/src/mongo/db/repl/replication_coordinator_impl_heartbeat.cpp @@ -39,6 +39,7 @@ #include "mongo/db/repl/elect_cmd_runner.h" #include "mongo/db/repl/freshness_checker.h" #include "mongo/db/repl/heartbeat_response_action.h" +#include "mongo/db/repl/member_heartbeat_data.h" #include "mongo/db/repl/repl_set_config_checks.h" #include "mongo/db/repl/repl_set_heartbeat_args.h" #include "mongo/db/repl/repl_set_heartbeat_args_v1.h" @@ -191,7 +192,6 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( } } const Date_t now = _replExecutor->now(); - const OpTime lastApplied = _getMyLastAppliedOpTime_inlock(); Milliseconds networkTime(0); StatusWith<ReplSetHeartbeatResponse> hbStatusResponse(hbResponse); @@ -213,21 +213,14 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( hbStatusResponse = StatusWith<ReplSetHeartbeatResponse>(responseStatus); } - HeartbeatResponseAction action = _topCoord->processHeartbeatResponse( - now, networkTime, target, hbStatusResponse, lastApplied); + HeartbeatResponseAction action = + _topCoord->processHeartbeatResponse(now, networkTime, target, hbStatusResponse); if (action.getAction() == HeartbeatResponseAction::NoAction && hbStatusResponse.isOK() && - targetIndex >= 0 && hbStatusResponse.getValue().hasState() && - hbStatusResponse.getValue().getState() != MemberState::RS_PRIMARY) { - ReplSetHeartbeatResponse hbResp = hbStatusResponse.getValue(); - if (hbResp.hasAppliedOpTime()) { - if (hbResp.getConfigVersion() == _rsConfig.getConfigVersion()) { - _updateOpTimesFromHeartbeat_inlock( - targetIndex, - hbResp.hasDurableOpTime() ? hbResp.getDurableOpTime() : OpTime(), - hbResp.getAppliedOpTime()); - } - } + hbStatusResponse.getValue().hasState() && + hbStatusResponse.getValue().getState() != MemberState::RS_PRIMARY && + action.getAdvancedOpTime()) { + _updateLastCommittedOpTime_inlock(); } // Wake the stepdown waiter when our updated OpTime allows it to finish stepping down. @@ -244,21 +237,6 @@ void ReplicationCoordinatorImpl::_handleHeartbeatResponse( _handleHeartbeatResponseAction_inlock(action, hbStatusResponse, std::move(lk)); } -void ReplicationCoordinatorImpl::_updateOpTimesFromHeartbeat_inlock(int targetIndex, - const OpTime& durableOpTime, - const OpTime& appliedOpTime) { - invariant(_selfIndex >= 0); - invariant(targetIndex >= 0); - - SlaveInfo& slaveInfo = _slaveInfo[targetIndex]; - if (appliedOpTime > slaveInfo.lastAppliedOpTime) { - _updateSlaveInfoAppliedOpTime_inlock(&slaveInfo, appliedOpTime); - } - if (durableOpTime > slaveInfo.lastDurableOpTime) { - _updateSlaveInfoDurableOpTime_inlock(&slaveInfo, durableOpTime); - } -} - stdx::unique_lock<stdx::mutex> ReplicationCoordinatorImpl::_handleHeartbeatResponseAction_inlock( const HeartbeatResponseAction& action, const StatusWith<ReplSetHeartbeatResponse>& responseStatus, @@ -691,10 +669,7 @@ void ReplicationCoordinatorImpl::_startHeartbeats_inlock() { _topCoord->restartHeartbeats(); if (isV1ElectionProtocol()) { - for (auto&& slaveInfo : _slaveInfo) { - slaveInfo.lastUpdate = _replExecutor->now(); - slaveInfo.down = false; - } + _topCoord->resetAllMemberTimeouts(_replExecutor->now()); _scheduleNextLivenessUpdate_inlock(); } } @@ -714,37 +689,12 @@ void ReplicationCoordinatorImpl::_handleLivenessTimeout( } // Scan liveness table for problems and mark nodes as down by calling into topocoord. - auto now(_replExecutor->now()); - for (auto&& slaveInfo : _slaveInfo) { - if (slaveInfo.self) { - continue; - } - if (slaveInfo.down) { - continue; - } + HeartbeatResponseAction action = _topCoord->checkMemberTimeouts(_replExecutor->now()); + // Don't mind potential asynchronous stepdown as this is the last step of + // liveness check. + lk = _handleHeartbeatResponseAction_inlock( + action, makeStatusWith<ReplSetHeartbeatResponse>(), std::move(lk)); - if (now - slaveInfo.lastUpdate >= _rsConfig.getElectionTimeoutPeriod()) { - int memberIndex = _rsConfig.findMemberIndexByConfigId(slaveInfo.memberId); - if (memberIndex == -1) { - continue; - } - - slaveInfo.down = true; - - if (_memberState.primary()) { - // Only adjust hbdata if we are primary, since only the primary has a full view - // of the entire cluster. - // Secondaries might not see other secondaries in the cluster if they are not - // downstream. - HeartbeatResponseAction action = - _topCoord->setMemberAsDown(now, memberIndex, _getMyLastDurableOpTime_inlock()); - // Don't mind potential asynchronous stepdown as this is the last step of - // liveness check. - lk = _handleHeartbeatResponseAction_inlock( - action, makeStatusWith<ReplSetHeartbeatResponse>(), std::move(lk)); - } - } - } _scheduleNextLivenessUpdate_inlock(); } @@ -754,23 +704,10 @@ void ReplicationCoordinatorImpl::_scheduleNextLivenessUpdate_inlock() { } // Scan liveness table for earliest date; schedule a run at (that date plus election // timeout). - Date_t earliestDate = Date_t::max(); - int earliestMemberId = -1; - for (auto&& slaveInfo : _slaveInfo) { - if (slaveInfo.self) { - continue; - } - if (slaveInfo.down) { - // Already down. - continue; - } - LOG(3) << "slaveinfo lastupdate is: " << slaveInfo.lastUpdate; - if (earliestDate > slaveInfo.lastUpdate) { - earliestDate = slaveInfo.lastUpdate; - earliestMemberId = slaveInfo.memberId; - } - } - LOG(3) << "earliest member " << earliestMemberId << " date: " << earliestDate; + Date_t earliestDate; + int earliestMemberId; + std::tie(earliestMemberId, earliestDate) = _topCoord->getStalestLiveMember(); + if (earliestMemberId == -1 || earliestDate == Date_t::max()) { _earliestMemberId = -1; // Nobody here but us. @@ -874,10 +811,8 @@ void ReplicationCoordinatorImpl::_startElectSelfIfEligibleV1(StartElectionV1Reas } } - const auto status = - _topCoord->becomeCandidateIfElectable(_replExecutor->now(), - _getMyLastAppliedOpTime_inlock(), - reason == StartElectionV1Reason::kPriorityTakeover); + const auto status = _topCoord->becomeCandidateIfElectable( + _replExecutor->now(), reason == StartElectionV1Reason::kPriorityTakeover); if (!status.isOK()) { switch (reason) { case StartElectionV1Reason::kElectionTimeout: diff --git a/src/mongo/db/repl/topology_coordinator.h b/src/mongo/db/repl/topology_coordinator.h index d45117847cb..8c915cbedea 100644 --- a/src/mongo/db/repl/topology_coordinator.h +++ b/src/mongo/db/repl/topology_coordinator.h @@ -45,6 +45,7 @@ class Timestamp; namespace repl { class HeartbeatResponseAction; +class MemberHeartbeatData; class OpTime; class ReplSetHeartbeatArgs; class ReplSetConfig; @@ -169,7 +170,6 @@ public: * TODO (SERVER-27668): Make OplogQueryMetadata non-optional in mongodb 3.8. */ virtual bool shouldChangeSyncSource(const HostAndPort& currentSource, - const OpTime& myLastOpTime, const rpc::ReplSetMetadata& replMetadata, boost::optional<rpc::OplogQueryMetadata> oqMetadata, Date_t now) const = 0; @@ -199,6 +199,34 @@ public: virtual void setFollowerMode(MemberState::MS newMode) = 0; /** + * Scan the memberHeartbeatData and determine the highest last applied or last + * durable optime present on a majority of servers; set _lastCommittedOpTime to this + * new entry. + * Whether the last applied or last durable op time is used depends on whether + * the config getWriteConcernMajorityShouldJournal is set. + * Returns true if the _lastCommittedOpTime was changed. + */ + virtual bool updateLastCommittedOpTime() = 0; + + /** + * Updates _lastCommittedOpTime to be "committedOpTime" if it is more recent than the + * current last committed OpTime. Returns true if _lastCommittedOpTime is changed. + */ + virtual bool advanceLastCommittedOpTime(const OpTime& committedOpTime) = 0; + + /** + * Returns the OpTime of the latest majority-committed op known to this server. + */ + virtual OpTime getLastCommittedOpTime() const = 0; + + /** + * This is used to set a floor of "newOpTime" on the OpTimes we will consider committed. + * This prevents entries from before our election from counting as committed in our view, + * until our election (the "newOpTime" op) has been committed. + */ + virtual void setFirstOpTimeOfMyTerm(const OpTime& newOpTime) = 0; + + /** * Adjusts the maintenance mode count by "inc". * * It is an error to call this method if getRole() does not return Role::follower. @@ -214,21 +242,18 @@ public: // produces a reply to a replSetSyncFrom command virtual void prepareSyncFromResponse(const HostAndPort& target, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) = 0; // produce a reply to a replSetFresh command virtual void prepareFreshResponse(const ReplicationCoordinator::ReplSetFreshArgs& args, Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) = 0; // produce a reply to a received electCmd virtual void prepareElectResponse(const ReplicationCoordinator::ReplSetElectArgs& args, Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) = 0; @@ -236,24 +261,17 @@ public: virtual Status prepareHeartbeatResponse(Date_t now, const ReplSetHeartbeatArgs& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response) = 0; // produce a reply to a V1 heartbeat virtual Status prepareHeartbeatResponseV1(Date_t now, const ReplSetHeartbeatArgsV1& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response) = 0; struct ReplSetStatusArgs { Date_t now; unsigned selfUptime; - const OpTime& lastOpApplied; - const OpTime& lastOpDurable; - const OpTime& lastCommittedOpTime; const OpTime& readConcernMajorityOpTime; const BSONObj& initialSyncStatus; }; @@ -263,10 +281,18 @@ public: BSONObjBuilder* response, Status* result) = 0; + // Produce a replSetUpdatePosition command to be sent to the node's sync source. + virtual StatusWith<BSONObj> prepareReplSetUpdatePositionCommand( + ReplicationCoordinator::ReplSetUpdatePositionCommandStyle commandStyle, + OpTime currentCommittedSnapshotOpTime) const = 0; + // produce a reply to an ismaster request. It is only valid to call this if we are a // replset. virtual void fillIsMasterForReplSet(IsMasterResponse* response) = 0; + // Produce member data for the serverStatus command and diagnostic logging. + virtual void fillMemberData(BSONObjBuilder* result) = 0; + enum class PrepareFreezeResponseResult { kNoAction, kElectSelf }; /** @@ -294,10 +320,7 @@ public: * newConfig.isInitialized() should be true, though implementations may accept * configurations where this is not true, for testing purposes. */ - virtual void updateConfig(const ReplSetConfig& newConfig, - int selfIndex, - Date_t now, - const OpTime& lastOpApplied) = 0; + virtual void updateConfig(const ReplSetConfig& newConfig, int selfIndex, Date_t now) = 0; /** * Prepares a heartbeat request appropriate for sending to "target", assuming the @@ -346,16 +369,99 @@ public: Date_t now, Milliseconds networkRoundTripTime, const HostAndPort& target, - const StatusWith<ReplSetHeartbeatResponse>& hbResponse, - const OpTime& myLastOpApplied) = 0; + const StatusWith<ReplSetHeartbeatResponse>& hbResponse) = 0; + + /** + * Returns whether or not at least 'numNodes' have reached the given opTime. + * "durablyWritten" indicates whether the operation has to be durably applied. + */ + virtual bool haveNumNodesReachedOpTime(const OpTime& opTime, + int numNodes, + bool durablyWritten) = 0; + + /** + * Returns whether or not at least one node matching the tagPattern has reached + * the given opTime. + * "durablyWritten" indicates whether the operation has to be durably applied. + */ + virtual bool haveTaggedNodesReachedOpTime(const OpTime& opTime, + const ReplSetTagPattern& tagPattern, + bool durablyWritten) = 0; + + /** + * Returns a vector of members that have applied the operation with OpTime 'op'. + * "durablyWritten" indicates whether the operation has to be durably applied. + * "skipSelf" means to exclude this node whether or not the op has been applied. + */ + virtual std::vector<HostAndPort> getHostsWrittenTo(const OpTime& op, + bool durablyWritten, + bool skipSelf) = 0; /** * Marks a member has down from our persepctive and returns a HeartbeatResponseAction, which * will be StepDownSelf if we can no longer see a majority of the nodes. */ - virtual HeartbeatResponseAction setMemberAsDown(Date_t now, - const int memberIndex, - const OpTime& myLastOpApplied) = 0; + virtual HeartbeatResponseAction setMemberAsDown(Date_t now, const int memberIndex) = 0; + + /** + * Goes through the memberHeartbeatData and determines which member that is currently live + * has the stalest (earliest) last update time. Returns (-1, Date_t::max()) if there are + * no other members. + */ + virtual std::pair<int, Date_t> getStalestLiveMember() const = 0; + + /** + * Go through the memberHeartbeatData, and mark nodes which haven't been updated + * recently (within an election timeout) as "down". Returns a HeartbeatResponseAction, which + * will be StepDownSelf if we can no longer see a majority of the nodes, otherwise NoAction. + */ + virtual HeartbeatResponseAction checkMemberTimeouts(Date_t now) = 0; + + /** + * Set all nodes in memberHeartbeatData to not stale with a lastUpdate of "now". + */ + virtual void resetAllMemberTimeouts(Date_t now) = 0; + + /** + * Set all nodes in memberHeartbeatData that are present in member_set + * to not stale with a lastUpdate of "now". + */ + virtual void resetMemberTimeouts(Date_t now, + const stdx::unordered_set<HostAndPort>& member_set) = 0; + + /* + * Returns the last optime that this node has applied, whether or not it has been journaled. + */ + virtual OpTime getMyLastAppliedOpTime() const = 0; + + /* + * Returns the last optime that this node has applied and journaled. + */ + virtual OpTime getMyLastDurableOpTime() const = 0; + + /* + * Returns information we have on the state of this node. + */ + virtual MemberHeartbeatData* getMyMemberHeartbeatData() = 0; + + /* + * Returns information we have on the state of the node identified by memberId. Returns + * nullptr if memberId is not found in the configuration. + */ + virtual MemberHeartbeatData* findMemberHeartbeatDataByMemberId(const int memberId) = 0; + + /* + * Returns information we have on the state of the node identified by rid. Returns + * nullptr if rid is not found in the heartbeat data. This method is used only for + * master/slave replication. + */ + virtual MemberHeartbeatData* findMemberHeartbeatDataByRid(const OID rid) = 0; + + /* + * Adds and returns a memberHeartbeatData entry for the given RID. + * Used only in master/slave mode. + */ + virtual MemberHeartbeatData* addSlaveMemberData(const OID rid) = 0; /** * If getRole() == Role::candidate and this node has not voted too recently, updates the @@ -412,7 +518,7 @@ public: * * NOTE: It is illegal to call this method if the node is not a primary. */ - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied) = 0; + virtual bool stepDown(Date_t until, bool force) = 0; /** * Sometimes a request to step down comes in (like via a heartbeat), but we don't have the @@ -433,7 +539,7 @@ public: * Considers whether or not this node should stand for election, and returns true * if the node has transitioned to candidate role as a result of the call. */ - virtual Status checkShouldStandForElection(Date_t now, const OpTime& lastOpApplied) const = 0; + virtual Status checkShouldStandForElection(Date_t now) const = 0; /** * Set the outgoing heartbeat message from self @@ -444,16 +550,13 @@ public: * Prepares a ReplSetMetadata object describing the current term, primary, and lastOp * information. */ - virtual rpc::ReplSetMetadata prepareReplSetMetadata( - const OpTime& lastVisibleOpTime, const OpTime& lastCommittedOpTime) const = 0; + virtual rpc::ReplSetMetadata prepareReplSetMetadata(const OpTime& lastVisibleOpTime) const = 0; /** * Prepares an OplogQueryMetadata object describing the current sync source, rbid, primary, * lastOpApplied, and lastOpCommitted. */ - virtual rpc::OplogQueryMetadata prepareOplogQueryMetadata(const OpTime& lastCommittedOpTime, - const OpTime& lastAppliedOpTime, - int rbid) const = 0; + virtual rpc::OplogQueryMetadata prepareOplogQueryMetadata(int rbid) const = 0; /** * Writes into 'output' all the information needed to generate a summary of the current @@ -465,8 +568,7 @@ public: * Prepares a ReplSetRequestVotesResponse. */ virtual void processReplSetRequestVotes(const ReplSetRequestVotesArgs& args, - ReplSetRequestVotesResponse* response, - const OpTime& lastAppliedOpTime) = 0; + ReplSetRequestVotesResponse* response) = 0; /** * Loads an initial LastVote document, which was read from local storage. @@ -488,9 +590,7 @@ public: /** * Transitions to the candidate role if the node is electable. */ - virtual Status becomeCandidateIfElectable(const Date_t now, - const OpTime& lastOpApplied, - bool isPriorityTakeover) = 0; + virtual Status becomeCandidateIfElectable(const Date_t now, bool isPriorityTakeover) = 0; /** * Updates the storage engine read committed support in the TopologyCoordinator options after diff --git a/src/mongo/db/repl/topology_coordinator_impl.cpp b/src/mongo/db/repl/topology_coordinator_impl.cpp index a3bcbfaa8c0..e8785e531fb 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl.cpp @@ -46,6 +46,7 @@ #include "mongo/db/repl/repl_set_html_summary.h" #include "mongo/db/repl/repl_set_request_votes_args.h" #include "mongo/db/repl/rslog.h" +#include "mongo/db/repl/update_position_args.h" #include "mongo/db/server_parameters.h" #include "mongo/rpc/metadata/oplog_query_metadata.h" #include "mongo/rpc/metadata/repl_set_metadata.h" @@ -57,9 +58,7 @@ namespace mongo { namespace repl { - using std::vector; - const Seconds TopologyCoordinatorImpl::VoteLease::leaseTime = Seconds(30); // Controls how caught up in replication a secondary with higher priority than the current primary @@ -144,6 +143,9 @@ TopologyCoordinatorImpl::TopologyCoordinatorImpl(Options options) _maintenanceModeCalls(0), _followerMode(MemberState::RS_STARTUP2) { invariant(getMemberState() == MemberState::RS_STARTUP); + // Need an entry for self in the memberHearbeatData. + _hbdata.emplace_back(); + _hbdata.back().setIsSelf(true); } TopologyCoordinator::Role TopologyCoordinatorImpl::getRole() const { @@ -227,7 +229,7 @@ HostAndPort TopologyCoordinatorImpl::chooseNewSyncSource(Date_t now, // Find primary's oplog time. Reject sync candidates that are more than // _options.maxSyncSourceLagSecs seconds behind. if (_currentPrimaryIndex != -1) { - OpTime primaryOpTime = _hbdata.at(_currentPrimaryIndex).getAppliedOpTime(); + OpTime primaryOpTime = _hbdata.at(_currentPrimaryIndex).getHeartbeatAppliedOpTime(); // Check if primaryOpTime is still close to 0 because we haven't received // our first heartbeat from a new primary yet. @@ -289,10 +291,10 @@ HostAndPort TopologyCoordinatorImpl::chooseNewSyncSource(Date_t now, continue; } // Candidates cannot be excessively behind. - if (it->getAppliedOpTime() < oldestSyncOpTime) { + if (it->getHeartbeatAppliedOpTime() < oldestSyncOpTime) { LOG(2) << "Cannot select sync source because it is too far behind." << "Latest optime of sync candidate " << itMemberConfig.getHostAndPort() - << ": " << it->getAppliedOpTime() + << ": " << it->getHeartbeatAppliedOpTime() << ", oldest acceptable optime: " << oldestSyncOpTime; continue; } @@ -312,12 +314,12 @@ HostAndPort TopologyCoordinatorImpl::chooseNewSyncSource(Date_t now, } } // only consider candidates that are ahead of where we are - if (it->getAppliedOpTime() <= lastOpTimeFetched) { + if (it->getHeartbeatAppliedOpTime() <= lastOpTimeFetched) { LOG(1) << "Cannot select sync source equal to or behind our last fetched optime. " << "My last fetched oplog optime: " << lastOpTimeFetched.toBSON() << ", latest oplog optime of sync candidate " << itMemberConfig.getHostAndPort() << ": " - << it->getAppliedOpTime().toBSON(); + << it->getHeartbeatAppliedOpTime().toBSON(); continue; } // Candidate cannot be more latent than anything we've already considered. @@ -392,7 +394,6 @@ void TopologyCoordinatorImpl::clearSyncSourceBlacklist() { } void TopologyCoordinatorImpl::prepareSyncFromResponse(const HostAndPort& target, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) { response->append("syncFromRequested", target.toString()); @@ -465,9 +466,10 @@ void TopologyCoordinatorImpl::prepareSyncFromResponse(const HostAndPort& target, str::stream() << "I cannot reach the requested member: " << target.toString()); return; } - if (hbdata.getAppliedOpTime().getSecs() + 10 < lastOpApplied.getSecs()) { + const OpTime lastOpApplied = getMyLastAppliedOpTime(); + if (hbdata.getHeartbeatAppliedOpTime().getSecs() + 10 < lastOpApplied.getSecs()) { warning() << "attempting to sync from " << target << ", but its latest opTime is " - << hbdata.getAppliedOpTime().getSecs() << " and ours is " + << hbdata.getHeartbeatAppliedOpTime().getSecs() << " and ours is " << lastOpApplied.getSecs() << " so this may not work"; response->append("warning", str::stream() << "requested member \"" << target.toString() @@ -487,7 +489,6 @@ void TopologyCoordinatorImpl::prepareSyncFromResponse(const HostAndPort& target, void TopologyCoordinatorImpl::prepareFreshResponse( const ReplicationCoordinator::ReplSetFreshArgs& args, const Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) { if (_rsConfig.getProtocolVersion() != 0) { @@ -521,6 +522,7 @@ void TopologyCoordinatorImpl::prepareFreshResponse( } bool weAreFresher = false; + const OpTime lastOpApplied = getMyLastAppliedOpTime(); if (_rsConfig.getConfigVersion() > args.cfgver) { log() << "replSet member " << args.who << " is not yet aware its cfg version " << args.cfgver << " is stale"; @@ -528,7 +530,7 @@ void TopologyCoordinatorImpl::prepareFreshResponse( weAreFresher = true; } // check not only our own optime, but any other member we can reach - else if (OpTime(args.opTime, _term) < _latestKnownOpTime(lastOpApplied)) { + else if (OpTime(args.opTime, _term) < _latestKnownOpTime()) { weAreFresher = true; } response->appendDate("opTime", @@ -536,7 +538,7 @@ void TopologyCoordinatorImpl::prepareFreshResponse( response->append("fresher", weAreFresher); std::string errmsg; - bool doVeto = _shouldVetoMember(args, now, lastOpApplied, &errmsg); + bool doVeto = _shouldVetoMember(args, now, &errmsg); response->append("veto", doVeto); if (doVeto) { response->append("errmsg", errmsg); @@ -547,7 +549,6 @@ void TopologyCoordinatorImpl::prepareFreshResponse( bool TopologyCoordinatorImpl::_shouldVetoMember( const ReplicationCoordinator::ReplSetFreshArgs& args, const Date_t& now, - const OpTime& lastOpApplied, std::string* errmsg) const { if (_rsConfig.getConfigVersion() < args.cfgver) { // We are stale; do not veto. @@ -557,14 +558,14 @@ bool TopologyCoordinatorImpl::_shouldVetoMember( const unsigned int memberID = args.id; const int hopefulIndex = _getMemberIndex(memberID); invariant(hopefulIndex != _selfIndex); - const int highestPriorityIndex = _getHighestPriorityElectableIndex(now, lastOpApplied); + const int highestPriorityIndex = _getHighestPriorityElectableIndex(now); if (hopefulIndex == -1) { *errmsg = str::stream() << "replSet couldn't find member with id " << memberID; return true; } - - if (_iAmPrimary() && lastOpApplied >= _hbdata.at(hopefulIndex).getAppliedOpTime()) { + const OpTime lastOpApplied = getMyLastAppliedOpTime(); + if (_iAmPrimary() && lastOpApplied >= _hbdata.at(hopefulIndex).getHeartbeatAppliedOpTime()) { // hbinfo is not updated for ourself, so if we are primary we have to check the // primary's last optime separately *errmsg = str::stream() << "I am already primary, " @@ -574,8 +575,8 @@ bool TopologyCoordinatorImpl::_shouldVetoMember( } if (_currentPrimaryIndex != -1 && (hopefulIndex != _currentPrimaryIndex) && - (_hbdata.at(_currentPrimaryIndex).getAppliedOpTime() >= - _hbdata.at(hopefulIndex).getAppliedOpTime())) { + (_hbdata.at(_currentPrimaryIndex).getHeartbeatAppliedOpTime() >= + _hbdata.at(hopefulIndex).getHeartbeatAppliedOpTime())) { // other members might be aware of more up-to-date nodes *errmsg = str::stream() << _rsConfig.getMemberAt(hopefulIndex).getHostAndPort().toString() @@ -598,7 +599,7 @@ bool TopologyCoordinatorImpl::_shouldVetoMember( } } - UnelectableReasonMask reason = _getUnelectableReason(hopefulIndex, lastOpApplied); + UnelectableReasonMask reason = _getUnelectableReason(hopefulIndex); reason &= ~RefusesToStand; if (reason) { *errmsg = str::stream() << "I don't think " @@ -615,7 +616,6 @@ bool TopologyCoordinatorImpl::_shouldVetoMember( void TopologyCoordinatorImpl::prepareElectResponse( const ReplicationCoordinator::ReplSetElectArgs& args, const Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result) { if (_rsConfig.getProtocolVersion() != 0) { @@ -631,7 +631,7 @@ void TopologyCoordinatorImpl::prepareElectResponse( } const long long myver = _rsConfig.getConfigVersion(); - const int highestPriorityIndex = _getHighestPriorityElectableIndex(now, lastOpApplied); + const int highestPriorityIndex = _getHighestPriorityElectableIndex(now); const MemberConfig* primary = _currentPrimaryMember(); const MemberConfig* hopeful = _rsConfig.findMemberByID(args.whoid); @@ -693,8 +693,6 @@ void TopologyCoordinatorImpl::prepareElectResponse( Status TopologyCoordinatorImpl::prepareHeartbeatResponse(Date_t now, const ReplSetHeartbeatArgs& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response) { if (args.getProtocolVersion() != 1) { return Status(ErrorCodes::BadValue, @@ -739,8 +737,11 @@ Status TopologyCoordinatorImpl::prepareHeartbeatResponse(Date_t now, response->setElectionTime(_electionTime); } + const OpTime lastOpApplied = getMyLastAppliedOpTime(); + const OpTime lastOpDurable = getMyLastDurableOpTime(); + // Are we electable - response->setElectable(!_getMyUnelectableReason(now, lastOpApplied, false)); + response->setElectable(!_getMyUnelectableReason(now, false)); // Heartbeat status message response->setHbMsg(_getHbmsg(now)); @@ -788,8 +789,6 @@ Status TopologyCoordinatorImpl::prepareHeartbeatResponse(Date_t now, Status TopologyCoordinatorImpl::prepareHeartbeatResponseV1(Date_t now, const ReplSetHeartbeatArgsV1& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response) { // Verify that replica set names match const std::string rshb = args.getSetName(); @@ -825,6 +824,8 @@ Status TopologyCoordinatorImpl::prepareHeartbeatResponseV1(Date_t now, response->setElectionTime(_electionTime); } + const OpTime lastOpApplied = getMyLastAppliedOpTime(); + const OpTime lastOpDurable = getMyLastDurableOpTime(); response->setAppliedOpTime(lastOpApplied); response->setDurableOpTime(lastOpDurable); @@ -951,8 +952,7 @@ HeartbeatResponseAction TopologyCoordinatorImpl::processHeartbeatResponse( Date_t now, Milliseconds networkRoundTripTime, const HostAndPort& target, - const StatusWith<ReplSetHeartbeatResponse>& hbResponse, - const OpTime& myLastOpApplied) { + const StatusWith<ReplSetHeartbeatResponse>& hbResponse) { const MemberState originalState = getMemberState(); PingStats& hbStats = _pings[target]; invariant(hbStats.getLastHeartbeatStartDate() != Date_t()); @@ -1032,6 +1032,14 @@ HeartbeatResponseAction TopologyCoordinatorImpl::processHeartbeatResponse( nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); return nextAction; } + // If we're not in the config, we don't need to respond to heartbeats. + if (_selfIndex == -1) { + LOG(1) << "Could not find ourself in current config so ignoring heartbeat from " << target + << " -- current config: " << _rsConfig.toBSON(); + HeartbeatResponseAction nextAction = HeartbeatResponseAction::makeNoAction(); + nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); + return nextAction; + } const int memberIndex = _rsConfig.findMemberIndexByHostAndPort(target); if (memberIndex == -1) { LOG(1) << "Could not find " << target << " in current config so ignoring --" @@ -1046,6 +1054,7 @@ HeartbeatResponseAction TopologyCoordinatorImpl::processHeartbeatResponse( MemberHeartbeatData& hbData = _hbdata.at(memberIndex); const MemberConfig member = _rsConfig.getMemberAt(memberIndex); + bool advancedOpTime = false; if (!hbResponse.isOK()) { if (isUnauthorized) { LOG(1) << "setAuthIssue: heartbeat response failed due to authentication" @@ -1067,30 +1076,120 @@ HeartbeatResponseAction TopologyCoordinatorImpl::processHeartbeatResponse( ReplSetHeartbeatResponse hbr = std::move(hbResponse.getValue()); LOG(3) << "setUpValues: heartbeat response good for member _id:" << member.getId() << ", msg: " << hbr.getHbMsg(); - hbData.setUpValues(now, member.getHostAndPort(), std::move(hbr)); + advancedOpTime = hbData.setUpValues(now, member.getHostAndPort(), std::move(hbr)); } HeartbeatResponseAction nextAction; if (_rsConfig.getProtocolVersion() == 0) { - nextAction = _updatePrimaryFromHBData(memberIndex, originalState, now, myLastOpApplied); + nextAction = _updatePrimaryFromHBData(memberIndex, originalState, now); } else { - nextAction = _updatePrimaryFromHBDataV1(memberIndex, originalState, now, myLastOpApplied); + nextAction = _updatePrimaryFromHBDataV1(memberIndex, originalState, now); } nextAction.setNextHeartbeatStartDate(nextHeartbeatStartDate); + nextAction.setAdvancedOpTime(advancedOpTime); return nextAction; } +bool TopologyCoordinatorImpl::haveNumNodesReachedOpTime(const OpTime& targetOpTime, + int numNodes, + bool durablyWritten) { + // Replication progress that is for some reason ahead of us should not allow us to + // satisfy a write concern if we aren't caught up ourselves. + OpTime myOpTime = durablyWritten ? getMyLastDurableOpTime() : getMyLastAppliedOpTime(); + if (myOpTime < targetOpTime) { + return false; + } + + for (auto&& memberHeartbeatData : _hbdata) { + const OpTime& memberOpTime = durablyWritten ? memberHeartbeatData.getLastDurableOpTime() + : memberHeartbeatData.getLastAppliedOpTime(); + if (memberOpTime >= targetOpTime) { + --numNodes; + } + + if (numNodes <= 0) { + return true; + } + } + return false; +} + +bool TopologyCoordinatorImpl::haveTaggedNodesReachedOpTime(const OpTime& opTime, + const ReplSetTagPattern& tagPattern, + bool durablyWritten) { + ReplSetTagMatch matcher(tagPattern); + for (auto&& memberHeartbeatData : _hbdata) { + const OpTime& memberOpTime = durablyWritten ? memberHeartbeatData.getLastDurableOpTime() + : memberHeartbeatData.getLastAppliedOpTime(); + if (memberOpTime >= opTime) { + // This node has reached the desired optime, now we need to check if it is a part + // of the tagPattern. + int memberIndex = memberHeartbeatData.getConfigIndex(); + invariant(memberIndex >= 0); + const MemberConfig& memberConfig = _rsConfig.getMemberAt(memberIndex); + for (MemberConfig::TagIterator it = memberConfig.tagsBegin(); + it != memberConfig.tagsEnd(); + ++it) { + if (matcher.update(*it)) { + return true; + } + } + } + } + return false; +} + +HeartbeatResponseAction TopologyCoordinatorImpl::checkMemberTimeouts(Date_t now) { + HeartbeatResponseAction result = HeartbeatResponseAction::makeNoAction(); + for (int memberIndex = 0; memberIndex < static_cast<int>(_hbdata.size()); memberIndex++) { + auto& memberHeartbeatData = _hbdata[memberIndex]; + if (!memberHeartbeatData.isSelf() && !memberHeartbeatData.lastUpdateStale() && + now - memberHeartbeatData.getLastUpdate() >= _rsConfig.getElectionTimeoutPeriod()) { + memberHeartbeatData.markLastUpdateStale(); + if (getMemberState().primary()) { + HeartbeatResponseAction action = setMemberAsDown(now, memberIndex); + if (action.getAction() != HeartbeatResponseAction::NoAction) { + invariant(action.getAction() == HeartbeatResponseAction::StepDownSelf); + result = action; + } + } + } + } + return result; +} + +std::vector<HostAndPort> TopologyCoordinatorImpl::getHostsWrittenTo(const OpTime& op, + bool durablyWritten, + bool skipSelf) { + std::vector<HostAndPort> hosts; + for (const auto& memberHeartbeatData : _hbdata) { + if (skipSelf && memberHeartbeatData.isSelf()) { + continue; + } + + if (durablyWritten) { + if (memberHeartbeatData.getLastDurableOpTime() < op) { + continue; + } + } else if (memberHeartbeatData.getLastAppliedOpTime() < op) { + continue; + } + + hosts.push_back(memberHeartbeatData.getHostAndPort()); + } + return hosts; +} + HeartbeatResponseAction TopologyCoordinatorImpl::setMemberAsDown(Date_t now, - const int memberIndex, - const OpTime& myLastOpApplied) { + const int memberIndex) { invariant(memberIndex != _selfIndex); invariant(memberIndex != -1); invariant(_currentPrimaryIndex == _selfIndex); MemberHeartbeatData& hbData = _hbdata.at(memberIndex); hbData.setDownValues(now, "no response within election timeout period"); - if (CannotSeeMajority & _getMyUnelectableReason(now, myLastOpApplied, false)) { + if (CannotSeeMajority & _getMyUnelectableReason(now, false)) { if (_stepDownPending) { return HeartbeatResponseAction::makeNoAction(); } @@ -1102,11 +1201,79 @@ HeartbeatResponseAction TopologyCoordinatorImpl::setMemberAsDown(Date_t now, return HeartbeatResponseAction::makeNoAction(); } +std::pair<int, Date_t> TopologyCoordinatorImpl::getStalestLiveMember() const { + Date_t earliestDate = Date_t::max(); + int earliestMemberId = -1; + for (const auto& memberHeartbeatData : _hbdata) { + if (memberHeartbeatData.isSelf()) { + continue; + } + if (memberHeartbeatData.lastUpdateStale()) { + // Already stale. + continue; + } + LOG(3) << "memberHeartbeatData lastupdate is: " << memberHeartbeatData.getLastUpdate(); + if (earliestDate > memberHeartbeatData.getLastUpdate()) { + earliestDate = memberHeartbeatData.getLastUpdate(); + earliestMemberId = memberHeartbeatData.getMemberId(); + } + } + LOG(3) << "stalest member " << earliestMemberId << " date: " << earliestDate; + return std::make_pair(earliestMemberId, earliestDate); +} + +void TopologyCoordinatorImpl::resetAllMemberTimeouts(Date_t now) { + for (auto&& memberHeartbeatData : _hbdata) + memberHeartbeatData.updateLiveness(now); +} + +void TopologyCoordinatorImpl::resetMemberTimeouts( + Date_t now, const stdx::unordered_set<HostAndPort>& member_set) { + for (auto&& memberHeartbeatData : _hbdata) { + if (member_set.count(memberHeartbeatData.getHostAndPort())) + memberHeartbeatData.updateLiveness(now); + } +} + +OpTime TopologyCoordinatorImpl::getMyLastAppliedOpTime() const { + return _selfMemberHeartbeatData().getLastAppliedOpTime(); +} + +OpTime TopologyCoordinatorImpl::getMyLastDurableOpTime() const { + return _selfMemberHeartbeatData().getLastDurableOpTime(); +} + +MemberHeartbeatData* TopologyCoordinatorImpl::getMyMemberHeartbeatData() { + return &_hbdata[_selfMemberHeartbeatDataIndex()]; +} + +MemberHeartbeatData* TopologyCoordinatorImpl::findMemberHeartbeatDataByMemberId( + const int memberId) { + const int memberIndex = _getMemberIndex(memberId); + if (memberIndex >= 0) + return &_hbdata[memberIndex]; + return nullptr; +} + +MemberHeartbeatData* TopologyCoordinatorImpl::findMemberHeartbeatDataByRid(const OID rid) { + for (auto& memberHeartbeatData : _hbdata) { + if (memberHeartbeatData.getRid() == rid) + return &memberHeartbeatData; + } + return nullptr; +} + +MemberHeartbeatData* TopologyCoordinatorImpl::addSlaveMemberData(const OID rid) { + invariant(!_hbdata.empty()); // Must always have our own entry first. + invariant(!_rsConfig.isInitialized()); // Used only for master-slave. + _hbdata.emplace_back(); + auto* result = &_hbdata.back(); + result->setRid(rid); + return result; +} + HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBDataV1( - int updatedConfigIndex, - const MemberState& originalState, - Date_t now, - const OpTime& lastOpApplied) { + int updatedConfigIndex, const MemberState& originalState, Date_t now) { // // Updates the local notion of which remote node, if any is primary. // Start the priority takeover process if we are eligible. @@ -1161,10 +1328,7 @@ HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBDataV1( } HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBData( - int updatedConfigIndex, - const MemberState& originalState, - Date_t now, - const OpTime& lastOpApplied) { + int updatedConfigIndex, const MemberState& originalState, Date_t now) { // This method has two interrelated responsibilities, performed in two phases. // // First, it updates the local notion of which remote node, if any is primary. In the @@ -1199,17 +1363,17 @@ HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBData( // have them/me stepdown. if (_currentPrimaryIndex != -1) { // check if we should ask the primary (possibly ourselves) to step down - const int highestPriorityIndex = _getHighestPriorityElectableIndex(now, lastOpApplied); + const int highestPriorityIndex = _getHighestPriorityElectableIndex(now); if (highestPriorityIndex != -1) { const MemberConfig& currentPrimaryMember = _rsConfig.getMemberAt(_currentPrimaryIndex); const MemberConfig& highestPriorityMember = _rsConfig.getMemberAt(highestPriorityIndex); const OpTime highestPriorityMemberOptime = highestPriorityIndex == _selfIndex - ? lastOpApplied - : _hbdata.at(highestPriorityIndex).getAppliedOpTime(); + ? getMyLastAppliedOpTime() + : _hbdata.at(highestPriorityIndex).getHeartbeatAppliedOpTime(); if ((highestPriorityMember.getPriority() > currentPrimaryMember.getPriority()) && - _isOpTimeCloseEnoughToLatestToElect(highestPriorityMemberOptime, lastOpApplied)) { - const OpTime latestOpTime = _latestKnownOpTime(lastOpApplied); + _isOpTimeCloseEnoughToLatestToElect(highestPriorityMemberOptime)) { + const OpTime latestOpTime = _latestKnownOpTime(); if (_iAmPrimary()) { if (_stepDownPending) { @@ -1313,7 +1477,7 @@ HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBData( // If we are primary, check if we can still see majority of the set; // stepdown if we can't. if (_iAmPrimary()) { - if (CannotSeeMajority & _getMyUnelectableReason(now, lastOpApplied, false)) { + if (CannotSeeMajority & _getMyUnelectableReason(now, false)) { if (_stepDownPending) { return HeartbeatResponseAction::makeNoAction(); } @@ -1342,18 +1506,17 @@ HeartbeatResponseAction TopologyCoordinatorImpl::_updatePrimaryFromHBData( } // At this point, there is no primary anywhere. Check to see if we should become a candidate. - const auto status = checkShouldStandForElection(now, lastOpApplied); + const auto status = checkShouldStandForElection(now); if (!status.isOK()) { // NOTE: This log line is checked in unit test(s). LOG(2) << "TopologyCoordinatorImpl::_updatePrimaryFromHBData - " << status.reason(); return HeartbeatResponseAction::makeNoAction(); } - fassertStatusOK(28816, becomeCandidateIfElectable(now, lastOpApplied, false)); + fassertStatusOK(28816, becomeCandidateIfElectable(now, false)); return HeartbeatResponseAction::makeElectAction(); } -Status TopologyCoordinatorImpl::checkShouldStandForElection(Date_t now, - const OpTime& lastOpApplied) const { +Status TopologyCoordinatorImpl::checkShouldStandForElection(Date_t now) const { if (_currentPrimaryIndex != -1) { return {ErrorCodes::NodeNotElectable, "Not standing for election since there is a Primary"}; } @@ -1363,16 +1526,15 @@ Status TopologyCoordinatorImpl::checkShouldStandForElection(Date_t now, return {ErrorCodes::NodeNotElectable, "Not standing for election again; already candidate"}; } - const UnelectableReasonMask unelectableReason = - _getMyUnelectableReason(now, lastOpApplied, false); + const UnelectableReasonMask unelectableReason = _getMyUnelectableReason(now, false); if (NotCloseEnoughToLatestOptime & unelectableReason) { return {ErrorCodes::NodeNotElectable, str::stream() << "Not standing for election because " << _getUnelectableReasonString(unelectableReason) << "; my last optime is " - << lastOpApplied.toString() + << getMyLastAppliedOpTime().toString() << " and the newest is " - << _latestKnownOpTime(lastOpApplied).toString()}; + << _latestKnownOpTime().toString()}; } if (unelectableReason) { return {ErrorCodes::NodeNotElectable, @@ -1427,16 +1589,14 @@ bool TopologyCoordinatorImpl::_canSeeHealthyPrimaryOfEqualOrGreaterPriority( return false; } -bool TopologyCoordinatorImpl::_isOpTimeCloseEnoughToLatestToElect( - const OpTime& otherOpTime, const OpTime& ourLastOpApplied) const { - const OpTime latestKnownOpTime = _latestKnownOpTime(ourLastOpApplied); +bool TopologyCoordinatorImpl::_isOpTimeCloseEnoughToLatestToElect(const OpTime& otherOpTime) const { + const OpTime latestKnownOpTime = _latestKnownOpTime(); // Use addition instead of subtraction to avoid overflow. return otherOpTime.getSecs() + 10 >= (latestKnownOpTime.getSecs()); } -bool TopologyCoordinatorImpl::_amIFreshEnoughForPriorityTakeover( - const OpTime& ourLastOpApplied) const { - const OpTime latestKnownOpTime = _latestKnownOpTime(ourLastOpApplied); +bool TopologyCoordinatorImpl::_amIFreshEnoughForPriorityTakeover() const { + const OpTime latestKnownOpTime = _latestKnownOpTime(); // Rules are: // - If the terms don't match, we don't call for priority takeover. @@ -1449,6 +1609,7 @@ bool TopologyCoordinatorImpl::_amIFreshEnoughForPriorityTakeover( // component of all future oplog entries generated will be the same, until real world time // passes the timestamp component of the last oplog entry. + const OpTime ourLastOpApplied = getMyLastAppliedOpTime(); if (ourLastOpApplied.getTerm() != latestKnownOpTime.getTerm()) { return false; } @@ -1470,15 +1631,15 @@ bool TopologyCoordinatorImpl::_iAmPrimary() const { return false; } -OpTime TopologyCoordinatorImpl::_latestKnownOpTime(const OpTime& ourLastOpApplied) const { - OpTime latest = ourLastOpApplied; - +OpTime TopologyCoordinatorImpl::_latestKnownOpTime() const { + OpTime latest = getMyLastAppliedOpTime(); for (std::vector<MemberHeartbeatData>::const_iterator it = _hbdata.begin(); it != _hbdata.end(); ++it) { - if (indexOfIterator(_hbdata, it) == _selfIndex) { + // Ignore self + // TODO(russotto): Simplify when heartbeat and spanning tree times are combined. + if (it->isSelf()) { continue; } - // Ignore down members if (!it->up()) { continue; @@ -1488,7 +1649,7 @@ OpTime TopologyCoordinatorImpl::_latestKnownOpTime(const OpTime& ourLastOpApplie continue; } - OpTime optime = it->getAppliedOpTime(); + OpTime optime = it->getHeartbeatAppliedOpTime(); if (optime > latest) { latest = optime; @@ -1510,13 +1671,12 @@ bool TopologyCoordinatorImpl::_isMemberHigherPriority(int memberOneIndex, _rsConfig.getMemberAt(memberTwoIndex).getPriority(); } -int TopologyCoordinatorImpl::_getHighestPriorityElectableIndex(Date_t now, - const OpTime& lastOpApplied) const { +int TopologyCoordinatorImpl::_getHighestPriorityElectableIndex(Date_t now) const { int maxIndex = -1; for (int currentIndex = 0; currentIndex < _rsConfig.getNumMembers(); currentIndex++) { UnelectableReasonMask reason = currentIndex == _selfIndex - ? _getMyUnelectableReason(now, lastOpApplied, false) - : _getUnelectableReason(currentIndex, lastOpApplied); + ? _getMyUnelectableReason(now, false) + : _getUnelectableReason(currentIndex); if (None == reason && _isMemberHigherPriority(currentIndex, maxIndex)) { maxIndex = currentIndex; } @@ -1552,7 +1712,7 @@ void TopologyCoordinatorImpl::changeMemberState_forTest(const MemberState& newMe } break; case MemberState::RS_STARTUP: - updateConfig(ReplSetConfig(), -1, Date_t(), OpTime()); + updateConfig(ReplSetConfig(), -1, Date_t()); break; default: severe() << "Cannot switch to state " << newMemberState; @@ -1577,7 +1737,7 @@ void TopologyCoordinatorImpl::_setCurrentPrimaryForTest(int primaryIndex) { ReplSetHeartbeatResponse hbResponse; hbResponse.setState(MemberState::RS_PRIMARY); hbResponse.setElectionTime(Timestamp()); - hbResponse.setAppliedOpTime(_hbdata.at(primaryIndex).getAppliedOpTime()); + hbResponse.setAppliedOpTime(_hbdata.at(primaryIndex).getHeartbeatAppliedOpTime()); hbResponse.setSyncingTo(HostAndPort()); hbResponse.setHbMsg(""); _hbdata.at(primaryIndex) @@ -1603,8 +1763,8 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS vector<BSONObj> membersOut; const MemberState myState = getMemberState(); const Date_t now = rsStatusArgs.now; - const OpTime& lastOpApplied = rsStatusArgs.lastOpApplied; - const OpTime& lastOpDurable = rsStatusArgs.lastOpDurable; + const OpTime lastOpApplied = getMyLastAppliedOpTime(); + const OpTime lastOpDurable = getMyLastDurableOpTime(); const BSONObj& initialSyncStatus = rsStatusArgs.initialSyncStatus; if (_selfIndex == -1) { @@ -1688,16 +1848,19 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS it->getUpSince() != Date_t() ? durationCount<Seconds>(now - it->getUpSince()) : 0)); bb.append("uptime", uptime); if (!itConfig.isArbiter()) { - appendOpTime(&bb, "optime", it->getAppliedOpTime(), _rsConfig.getProtocolVersion()); appendOpTime( - &bb, "optimeDurable", it->getDurableOpTime(), _rsConfig.getProtocolVersion()); + &bb, "optime", it->getHeartbeatAppliedOpTime(), _rsConfig.getProtocolVersion()); + appendOpTime(&bb, + "optimeDurable", + it->getHeartbeatDurableOpTime(), + _rsConfig.getProtocolVersion()); - bb.appendDate( - "optimeDate", - Date_t::fromDurationSinceEpoch(Seconds(it->getAppliedOpTime().getSecs()))); - bb.appendDate( - "optimeDurableDate", - Date_t::fromDurationSinceEpoch(Seconds(it->getDurableOpTime().getSecs()))); + bb.appendDate("optimeDate", + Date_t::fromDurationSinceEpoch( + Seconds(it->getHeartbeatAppliedOpTime().getSecs()))); + bb.appendDate("optimeDurableDate", + Date_t::fromDurationSinceEpoch( + Seconds(it->getHeartbeatDurableOpTime().getSecs()))); } bb.appendDate("lastHeartbeat", it->getLastHeartbeat()); bb.appendDate("lastHeartbeatRecv", it->getLastHeartbeatRecv()); @@ -1749,7 +1912,7 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS // New optimes, to hold them all. BSONObjBuilder optimes; - rsStatusArgs.lastCommittedOpTime.append(&optimes, "lastCommittedOpTime"); + _lastCommittedOpTime.append(&optimes, "lastCommittedOpTime"); if (!rsStatusArgs.readConcernMajorityOpTime.isNull()) { rsStatusArgs.readConcernMajorityOpTime.append(&optimes, "readConcernMajorityOpTime"); } @@ -1766,6 +1929,84 @@ void TopologyCoordinatorImpl::prepareStatusResponse(const ReplSetStatusArgs& rsS *result = Status::OK(); } +StatusWith<BSONObj> TopologyCoordinatorImpl::prepareReplSetUpdatePositionCommand( + ReplicationCoordinator::ReplSetUpdatePositionCommandStyle commandStyle, + OpTime currentCommittedSnapshotOpTime) const { + BSONObjBuilder cmdBuilder; + invariant(_rsConfig.isInitialized()); + // Do not send updates if we have been removed from the config. + if (_selfIndex == -1) { + return Status(ErrorCodes::NodeNotFound, + "This node is not in the current replset configuration."); + } + cmdBuilder.append(UpdatePositionArgs::kCommandFieldName, 1); + // Create an array containing objects each live member connected to us and for ourself. + BSONArrayBuilder arrayBuilder(cmdBuilder.subarrayStart("optimes")); + for (const auto& memberHeartbeatData : _hbdata) { + if (memberHeartbeatData.getLastAppliedOpTime().isNull()) { + // Don't include info on members we haven't heard from yet. + continue; + } + // Don't include members we think are down. + if (!memberHeartbeatData.isSelf() && memberHeartbeatData.lastUpdateStale()) { + continue; + } + + BSONObjBuilder entry(arrayBuilder.subobjStart()); + switch (commandStyle) { + case ReplicationCoordinator::ReplSetUpdatePositionCommandStyle::kNewStyle: + memberHeartbeatData.getLastDurableOpTime().append( + &entry, UpdatePositionArgs::kDurableOpTimeFieldName); + memberHeartbeatData.getLastAppliedOpTime().append( + &entry, UpdatePositionArgs::kAppliedOpTimeFieldName); + break; + case ReplicationCoordinator::ReplSetUpdatePositionCommandStyle::kOldStyle: + entry.append("_id", memberHeartbeatData.getRid()); + if (_rsConfig.getProtocolVersion() == 1) { + memberHeartbeatData.getLastDurableOpTime().append(&entry, "optime"); + } else { + entry.append("optime", + memberHeartbeatData.getLastDurableOpTime().getTimestamp()); + } + break; + } + entry.append(UpdatePositionArgs::kMemberIdFieldName, memberHeartbeatData.getMemberId()); + entry.append(UpdatePositionArgs::kConfigVersionFieldName, _rsConfig.getConfigVersion()); + } + arrayBuilder.done(); + + // Add metadata to command. Old style parsing logic will reject the metadata. + if (commandStyle == ReplicationCoordinator::ReplSetUpdatePositionCommandStyle::kNewStyle) { + prepareReplSetMetadata(currentCommittedSnapshotOpTime).writeToMetadata(&cmdBuilder); + } + return cmdBuilder.obj(); +} + +void TopologyCoordinatorImpl::fillMemberData(BSONObjBuilder* result) { + BSONArrayBuilder replicationProgress(result->subarrayStart("replicationProgress")); + { + for (const auto& memberHeartbeatData : _hbdata) { + BSONObjBuilder entry(replicationProgress.subobjStart()); + entry.append("rid", memberHeartbeatData.getRid()); + const auto lastDurableOpTime = memberHeartbeatData.getLastDurableOpTime(); + if (_rsConfig.getProtocolVersion() == 1) { + BSONObjBuilder opTime(entry.subobjStart("optime")); + opTime.append("ts", lastDurableOpTime.getTimestamp()); + opTime.append("term", lastDurableOpTime.getTerm()); + opTime.done(); + } else { + entry.append("optime", lastDurableOpTime.getTimestamp()); + } + entry.append("host", memberHeartbeatData.getHostAndPort().toString()); + if (_selfIndex >= 0) { + const int memberId = memberHeartbeatData.getMemberId(); + invariant(memberId >= 0); + entry.append("memberId", memberId); + } + } + } +} + void TopologyCoordinatorImpl::fillIsMasterForReplSet(IsMasterResponse* response) { const MemberState myState = getMemberState(); if (!_rsConfig.isInitialized()) { @@ -1919,24 +2160,39 @@ void TopologyCoordinatorImpl::_updateHeartbeatDataForReconfig(const ReplSetConfi for (ReplSetConfig::MemberIterator it = newConfig.membersBegin(); it != newConfig.membersEnd(); ++it, ++index) { const MemberConfig& newMemberConfig = *it; - // TODO: C++11: use emplace_back() - if (index == selfIndex) { - // Insert placeholder for ourself, though we will never consult it. - _hbdata.push_back(MemberHeartbeatData()); - } else { - MemberHeartbeatData newHeartbeatData; - for (int oldIndex = 0; oldIndex < _rsConfig.getNumMembers(); ++oldIndex) { - const MemberConfig& oldMemberConfig = _rsConfig.getMemberAt(oldIndex); - if (oldMemberConfig.getId() == newMemberConfig.getId() && - oldMemberConfig.getHostAndPort() == newMemberConfig.getHostAndPort()) { - // This member existed in the old config with the same member ID and - // HostAndPort, so copy its heartbeat data over. - newHeartbeatData = oldHeartbeats[oldIndex]; - break; - } + MemberHeartbeatData newHeartbeatData; + for (auto&& oldMemberHeartbeatData : oldHeartbeats) { + if ((oldMemberHeartbeatData.getMemberId() == newMemberConfig.getId() && + oldMemberHeartbeatData.getHostAndPort() == newMemberConfig.getHostAndPort()) || + (index == selfIndex && oldMemberHeartbeatData.isSelf())) { + // This member existed in the old config with the same member ID and + // HostAndPort, so copy its heartbeat data over. + newHeartbeatData = oldMemberHeartbeatData; + break; } - _hbdata.push_back(newHeartbeatData); } + newHeartbeatData.setConfigIndex(index); + newHeartbeatData.setIsSelf(index == selfIndex); + newHeartbeatData.setHostAndPort(newMemberConfig.getHostAndPort()); + newHeartbeatData.setMemberId(newMemberConfig.getId()); + _hbdata.push_back(newHeartbeatData); + } + if (selfIndex < 0) { + // It's necessary to have self member data even if self isn't in the configuration. + // We don't need data for the other nodes (which no longer know about us, or soon won't) + _hbdata.clear(); + // We're not in the config, we can't sync any more. + _syncSource = HostAndPort(); + MemberHeartbeatData newHeartbeatData; + for (auto&& oldMemberHeartbeatData : oldHeartbeats) { + if (oldMemberHeartbeatData.isSelf()) { + newHeartbeatData = oldMemberHeartbeatData; + break; + } + } + newHeartbeatData.setConfigIndex(-1); + newHeartbeatData.setIsSelf(true); + _hbdata.push_back(newHeartbeatData); } } @@ -1944,8 +2200,7 @@ void TopologyCoordinatorImpl::_updateHeartbeatDataForReconfig(const ReplSetConfi // that reflect the new config. void TopologyCoordinatorImpl::updateConfig(const ReplSetConfig& newConfig, int selfIndex, - Date_t now, - const OpTime& lastOpApplied) { + Date_t now) { invariant(_role != Role::candidate); invariant(selfIndex < newConfig.getNumMembers()); @@ -2008,8 +2263,21 @@ const MemberConfig& TopologyCoordinatorImpl::_selfConfig() const { return _rsConfig.getMemberAt(_selfIndex); } +const MemberHeartbeatData& TopologyCoordinatorImpl::_selfMemberHeartbeatData() const { + return _hbdata[_selfMemberHeartbeatDataIndex()]; +} + +const int TopologyCoordinatorImpl::_selfMemberHeartbeatDataIndex() const { + invariant(!_hbdata.empty()); + if (_selfIndex >= 0) + return _selfIndex; + // In master-slave mode, the first entry is for self. If there is no config + // or we're not in the config, the first-and-only entry should be for self. + return 0; +} + TopologyCoordinatorImpl::UnelectableReasonMask TopologyCoordinatorImpl::_getUnelectableReason( - int index, const OpTime& lastOpApplied) const { + int index) const { invariant(index != _selfIndex); const MemberConfig& memberConfig = _rsConfig.getMemberAt(index); const MemberHeartbeatData& hbData = _hbdata.at(index); @@ -2024,7 +2292,7 @@ TopologyCoordinatorImpl::UnelectableReasonMask TopologyCoordinatorImpl::_getUnel result |= NotSecondary; } if (_rsConfig.getProtocolVersion() == 0 && - !_isOpTimeCloseEnoughToLatestToElect(hbData.getAppliedOpTime(), lastOpApplied)) { + !_isOpTimeCloseEnoughToLatestToElect(hbData.getHeartbeatAppliedOpTime())) { result |= NotCloseEnoughToLatestOptime; } if (hbData.up() && hbData.isUnelectable()) { @@ -2035,8 +2303,9 @@ TopologyCoordinatorImpl::UnelectableReasonMask TopologyCoordinatorImpl::_getUnel } TopologyCoordinatorImpl::UnelectableReasonMask TopologyCoordinatorImpl::_getMyUnelectableReason( - const Date_t now, const OpTime& lastApplied, bool isPriorityTakeover) const { + const Date_t now, bool isPriorityTakeover) const { UnelectableReasonMask result = None; + const OpTime lastApplied = getMyLastAppliedOpTime(); if (lastApplied.isNull()) { result |= NoData; } @@ -2069,13 +2338,13 @@ TopologyCoordinatorImpl::UnelectableReasonMask TopologyCoordinatorImpl::_getMyUn _voteLease.when + VoteLease::leaseTime >= now) { result |= VotedTooRecently; } - if (!_isOpTimeCloseEnoughToLatestToElect(lastApplied, lastApplied)) { + if (!_isOpTimeCloseEnoughToLatestToElect(lastApplied)) { result |= NotCloseEnoughToLatestOptime; } } else { // Election rules only for protocol version 1. invariant(_rsConfig.getProtocolVersion() == 1); - if (isPriorityTakeover && !_amIFreshEnoughForPriorityTakeover(lastApplied)) { + if (isPriorityTakeover && !_amIFreshEnoughForPriorityTakeover()) { result |= NotCloseEnoughToLatestForPriorityTakeover; } } @@ -2292,7 +2561,7 @@ void TopologyCoordinatorImpl::processLoseElection() { } } -bool TopologyCoordinatorImpl::stepDown(Date_t until, bool force, const OpTime& lastOpApplied) { +bool TopologyCoordinatorImpl::stepDown(Date_t until, bool force) { // force==true overrides all other checks. if (force) { @@ -2303,13 +2572,14 @@ bool TopologyCoordinatorImpl::stepDown(Date_t until, bool force, const OpTime& l // We already checked in ReplicationCoordinator that a majority of nodes are caught up. // Here we must check that we also have at least one caught up node that is electable. + const OpTime lastOpApplied = getMyLastAppliedOpTime(); for (int memberIndex = 0; memberIndex < _rsConfig.getNumMembers(); memberIndex++) { // ignore your self if (memberIndex == _selfIndex) { continue; } - UnelectableReasonMask reason = _getUnelectableReason(memberIndex, lastOpApplied); - if (!reason && _hbdata.at(memberIndex).getAppliedOpTime() >= lastOpApplied) { + UnelectableReasonMask reason = _getUnelectableReason(memberIndex); + if (!reason && _hbdata.at(memberIndex).getHeartbeatAppliedOpTime() >= lastOpApplied) { // Found a caught up and electable node, succeed with step down. _stepDownUntil = until; _stepDownSelfAndReplaceWith(-1); @@ -2388,6 +2658,67 @@ void TopologyCoordinatorImpl::_stepDownSelfAndReplaceWith(int newPrimary) { _stepDownPending = false; } +bool TopologyCoordinatorImpl::updateLastCommittedOpTime() { + if (!getMemberState().primary() || isStepDownPending()) { + return false; + } + + // Whether we use the applied or durable OpTime for the commit point is decided here. + const bool useDurableOpTime = _rsConfig.getWriteConcernMajorityShouldJournal(); + + std::vector<OpTime> votingNodesOpTimes; + for (const auto& memberHeartbeatData : _hbdata) { + int memberIndex = memberHeartbeatData.getConfigIndex(); + invariant(memberIndex >= 0); + const auto& memberConfig = _rsConfig.getMemberAt(memberIndex); + if (memberConfig.isVoter()) { + const auto opTime = useDurableOpTime ? memberHeartbeatData.getLastDurableOpTime() + : memberHeartbeatData.getLastAppliedOpTime(); + votingNodesOpTimes.push_back(opTime); + } + } + + invariant(votingNodesOpTimes.size() > 0); + if (votingNodesOpTimes.size() < static_cast<unsigned long>(_rsConfig.getWriteMajority())) { + return false; + } + std::sort(votingNodesOpTimes.begin(), votingNodesOpTimes.end()); + + // need the majority to have this OpTime + OpTime committedOpTime = + votingNodesOpTimes[votingNodesOpTimes.size() - _rsConfig.getWriteMajority()]; + return advanceLastCommittedOpTime(committedOpTime); +} + +bool TopologyCoordinatorImpl::advanceLastCommittedOpTime(const OpTime& committedOpTime) { + if (committedOpTime == _lastCommittedOpTime) { + return false; // Hasn't changed, so ignore it. + } else if (committedOpTime < _lastCommittedOpTime) { + LOG(1) << "Ignoring older committed snapshot optime: " << committedOpTime + << ", currentCommittedOpTime: " << _lastCommittedOpTime; + return false; // This may have come from an out-of-order heartbeat. Ignore it. + } + + // This check is performed to ensure primaries do not commit an OpTime from a previous term. + if (getMemberState().primary() && committedOpTime < _firstOpTimeOfMyTerm) { + LOG(1) << "Ignoring older committed snapshot from before I became primary, optime: " + << committedOpTime << ", firstOpTimeOfMyTerm: " << _firstOpTimeOfMyTerm; + return false; + } + + LOG(2) << "Updating _lastCommittedOpTime to " << committedOpTime; + _lastCommittedOpTime = committedOpTime; + return true; +} + +OpTime TopologyCoordinatorImpl::getLastCommittedOpTime() const { + return _lastCommittedOpTime; +} + +void TopologyCoordinatorImpl::setFirstOpTimeOfMyTerm(const OpTime& newOpTime) { + _firstOpTimeOfMyTerm = newOpTime; +} + void TopologyCoordinatorImpl::adjustMaintenanceCountBy(int inc) { invariant(_role == Role::follower); _maintenanceModeCalls += inc; @@ -2425,7 +2756,6 @@ long long TopologyCoordinatorImpl::getTerm() { // replset. Passing metadata is unnecessary. bool TopologyCoordinatorImpl::shouldChangeSyncSource( const HostAndPort& currentSource, - const OpTime& myLastOpTime, const rpc::ReplSetMetadata& replMetadata, boost::optional<rpc::OplogQueryMetadata> oqMetadata, Date_t now) const { @@ -2436,6 +2766,11 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource( // If the currentSource has the same replication progress as we do and has no source for further // progress, return true. + if (_selfIndex == -1) { + log() << "Not choosing new sync source because we are not in the config."; + return false; + } + // If the user requested a sync source change, return true. if (_forceSyncSourceIndex != -1) { log() << "Choosing new sync source because the user has requested to use " @@ -2469,12 +2804,12 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource( int primaryIndex = -1; if (oqMetadata) { currentSourceOpTime = std::max(oqMetadata->getLastOpApplied(), - _hbdata.at(currentSourceIndex).getAppliedOpTime()); + _hbdata.at(currentSourceIndex).getHeartbeatAppliedOpTime()); syncSourceIndex = oqMetadata->getSyncSourceIndex(); primaryIndex = oqMetadata->getPrimaryIndex(); } else { currentSourceOpTime = std::max(replMetadata.getLastOpVisible(), - _hbdata.at(currentSourceIndex).getAppliedOpTime()); + _hbdata.at(currentSourceIndex).getHeartbeatAppliedOpTime()); syncSourceIndex = replMetadata.getSyncSourceIndex(); primaryIndex = replMetadata.getPrimaryIndex(); } @@ -2487,6 +2822,7 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource( // Change sync source if they are not ahead of us, and don't have a sync source, // unless they are primary. + const OpTime myLastOpTime = getMyLastAppliedOpTime(); if (_rsConfig.getProtocolVersion() == 1 && syncSourceIndex == -1 && currentSourceOpTime <= myLastOpTime && primaryIndex != currentSourceIndex) { std::stringstream logMessage; @@ -2520,13 +2856,14 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource( if (it->up() && (candidateConfig.isVoter() || !_selfConfig().isVoter()) && (candidateConfig.shouldBuildIndexes() || !_selfConfig().shouldBuildIndexes()) && it->getState().readable() && !_memberIsBlacklisted(candidateConfig, now) && - goalSecs < it->getAppliedOpTime().getSecs()) { + goalSecs < it->getHeartbeatAppliedOpTime().getSecs()) { log() << "Choosing new sync source because the most recent OpTime of our sync " "source, " << currentSource << ", is " << currentSourceOpTime.toString() << " which is more than " << _options.maxSyncSourceLagSecs << " behind member " << candidateConfig.getHostAndPort().toString() - << " whose most recent OpTime is " << it->getAppliedOpTime().toString(); + << " whose most recent OpTime is " + << it->getHeartbeatAppliedOpTime().toString(); invariant(itIndex != _selfIndex); return true; } @@ -2537,9 +2874,9 @@ bool TopologyCoordinatorImpl::shouldChangeSyncSource( } rpc::ReplSetMetadata TopologyCoordinatorImpl::prepareReplSetMetadata( - const OpTime& lastVisibleOpTime, const OpTime& lastCommittedOpTime) const { + const OpTime& lastVisibleOpTime) const { return rpc::ReplSetMetadata(_term, - lastCommittedOpTime, + _lastCommittedOpTime, lastVisibleOpTime, _rsConfig.getConfigVersion(), _rsConfig.getReplicaSetId(), @@ -2547,10 +2884,9 @@ rpc::ReplSetMetadata TopologyCoordinatorImpl::prepareReplSetMetadata( _rsConfig.findMemberIndexByHostAndPort(getSyncSourceAddress())); } -rpc::OplogQueryMetadata TopologyCoordinatorImpl::prepareOplogQueryMetadata( - const OpTime& lastCommittedOpTime, const OpTime& lastAppliedOpTime, int rbid) const { - return rpc::OplogQueryMetadata(lastCommittedOpTime, - lastAppliedOpTime, +rpc::OplogQueryMetadata TopologyCoordinatorImpl::prepareOplogQueryMetadata(int rbid) const { + return rpc::OplogQueryMetadata(_lastCommittedOpTime, + getMyLastAppliedOpTime(), rbid, _currentPrimaryIndex, _rsConfig.findMemberIndexByHostAndPort(getSyncSourceAddress())); @@ -2566,8 +2902,7 @@ void TopologyCoordinatorImpl::summarizeAsHtml(ReplSetHtmlSummary* output) { } void TopologyCoordinatorImpl::processReplSetRequestVotes(const ReplSetRequestVotesArgs& args, - ReplSetRequestVotesResponse* response, - const OpTime& lastAppliedOpTime) { + ReplSetRequestVotesResponse* response) { response->setTerm(_term); if (args.getTerm() < _term) { @@ -2579,7 +2914,7 @@ void TopologyCoordinatorImpl::processReplSetRequestVotes(const ReplSetRequestVot } else if (args.getSetName() != _rsConfig.getReplSetName()) { response->setVoteGranted(false); response->setReason("candidate's set name differs from mine"); - } else if (args.getLastDurableOpTime() < lastAppliedOpTime) { + } else if (args.getLastDurableOpTime() < getMyLastAppliedOpTime()) { response->setVoteGranted(false); response->setReason("candidate's data is staler than mine"); } else if (!args.isADryRun() && _lastVote.getTerm() == args.getTerm()) { @@ -2612,7 +2947,6 @@ void TopologyCoordinatorImpl::setPrimaryIndex(long long primaryIndex) { } Status TopologyCoordinatorImpl::becomeCandidateIfElectable(const Date_t now, - const OpTime& lastOpApplied, bool isPriorityTakeover) { if (_role == Role::leader) { return {ErrorCodes::NodeNotElectable, "Not standing for election again; already primary"}; @@ -2623,7 +2957,7 @@ Status TopologyCoordinatorImpl::becomeCandidateIfElectable(const Date_t now, } const UnelectableReasonMask unelectableReason = - _getMyUnelectableReason(now, lastOpApplied, isPriorityTakeover); + _getMyUnelectableReason(now, isPriorityTakeover); if (unelectableReason) { return {ErrorCodes::NodeNotElectable, str::stream() << "Not standing for election because " @@ -2664,8 +2998,8 @@ boost::optional<OpTime> TopologyCoordinatorImpl::latestKnownOpTimeSinceHeartbeat if (!peer.up()) { continue; } - if (peer.getAppliedOpTime() > latest) { - latest = peer.getAppliedOpTime(); + if (peer.getHeartbeatAppliedOpTime() > latest) { + latest = peer.getHeartbeatAppliedOpTime(); } } return latest; diff --git a/src/mongo/db/repl/topology_coordinator_impl.h b/src/mongo/db/repl/topology_coordinator_impl.h index 49771ff1728..16ab196842f 100644 --- a/src/mongo/db/repl/topology_coordinator_impl.h +++ b/src/mongo/db/repl/topology_coordinator_impl.h @@ -157,51 +157,49 @@ public: virtual void unblacklistSyncSource(const HostAndPort& host, Date_t now); virtual void clearSyncSourceBlacklist(); virtual bool shouldChangeSyncSource(const HostAndPort& currentSource, - const OpTime& myLastOpTime, const rpc::ReplSetMetadata& replMetadata, boost::optional<rpc::OplogQueryMetadata> oqMetadata, Date_t now) const; virtual bool becomeCandidateIfStepdownPeriodOverAndSingleNodeSet(Date_t now); virtual void setElectionSleepUntil(Date_t newTime); virtual void setFollowerMode(MemberState::MS newMode); + virtual bool updateLastCommittedOpTime(); + virtual bool advanceLastCommittedOpTime(const OpTime& committedOpTime); + virtual OpTime getLastCommittedOpTime() const; + virtual void setFirstOpTimeOfMyTerm(const OpTime& newOpTime); virtual void adjustMaintenanceCountBy(int inc); virtual void prepareSyncFromResponse(const HostAndPort& target, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result); virtual void prepareFreshResponse(const ReplicationCoordinator::ReplSetFreshArgs& args, Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result); virtual void prepareElectResponse(const ReplicationCoordinator::ReplSetElectArgs& args, Date_t now, - const OpTime& lastOpApplied, BSONObjBuilder* response, Status* result); virtual Status prepareHeartbeatResponse(Date_t now, const ReplSetHeartbeatArgs& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response); virtual Status prepareHeartbeatResponseV1(Date_t now, const ReplSetHeartbeatArgsV1& args, const std::string& ourSetName, - const OpTime& lastOpApplied, - const OpTime& lastOpDurable, ReplSetHeartbeatResponse* response); virtual void prepareStatusResponse(const ReplSetStatusArgs& rsStatusArgs, BSONObjBuilder* response, Status* result); + virtual StatusWith<BSONObj> prepareReplSetUpdatePositionCommand( + ReplicationCoordinator::ReplSetUpdatePositionCommandStyle commandStyle, + OpTime currentCommittedSnapshotOpTime) const; + virtual void fillIsMasterForReplSet(IsMasterResponse* response); + virtual void fillMemberData(BSONObjBuilder* result); virtual StatusWith<PrepareFreezeResponseResult> prepareFreezeResponse(Date_t now, int secs, BSONObjBuilder* response); - virtual void updateConfig(const ReplSetConfig& newConfig, - int selfIndex, - Date_t now, - const OpTime& lastOpApplied); + virtual void updateConfig(const ReplSetConfig& newConfig, int selfIndex, Date_t now); virtual std::pair<ReplSetHeartbeatArgs, Milliseconds> prepareHeartbeatRequest( Date_t now, const std::string& ourSetName, const HostAndPort& target); virtual std::pair<ReplSetHeartbeatArgsV1, Milliseconds> prepareHeartbeatRequestV1( @@ -210,37 +208,46 @@ public: Date_t now, Milliseconds networkRoundTripTime, const HostAndPort& target, - const StatusWith<ReplSetHeartbeatResponse>& hbResponse, - const OpTime& myLastOpApplied); + const StatusWith<ReplSetHeartbeatResponse>& hbResponse); virtual bool voteForMyself(Date_t now); virtual void setElectionInfo(OID electionId, Timestamp electionOpTime); virtual void processWinElection(OID electionId, Timestamp electionOpTime); virtual void processLoseElection(); - virtual Status checkShouldStandForElection(Date_t now, const OpTime& lastOpApplied) const; + virtual Status checkShouldStandForElection(Date_t now) const; virtual void setMyHeartbeatMessage(const Date_t now, const std::string& message); - virtual bool stepDown(Date_t until, bool force, const OpTime& lastOpApplied); + virtual bool stepDown(Date_t until, bool force); virtual bool stepDownIfPending(); virtual bool isStepDownPending() const; virtual Date_t getStepDownTime() const; - virtual rpc::ReplSetMetadata prepareReplSetMetadata(const OpTime& lastVisibleOpTime, - const OpTime& lastCommitttedOpTime) const; - virtual rpc::OplogQueryMetadata prepareOplogQueryMetadata(const OpTime& lastCommittedOpTime, - const OpTime& lastAppliedOpTime, - int rbid) const; + virtual rpc::ReplSetMetadata prepareReplSetMetadata(const OpTime& lastVisibleOpTime) const; + virtual rpc::OplogQueryMetadata prepareOplogQueryMetadata(int rbid) const; virtual void processReplSetRequestVotes(const ReplSetRequestVotesArgs& args, - ReplSetRequestVotesResponse* response, - const OpTime& lastAppliedOpTime); + ReplSetRequestVotesResponse* response); virtual void summarizeAsHtml(ReplSetHtmlSummary* output); virtual void loadLastVote(const LastVote& lastVote); virtual void voteForMyselfV1(); virtual void prepareForStepDown(); virtual void setPrimaryIndex(long long primaryIndex); - virtual HeartbeatResponseAction setMemberAsDown(Date_t now, - const int memberIndex, - const OpTime& myLastOpApplied); - virtual Status becomeCandidateIfElectable(const Date_t now, - const OpTime& lastOpApplied, - bool isPriorityTakeover); + virtual bool haveNumNodesReachedOpTime(const OpTime& opTime, int numNodes, bool durablyWritten); + virtual bool haveTaggedNodesReachedOpTime(const OpTime& opTime, + const ReplSetTagPattern& tagPattern, + bool durablyWritten); + virtual std::vector<HostAndPort> getHostsWrittenTo(const OpTime& op, + bool durablyWritten, + bool skipSelf); + virtual HeartbeatResponseAction setMemberAsDown(Date_t now, const int memberIndex); + virtual std::pair<int, Date_t> getStalestLiveMember() const; + virtual HeartbeatResponseAction checkMemberTimeouts(Date_t now); + virtual void resetAllMemberTimeouts(Date_t now); + virtual void resetMemberTimeouts(Date_t now, + const stdx::unordered_set<HostAndPort>& member_set); + virtual OpTime getMyLastAppliedOpTime() const; + virtual OpTime getMyLastDurableOpTime() const; + virtual MemberHeartbeatData* getMyMemberHeartbeatData(); + virtual MemberHeartbeatData* findMemberHeartbeatDataByMemberId(const int memberId); + virtual MemberHeartbeatData* findMemberHeartbeatDataByRid(const OID rid); + virtual MemberHeartbeatData* addSlaveMemberData(const OID rid); + virtual Status becomeCandidateIfElectable(const Date_t now, bool isPriorityTakeover); virtual void setStorageEngineSupportsReadCommitted(bool supported); virtual void restartHeartbeats(); @@ -297,12 +304,10 @@ private: // Returns the current "ping" value for the given member by their address Milliseconds _getPing(const HostAndPort& host); - // Determines if we will veto the member specified by "args.id", given that the last op - // we have applied locally is "lastOpApplied". + // Determines if we will veto the member specified by "args.id". // If we veto, the errmsg will be filled in with a reason bool _shouldVetoMember(const ReplicationCoordinator::ReplSetFreshArgs& args, const Date_t& now, - const OpTime& lastOpApplied, std::string* errmsg) const; // Returns the index of the member with the matching id, or -1 if none match. @@ -317,19 +322,16 @@ private: // Is otherOpTime close enough (within 10 seconds) to the latest known optime to qualify // for an election - bool _isOpTimeCloseEnoughToLatestToElect(const OpTime& otherOpTime, - const OpTime& ourLastOpApplied) const; + bool _isOpTimeCloseEnoughToLatestToElect(const OpTime& otherOpTime) const; // Is our optime close enough to the latest known optime to call for a priority takeover. - bool _amIFreshEnoughForPriorityTakeover(const OpTime& ourLastOpApplied) const; + bool _amIFreshEnoughForPriorityTakeover() const; // Returns reason why "self" member is unelectable - UnelectableReasonMask _getMyUnelectableReason(const Date_t now, - const OpTime& lastOpApplied, - bool isPriorityTakeover) const; + UnelectableReasonMask _getMyUnelectableReason(const Date_t now, bool isPriorityTakeover) const; // Returns reason why memberIndex is unelectable - UnelectableReasonMask _getUnelectableReason(int memberIndex, const OpTime& lastOpApplied) const; + UnelectableReasonMask _getUnelectableReason(int memberIndex) const; // Returns the nice text of why the node is unelectable std::string _getUnelectableReasonString(UnelectableReasonMask ur) const; @@ -338,10 +340,10 @@ private: bool _iAmPrimary() const; // Scans through all members that are 'up' and return the latest known optime. - OpTime _latestKnownOpTime(const OpTime& ourLastOpApplied) const; + OpTime _latestKnownOpTime() const; // Scans the electable set and returns the highest priority member index - int _getHighestPriorityElectableIndex(Date_t now, const OpTime& lastOpApplied) const; + int _getHighestPriorityElectableIndex(Date_t now) const; // Returns true if "one" member is higher priority than "two" member bool _isMemberHigherPriority(int memberOneIndex, int memberTwoIndex) const; @@ -349,6 +351,12 @@ private: // Helper shortcut to self config const MemberConfig& _selfConfig() const; + // Helper shortcut to self member data + const MemberHeartbeatData& _selfMemberHeartbeatData() const; + + // Index of self member in member heartbeat data. + const int _selfMemberHeartbeatDataIndex() const; + // Returns NULL if there is no primary, or the MemberConfig* for the current primary const MemberConfig* _currentPrimaryMember() const; @@ -360,12 +368,10 @@ private: */ HeartbeatResponseAction _updatePrimaryFromHBData(int updatedConfigIndex, const MemberState& originalState, - Date_t now, - const OpTime& lastOpApplied); + Date_t now); HeartbeatResponseAction _updatePrimaryFromHBDataV1(int updatedConfigIndex, const MemberState& originalState, - Date_t now, - const OpTime& lastOpApplied); + Date_t now); /** * Updates _hbdata based on the newConfig, ensuring that every member in the newConfig @@ -441,6 +447,13 @@ private: // a new term. Date_t _electionSleepUntil; + // OpTime of the latest committed operation. + OpTime _lastCommittedOpTime; + + // OpTime representing our transition to PRIMARY and the start of our term. + // _lastCommittedOpTime cannot be set to an earlier OpTime. + OpTime _firstOpTimeOfMyTerm; + // The number of calls we have had to enter maintenance mode int _maintenanceModeCalls; diff --git a/src/mongo/db/repl/topology_coordinator_impl_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_test.cpp index 0cac80e1f52..29b09922cc5 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_test.cpp @@ -139,11 +139,11 @@ protected: _selfIndex = selfIndex; if (now == Date_t::fromMillisSinceEpoch(-1)) { - getTopoCoord().updateConfig(config, selfIndex, _now, lastOp); + getTopoCoord().updateConfig(config, selfIndex, _now); _now += Milliseconds(1); } else { invariant(now > _now); - getTopoCoord().updateConfig(config, selfIndex, now, lastOp); + getTopoCoord().updateConfig(config, selfIndex, now); _now = now + Milliseconds(1); } @@ -171,15 +171,13 @@ protected: const std::string& setName, MemberState memberState, const OpTime& electionTime, - const OpTime& lastOpTimeSender, - const OpTime& lastOpTimeReceiver) { + const OpTime& lastOpTimeSender) { return _receiveHeartbeatHelper(Status::OK(), member, setName, memberState, electionTime.getTimestamp(), lastOpTimeSender, - lastOpTimeReceiver, Milliseconds(1)); } @@ -197,7 +195,6 @@ protected: MemberState::RS_UNKNOWN, Timestamp(), OpTime(), - lastOpTimeReceiver, roundTripTime); } @@ -212,7 +209,6 @@ protected: memberState, Timestamp(), lastOpTimeSender, - OpTime(), roundTripTime); } @@ -223,7 +219,6 @@ private: MemberState memberState, Timestamp electionTime, const OpTime& lastOpTimeSender, - const OpTime& lastOpTimeReceiver, Milliseconds roundTripTime) { ReplSetHeartbeatResponse hb; hb.setConfigVersion(1); @@ -238,8 +233,7 @@ private: getTopoCoord().prepareHeartbeatRequest(now(), setName, member); now() += roundTripTime; - return getTopoCoord().processHeartbeatResponse( - now(), roundTripTime, member, hbResponse, lastOpTimeReceiver); + return getTopoCoord().processHeartbeatResponse(now(), roundTripTime, member, hbResponse); } private: @@ -734,9 +728,9 @@ TEST_F(TopoCoordTest, ChooseRequestedSyncSourceOnlyTheFirstTimeAfterTheSyncSourc // force should cause shouldChangeSyncSource() to return true // even if the currentSource is the force target ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("h2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("h2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("h3"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("h3"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); getTopoCoord().chooseNewSyncSource( now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress()); @@ -926,12 +920,8 @@ TEST_F(TopoCoordTest, NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp) { ASSERT_EQUALS(MemberState::RS_RECOVERING, getTopoCoord().getMemberState().s); // Having an auth error but with another node up should bring us out of RECOVERING - HeartbeatResponseAction action = receiveUpHeartbeat(HostAndPort("h2"), - "rs0", - MemberState::RS_SECONDARY, - OpTime(), - OpTime(Timestamp(2, 0), 0), - OpTime(Timestamp(2, 0), 0)); + HeartbeatResponseAction action = receiveUpHeartbeat( + HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(), OpTime(Timestamp(2, 0), 0)); ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s); // Test that the heartbeat that brings us from RECOVERING to SECONDARY doesn't initiate // an election (SERVER-17164) @@ -960,22 +950,16 @@ TEST_F(TopoCoordTest, NodeDoesNotActOnHeartbeatsWhenAbsentFromConfig) { } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunPriorToHavingAConfig) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; // if we do not have an index in the config, we should get ErrorCodes::NotSecondary - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("Removed and uninitialized nodes do not sync", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -994,15 +978,12 @@ TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter) { << "h1"))), 0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("arbiters don't sync", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1038,16 +1019,13 @@ TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary) { makeSelfPrimary(); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); getTopoCoord()._setCurrentPrimaryForTest(0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h3"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h3"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("primaries don't sync", result.reason()); ASSERT_EQUALS("h3:27017", response.obj()["syncFromRequested"].String()); } TEST_F(TopoCoordTest, NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfig) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1079,16 +1057,12 @@ TEST_F(TopoCoordTest, NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfi 0); setSelfMemberState(MemberState::RS_SECONDARY); - getTopoCoord().prepareSyncFromResponse( - HostAndPort("fakemember"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("fakemember"), &response, &result); ASSERT_EQUALS(ErrorCodes::NodeNotFound, result); ASSERT_EQUALS("Could not find member \"fakemember:27017\" in replica set", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1121,15 +1095,12 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf) { setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from self - getTopoCoord().prepareSyncFromResponse(HostAndPort("hself"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("hself"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("I cannot sync from myself", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1163,15 +1134,12 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter) { // Try to sync from an arbiter - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h1:27017\" because it is an arbiter", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbuilder) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1204,16 +1172,13 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbui setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from a node that doesn't build indexes - getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it does not build indexes", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1248,7 +1213,7 @@ TEST_F(TopoCoordTest, NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode) { // Try to sync from a member that is down receiveDownHeartbeat(HostAndPort("h4"), "rs0", OpTime()); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h4"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h4"), &response, &result); ASSERT_EQUALS(ErrorCodes::HostUnreachable, result); ASSERT_EQUALS("I cannot reach the requested member: h4:27017", result.reason()); } @@ -1287,12 +1252,13 @@ TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAStaleNode) { << "h6"))), 0); setSelfMemberState(MemberState::RS_SECONDARY); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); // Sync successfully from a member that is stale heartbeatFromMember( HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, staleOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_OK(result); ASSERT_EQUALS("requested member \"h5:27017\" is more than 10 seconds behind us", response.obj()["warning"].String()); @@ -1340,7 +1306,7 @@ TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAValidNode) { heartbeatFromMember( HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result); ASSERT_OK(result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); @@ -1389,7 +1355,7 @@ TEST_F(TopoCoordTest, HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); // node goes down between forceSync and chooseNewSyncSource - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); receiveDownHeartbeat(HostAndPort("h6"), "rs0", OpTime()); @@ -1436,7 +1402,7 @@ TEST_F(TopoCoordTest, NodeReturnsUnauthorizedWhenSyncFromRequestsANodeWeAreNotAu // Try to sync from a member that is unauth'd receiveDownHeartbeat(HostAndPort("h5"), "rs0", OpTime(), ErrorCodes::Unauthorized); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_NOT_OK(result); ASSERT_EQUALS(ErrorCodes::Unauthorized, result.code()); ASSERT_EQUALS("not authorized to communicate with h5:27017", result.reason()); @@ -1457,7 +1423,7 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenAskedToSyncFromANonVoterAsAVo "]}"), 0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it is not a voter", result.reason()); } @@ -1503,7 +1469,7 @@ TEST_F(TopoCoordTest, heartbeatFromMember( HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_OK(result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); @@ -1516,7 +1482,7 @@ TEST_F(TopoCoordTest, HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); // Sync successfully from another up-to-date member. - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response2, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response2, &result); BSONObj response2Obj = response2.obj(); ASSERT_FALSE(response2Obj.hasField("warning")); ASSERT_EQUALS(HostAndPort("h5").toString(), response2Obj["prevSyncTarget"].String()); @@ -1538,7 +1504,7 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { Seconds uptimeSecs(10); Date_t curTime = heartbeatTime + uptimeSecs; Timestamp electionTime(1, 2); - OpTime oplogProgress(Timestamp(3, 4), 0); + OpTime oplogProgress(Timestamp(3, 4), 2); OpTime oplogDurable(Timestamp(3, 4), 1); OpTime lastCommittedOpTime(Timestamp(2, 3), -1); OpTime readConcernMajorityOpTime(Timestamp(4, 5), -1); @@ -1569,7 +1535,7 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { HostAndPort member = HostAndPort("test0:1234"); getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(1), setName, member); getTopoCoord().processHeartbeatResponse( - startupTime + Milliseconds(2), Milliseconds(1), member, hbResponseGood, OpTime()); + startupTime + Milliseconds(2), Milliseconds(1), member, hbResponseGood); getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(3), setName, member); Date_t timeoutTime = startupTime + Milliseconds(3) + ReplSetConfig::kDefaultHeartbeatTimeoutPeriod; @@ -1578,13 +1544,16 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { StatusWith<ReplSetHeartbeatResponse>(Status(ErrorCodes::HostUnreachable, "")); getTopoCoord().processHeartbeatResponse( - timeoutTime, Milliseconds(5000), member, hbResponseDown, OpTime()); + timeoutTime, Milliseconds(5000), member, hbResponseDown); member = HostAndPort("test1:1234"); getTopoCoord().prepareHeartbeatRequest(startupTime + Milliseconds(2), setName, member); getTopoCoord().processHeartbeatResponse( - heartbeatTime, Milliseconds(4000), member, hbResponseGood, OpTime()); + heartbeatTime, Milliseconds(4000), member, hbResponseGood); makeSelfPrimary(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(oplogProgress, startupTime); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(oplogDurable, startupTime); + getTopoCoord().advanceLastCommittedOpTime(lastCommittedOpTime); // Now node 0 is down, node 1 is up, and for node 2 we have no heartbeat data yet. BSONObjBuilder statusBuilder; @@ -1593,9 +1562,6 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { TopologyCoordinator::ReplSetStatusArgs{ curTime, static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), - oplogProgress, - oplogDurable, - lastCommittedOpTime, readConcernMajorityOpTime, BSONObj()}, &statusBuilder, @@ -1711,9 +1677,6 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidReplicaSetConfigInResponseToGetStatusWhe TopologyCoordinator::ReplSetStatusArgs{ curTime, static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), - oplogProgress, - oplogProgress, - OpTime(), OpTime(), BSONObj()}, &statusBuilder, @@ -1732,7 +1695,7 @@ TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenFreshnessIsCheckedPriorTo // if we do not have an index in the config, we should get ErrorCodes::ReplicaSetNotFound BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status); ASSERT_EQUALS("Cannot participate in elections because not initialized", status.reason()); ASSERT_TRUE(responseBuilder.obj().isEmpty()); @@ -1771,7 +1734,7 @@ TEST_F(TopoCoordTest, BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status); ASSERT_TRUE(responseBuilder.obj().isEmpty()); } @@ -1812,7 +1775,8 @@ TEST_F(TopoCoordTest, NodeReturnsFresherWhenFreshnessIsCheckedWithStaleConfigVer BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_EQUALS("config version stale", response["info"].String()); @@ -1858,7 +1822,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWithAMemberWhoIsNotInTheConfig) BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_EQUALS(ourOpTime.getTimestamp(), Timestamp(response["opTime"].timestampValue())); @@ -1906,7 +1871,9 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWhilePrimary) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -1957,7 +1924,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedWhilePrimaryExists) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2010,7 +1978,8 @@ TEST_F(TopoCoordTest, NodeReturnsNotFreshestWhenFreshnessIsCheckedByALowPriority BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2063,7 +2032,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeWeBelieveToBeDown) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2117,7 +2087,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsPrimary) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2169,7 +2140,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsInStartup) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2221,7 +2193,8 @@ TEST_F(TopoCoordTest, VetoWhenFreshnessIsCheckedByANodeThatIsRecovering) { BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2275,7 +2248,8 @@ TEST_F(TopoCoordTest, BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")); @@ -2325,7 +2299,8 @@ TEST_F(TopoCoordTest, RespondPositivelyWhenFreshnessIsCheckedByAnElectableNode) BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_OK(status); BSONObj response = responseBuilder.obj(); ASSERT_FALSE(response.hasField("info")) << response.toString(); @@ -2371,7 +2346,8 @@ TEST_F(TopoCoordTest, NodeReturnsBadValueWhenFreshnessIsCheckedByANodeWithOurID) BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(args, Date_t(), ourOpTime, &responseBuilder, &status); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareFreshResponse(args, Date_t(), &responseBuilder, &status); ASSERT_EQUALS(ErrorCodes::BadValue, status); ASSERT_EQUALS( "Received replSetFresh command from member with the same member ID as ourself: 10", @@ -2391,12 +2367,8 @@ TEST_F(TopoCoordTest, HeartbeatFrequencyShouldBeHalfElectionTimeoutWhenArbiter) Date_t requestDate = now(); std::pair<ReplSetHeartbeatArgs, Milliseconds> uppingRequest = getTopoCoord().prepareHeartbeatRequest(requestDate, "myset", target); - auto action = - getTopoCoord().processHeartbeatResponse(requestDate, - Milliseconds(0), - target, - makeStatusWith<ReplSetHeartbeatResponse>(), - OpTime(Timestamp(0, 0), 0)); + auto action = getTopoCoord().processHeartbeatResponse( + requestDate, Milliseconds(0), target, makeStatusWith<ReplSetHeartbeatResponse>()); Date_t expected(now() + Milliseconds(2500)); ASSERT_EQUALS(expected, action.getNextHeartbeatStartDate()); } @@ -2436,8 +2408,7 @@ public: _upRequestDate, Milliseconds(0), _target, - makeStatusWith<ReplSetHeartbeatResponse>(), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + makeStatusWith<ReplSetHeartbeatResponse>()); // We've never applied anything. ASSERT_EQUALS(HeartbeatResponseAction::NoAction, upAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -2456,8 +2427,7 @@ public: _firstRequestDate + Seconds(4), // 4 seconds elapsed, retry allowed. Milliseconds(3990), // Spent 3.99 of the 4 seconds in the network. _target, - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -2474,13 +2444,8 @@ public: BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{_firstRequestDate + Milliseconds(4000), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + _firstRequestDate + Milliseconds(4000), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -2520,8 +2485,7 @@ public: // could retry. Milliseconds(400), // Spent 0.4 of the 0.5 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // Because the first retry failed without timing out, we expect to retry immediately. @@ -2538,13 +2502,8 @@ public: BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Seconds(4), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Seconds(4), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -2573,8 +2532,6 @@ public: TEST_F(HeartbeatResponseHighVerbosityTest, LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatBelievesWeAreDown) { - OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); - // request heartbeat std::pair<ReplSetHeartbeatArgs, Milliseconds> request = getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2")); @@ -2590,8 +2547,7 @@ TEST_F(HeartbeatResponseHighVerbosityTest, now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down")); @@ -2599,8 +2555,6 @@ TEST_F(HeartbeatResponseHighVerbosityTest, TEST_F(HeartbeatResponseHighVerbosityTest, LogMessageAndTakeNoActionWhenReceivingAHeartbeatResponseFromANodeThatIsNotInConfig) { - OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); - // request heartbeat std::pair<ReplSetHeartbeatArgs, Milliseconds> request = getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host5")); @@ -2616,8 +2570,7 @@ TEST_F(HeartbeatResponseHighVerbosityTest, now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host5"), - StatusWith<ReplSetHeartbeatResponse>(memberMissingResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(memberMissingResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("Could not find host5:27017 in current config")); @@ -2625,8 +2578,6 @@ TEST_F(HeartbeatResponseHighVerbosityTest, // TODO(dannenberg) figure out why this test is useful TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataSameConfig) { - OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); - // request heartbeat std::pair<ReplSetHeartbeatArgs, Milliseconds> request = getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2")); @@ -2661,8 +2612,7 @@ TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataSameConfig) { now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(sameConfigResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(sameConfigResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("Config from heartbeat response was same as ours.")); @@ -2670,8 +2620,6 @@ TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataSameConfig) { // TODO(dannenberg) change the name and functionality of this to match what this claims it is TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataOldConfig) { - OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); - // request heartbeat std::pair<ReplSetHeartbeatArgs, Milliseconds> request = getTopoCoord().prepareHeartbeatRequest(now()++, "rs0", HostAndPort("host2")); @@ -2687,8 +2635,7 @@ TEST_F(HeartbeatResponseHighVerbosityTest, UpdateHeartbeatDataOldConfig) { now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down")); @@ -2726,8 +2673,7 @@ TEST_F(HeartbeatResponseTestOneRetry, ReconfigWhenHeartbeatResponseContainsAConf firstRequestDate() + Milliseconds(4500), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(reconfigResponse), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(reconfigResponse)); ASSERT_EQUALS(HeartbeatResponseAction::Reconfig, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate()); @@ -2753,8 +2699,7 @@ TEST_F(HeartbeatResponseTestOneRetry, StepDownRemotePrimaryWhenWeWereElectedMore firstRequestDate() + Milliseconds(4500), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse), - OpTime()); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, action.getAction()); ASSERT_EQUALS(1, action.getPrimaryConfigIndex()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate()); @@ -2783,8 +2728,7 @@ TEST_F(HeartbeatResponseTestOneRetry, StepDownSelfWhenRemoteNodeWasElectedMoreRe firstRequestDate() + Milliseconds(4500), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, action.getAction()); ASSERT_EQUALS(0, action.getPrimaryConfigIndex()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate()); @@ -2805,16 +2749,14 @@ TEST_F(HeartbeatResponseTestOneRetry, // acknowledge the other member so that we see a majority OpTime election = OpTime(Timestamp(400, 0), 0); OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); - HeartbeatResponseAction action = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction action = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(action.getAction()); // make sure we are electable setSelfMemberState(MemberState::RS_SECONDARY); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); ReplSetHeartbeatResponse startElectionResponse; startElectionResponse.noteReplSet(); @@ -2826,8 +2768,7 @@ TEST_F(HeartbeatResponseTestOneRetry, firstRequestDate() + Milliseconds(4500), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(startElectionResponse), - election); + StatusWith<ReplSetHeartbeatResponse>(startElectionResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StartElection, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate()); @@ -2849,8 +2790,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, NodeDoesNotRetryHeartbeatsAfterFailingTw // could still retry. Milliseconds(100), // Spent 0.1 of the 0.3 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // Because this is the second retry, rather than retry again, we expect to wait for the @@ -2861,13 +2801,8 @@ TEST_F(HeartbeatResponseTestTwoRetries, NodeDoesNotRetryHeartbeatsAfterFailingTw BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Milliseconds(4900), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Milliseconds(4900), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -2911,8 +2846,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, ReconfigWhenHeartbeatResponseContainsACo firstRequestDate() + Milliseconds(4500), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(reconfigResponse), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(reconfigResponse)); ASSERT_EQUALS(HeartbeatResponseAction::Reconfig, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(6500), action.getNextHeartbeatStartDate()); @@ -2938,8 +2872,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, StepDownRemotePrimaryWhenWeWereElectedMo firstRequestDate() + Milliseconds(5000), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse), - OpTime()); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, action.getAction()); ASSERT_EQUALS(1, action.getPrimaryConfigIndex()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate()); @@ -2968,8 +2901,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, StepDownSelfWhenRemoteNodeWasElectedMore firstRequestDate() + Milliseconds(5000), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(electedMoreRecentlyResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, action.getAction()); ASSERT_EQUALS(0, action.getPrimaryConfigIndex()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate()); @@ -2990,16 +2922,14 @@ TEST_F(HeartbeatResponseTestTwoRetries, // acknowledge the other member so that we see a majority OpTime election = OpTime(Timestamp(400, 0), 0); OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); - HeartbeatResponseAction action = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction action = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(action.getAction()); // make sure we are electable setSelfMemberState(MemberState::RS_SECONDARY); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); ReplSetHeartbeatResponse startElectionResponse; startElectionResponse.noteReplSet(); @@ -3011,8 +2941,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, firstRequestDate() + Milliseconds(5000), // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(startElectionResponse), - election); + StatusWith<ReplSetHeartbeatResponse>(startElectionResponse)); ASSERT_EQUALS(HeartbeatResponseAction::StartElection, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); ASSERT_EQUALS(firstRequestDate() + Milliseconds(7000), action.getNextHeartbeatStartDate()); @@ -3037,8 +2966,7 @@ TEST_F(HeartbeatResponseTest, NodeDoesNotRetryHeartbeatIfTheFirstFailureTakesThe // no retry allowed. Milliseconds(4990), // Spent 4.99 of the 5 seconds in the network. target, - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3055,8 +2983,7 @@ TEST_F(HeartbeatResponseTestOneRetry, // no retry allowed. Milliseconds(1000), // Spent 1 of the 1.01 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3076,12 +3003,11 @@ TEST_F(HeartbeatResponseTestTwoRetries, response.setConfigVersion(5); // successful response (third response due to the two failures in setUp()) - HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - firstRequestDate() + Milliseconds(4500), - Milliseconds(400), - target(), - StatusWith<ReplSetHeartbeatResponse>(response), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + HeartbeatResponseAction action = + getTopoCoord().processHeartbeatResponse(firstRequestDate() + Milliseconds(4500), + Milliseconds(400), + target(), + StatusWith<ReplSetHeartbeatResponse>(response)); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3096,8 +3022,7 @@ TEST_F(HeartbeatResponseTestTwoRetries, firstRequestDate() + Milliseconds(7100), Milliseconds(400), target(), - StatusWith<ReplSetHeartbeatResponse>(Status{ErrorCodes::HostUnreachable, ""}), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(Status{ErrorCodes::HostUnreachable, ""})); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3106,13 +3031,8 @@ TEST_F(HeartbeatResponseTestTwoRetries, BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Milliseconds(7000), - 600, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Milliseconds(7000), 600, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -3129,12 +3049,9 @@ TEST_F(HeartbeatResponseTest, UpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOfAN OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3147,21 +3064,14 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election2, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election); // second primary does not change primary index ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); @@ -3175,21 +3085,14 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election2, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election); // second primary does not change primary index ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); @@ -3205,12 +3108,9 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); ASSERT_EQUALS(HeartbeatResponseAction::StepDownRemotePrimary, nextAction.getAction()); ASSERT_EQUALS(1, nextAction.getPrimaryConfigIndex()); @@ -3245,20 +3145,15 @@ TEST_F(HeartbeatResponseTest, UpdateHeartbeatDataStepDownPrimaryForHighPriorityF OpTime slightlyLessFreshLastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); nextAction = receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, - slightlyLessFreshLastOpTimeApplied, - lastOpTimeApplied); + slightlyLessFreshLastOpTimeApplied); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, nextAction.getAction()); } @@ -3294,7 +3189,7 @@ TEST_F( ASSERT_EQUALS(0, getCurrentPrimaryIndex()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election, election); + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, nextAction.getAction()); ASSERT_EQUALS(0, nextAction.getPrimaryConfigIndex()); @@ -3308,8 +3203,9 @@ TEST_F( hbArgs.setSenderId(1); hbArgs.setSenderHost(HostAndPort("host3", 27017)); ReplSetHeartbeatResponse hbResp; - ASSERT_OK( - getTopoCoord().prepareHeartbeatResponse(now(), hbArgs, "rs0", election, election, &hbResp)); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(election, Date_t()); + ASSERT_OK(getTopoCoord().prepareHeartbeatResponse(now(), hbArgs, "rs0", &hbResp)); ASSERT(!hbResp.hasIsElectable() || hbResp.isElectable()) << hbResp.toString(); } @@ -3340,8 +3236,9 @@ TEST_F(HeartbeatResponseTest, makeSelfPrimary(election.getTimestamp()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, staleTime, election); + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, staleTime); ASSERT_NO_ACTION(nextAction.getAction()); } @@ -3371,12 +3268,13 @@ TEST_F(HeartbeatResponseTest, OpTime stale = OpTime(); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election, election); + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, stale, election); + nextAction = + receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, stale); ASSERT_NO_ACTION(nextAction.getAction()); } @@ -3388,12 +3286,9 @@ TEST_F(HeartbeatResponseTest, StepDownSelfWhenRemoteNodeWasElectedMoreRecently) OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(HeartbeatResponseAction::StepDownSelf, nextAction.getAction()); ASSERT_EQUALS(0, nextAction.getPrimaryConfigIndex()); // Doesn't actually do the stepdown until stepDownIfPending is called @@ -3413,12 +3308,8 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -3451,21 +3342,13 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -3483,21 +3366,13 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); @@ -3514,12 +3389,8 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -3537,21 +3408,13 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); // freeze node to set stepdown wait @@ -3586,21 +3449,13 @@ TEST_F(HeartbeatResponseTest, OpTime election = OpTime(Timestamp(400, 0), 0); OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -3617,21 +3472,14 @@ TEST_F(HeartbeatResponseTest, StartElectionWhenPrimaryIsMarkedDownAndWeAreElecta OpTime lastOpTimeApplied = OpTime(Timestamp(399, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); @@ -3662,21 +3510,14 @@ TEST_F(HeartbeatResponseTest, NodeDoesNotStartElectionWhileAlreadyCandidate) { OID round = OID::gen(); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // candidate time! @@ -3686,12 +3527,8 @@ TEST_F(HeartbeatResponseTest, NodeDoesNotStartElectionWhileAlreadyCandidate) { ASSERT_TRUE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); // see the downed node as SECONDARY and decide to take no action, but are still a candidate - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); // normally this would trigger StartElection, but we are already a candidate @@ -3732,21 +3569,14 @@ TEST_F(HeartbeatResponseTest, LoseElectionWhenVotingForAnotherNodeWhileRunningTh OpTime fresherOpApplied = OpTime(Timestamp(200, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // candidate time! @@ -3767,8 +3597,8 @@ TEST_F(HeartbeatResponseTest, LoseElectionWhenVotingForAnotherNodeWhileRunningTh BSONObjBuilder freshResponseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareFreshResponse( - freshArgs, now()++, lastOpTimeApplied, &freshResponseBuilder, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result); BSONObj response = freshResponseBuilder.obj(); ASSERT_OK(result); ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue())); @@ -3790,8 +3620,7 @@ TEST_F(HeartbeatResponseTest, LoseElectionWhenVotingForAnotherNodeWhileRunningTh BSONObjBuilder electResponseBuilder; result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - electArgs, now()++, OpTime(), &electResponseBuilder, &result); + getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result); stopCapturingLogMessages(); response = electResponseBuilder.obj(); ASSERT_OK(result); @@ -3807,12 +3636,8 @@ TEST_F(HeartbeatResponseTest, LoseElectionWhenVotingForAnotherNodeWhileRunningTh ASSERT_FALSE(getTopoCoord().voteForMyself(now()++)); // receive a heartbeat indicating the other node was elected - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(2, getCurrentPrimaryIndex()); // make sure seeing a new primary does not change electionTime and electionId @@ -3853,21 +3678,14 @@ TEST_F(HeartbeatResponseTest, OID remoteRound = OID::gen(); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // candidate time! @@ -3886,8 +3704,8 @@ TEST_F(HeartbeatResponseTest, BSONObjBuilder freshResponseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareFreshResponse( - freshArgs, now()++, lastOpTimeApplied, &freshResponseBuilder, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result); BSONObj response = freshResponseBuilder.obj(); ASSERT_OK(result); ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue())); @@ -3914,8 +3732,7 @@ TEST_F(HeartbeatResponseTest, BSONObjBuilder electResponseBuilder; result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - electArgs, now()++, OpTime(), &electResponseBuilder, &result); + getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result); stopCapturingLogMessages(); response = electResponseBuilder.obj(); ASSERT_OK(result); @@ -3951,21 +3768,14 @@ TEST_F(HeartbeatResponseTest, OID remoteRound = OID::gen(); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // candidate time! @@ -3991,8 +3801,8 @@ TEST_F(HeartbeatResponseTest, BSONObjBuilder freshResponseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareFreshResponse( - freshArgs, now()++, lastOpTimeApplied, &freshResponseBuilder, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + getTopoCoord().prepareFreshResponse(freshArgs, now()++, &freshResponseBuilder, &result); BSONObj response = freshResponseBuilder.obj(); ASSERT_OK(result); ASSERT_EQUALS(lastOpTimeApplied.getTimestamp(), Timestamp(response["opTime"].timestampValue())); @@ -4011,8 +3821,7 @@ TEST_F(HeartbeatResponseTest, BSONObjBuilder electResponseBuilder; result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - electArgs, now()++, OpTime(), &electResponseBuilder, &result); + getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result); stopCapturingLogMessages(); response = electResponseBuilder.obj(); ASSERT_OK(result); @@ -4069,12 +3878,9 @@ TEST_F(HeartbeatResponseTest, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -4088,12 +3894,8 @@ TEST_F(HeartbeatResponseTest, ASSERT_NO_ACTION(nextAction.getAction()); nextAction = receiveDownHeartbeat(HostAndPort("host6"), "rs0", lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host7"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host7"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); @@ -4176,8 +3978,7 @@ TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasTheWrongRep BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4190,7 +3991,7 @@ TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasTheWrongRep // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) args.set = "rs0"; BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4207,8 +4008,7 @@ TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasANewerConfi BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4219,7 +4019,7 @@ TEST_F(PrepareElectResponseTest, RespondNegativelyWhenElectCommandHasANewerConfi // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) args.cfgver = 10; BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4236,8 +4036,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasAnOlderConfi BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4248,7 +4047,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasAnOlderConfi // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) args.cfgver = 10; BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4265,8 +4064,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasANonExistent BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4277,7 +4075,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandHasANonExistent // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) args.whoid = 1; BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4296,8 +4094,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedByPri BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4308,7 +4105,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedByPri // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) getTopoCoord()._setCurrentPrimaryForTest(-1); BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4326,8 +4123,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedWhile BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4338,7 +4134,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenElectCommandIsReceivedWhile // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) getTopoCoord()._setCurrentPrimaryForTest(-1); BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4357,8 +4153,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenAHigherPriorityNodeExistsDu BSONObjBuilder responseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_OK(result); @@ -4369,7 +4164,7 @@ TEST_F(PrepareElectResponseTest, RespondWithAVetoWhenAHigherPriorityNodeExistsDu // Make sure nay votes, do not prevent subsequent yeas (the way a yea vote would) args.whoid = 3; BSONObjBuilder responseBuilder2; - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder2, &result); BSONObj response2 = responseBuilder2.obj(); ASSERT_EQUALS(1, response2["vote"].Int()); ASSERT_EQUALS(round, response2["round"].OID()); @@ -4390,8 +4185,7 @@ TEST_F(PrepareElectResponseTest, RespondPositivelyWhenElectCommandComesFromHighe BSONObjBuilder responseBuilder; Status result = Status::OK(); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder, &result); stopCapturingLogMessages(); BSONObj response = responseBuilder.obj(); ASSERT_EQUALS(1, response["vote"].Int()); @@ -4411,8 +4205,7 @@ TEST_F(PrepareElectResponseTest, BSONObjBuilder responseBuilder1; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); startCapturingLogMessages(); - getTopoCoord().prepareElectResponse( - args, now += Seconds(60), OpTime(), &responseBuilder1, &result); + getTopoCoord().prepareElectResponse(args, now += Seconds(60), &responseBuilder1, &result); stopCapturingLogMessages(); BSONObj response1 = responseBuilder1.obj(); ASSERT_OK(result); @@ -4425,7 +4218,7 @@ TEST_F(PrepareElectResponseTest, BSONObjBuilder responseBuilder2; startCapturingLogMessages(); - getTopoCoord().prepareElectResponse(args, now, OpTime(), &responseBuilder2, &result); + getTopoCoord().prepareElectResponse(args, now, &responseBuilder2, &result); stopCapturingLogMessages(); BSONObj response2 = responseBuilder2.obj(); ASSERT_OK(result); @@ -4440,7 +4233,7 @@ TEST_F(PrepareElectResponseTest, BSONObjBuilder responseBuilder3; startCapturingLogMessages(); - getTopoCoord().prepareElectResponse(args, now++, OpTime(), &responseBuilder3, &result); + getTopoCoord().prepareElectResponse(args, now++, &responseBuilder3, &result); stopCapturingLogMessages(); BSONObj response3 = responseBuilder3.obj(); ASSERT_OK(result); @@ -4476,7 +4269,7 @@ TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhil ReplicationCoordinator::ReplSetElectArgs args; BSONObjBuilder response; Status status = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareElectResponse(args, now(), OpTime(), &response, &status); + getTopoCoord().prepareElectResponse(args, now(), &response, &status); ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status); ASSERT_EQUALS("Cannot participate in election because not initialized", status.reason()); } @@ -4485,7 +4278,7 @@ TEST_F(TopoCoordTest, NodeReturnsReplicaSetNotFoundWhenReceivingElectCommandWhil ReplicationCoordinator::ReplSetElectArgs args; BSONObjBuilder response; Status status = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareElectResponse(args, now(), OpTime(), &response, &status); + getTopoCoord().prepareElectResponse(args, now(), &response, &status); ASSERT_EQUALS(ErrorCodes::ReplicaSetNotFound, status); ASSERT_EQUALS("Cannot participate in election because not initialized", status.reason()); } @@ -4641,8 +4434,9 @@ public: OpTime lastOpApplied, ReplSetHeartbeatResponse* response, Status* result) { - *result = getTopoCoord().prepareHeartbeatResponse( - now()++, args, "rs0", lastOpApplied, lastOpApplied, response); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpApplied, Date_t()); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(lastOpApplied, Date_t()); + *result = getTopoCoord().prepareHeartbeatResponse(now()++, args, "rs0", response); } }; @@ -4855,8 +4649,7 @@ TEST_F(TopoCoordTest, SetConfigVersionToNegativeTwoInHeartbeatResponseWhenNoConf args.setSenderId(20); ReplSetHeartbeatResponse response; // prepare response and check the results - Status result = getTopoCoord().prepareHeartbeatResponse( - now()++, args, "rs0", OpTime(), OpTime(), &response); + Status result = getTopoCoord().prepareHeartbeatResponse(now()++, args, "rs0", &response); ASSERT_OK(result); ASSERT_FALSE(response.isElectable()); ASSERT_TRUE(response.isReplSet()); @@ -4965,7 +4758,7 @@ TEST_F(TopoCoordTest, BecomeCandidateWhenReconfigToBeElectableInSingleNodeSet) { << "hself" << "priority" << 0)))); - getTopoCoord().updateConfig(cfg, 0, now()++, OpTime()); + getTopoCoord().updateConfig(cfg, 0, now()++); ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s); ASSERT_FALSE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); @@ -5000,7 +4793,7 @@ TEST_F(TopoCoordTest, NodeDoesNotBecomeCandidateWhenBecomingSecondaryInSingleNod << "priority" << 0)))); - getTopoCoord().updateConfig(cfg, 0, now()++, OpTime()); + getTopoCoord().updateConfig(cfg, 0, now()++); ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s); // despite being the only node, we are unelectable, so we should not become a candidate @@ -5265,20 +5058,12 @@ TEST_F(HeartbeatResponseTest, ReconfigBetweenHeartbeatRequestAndRepsonse) { // all three members up and secondaries setSelfMemberState(MemberState::RS_SECONDARY); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // now request from host3 and receive after host2 has been removed via reconfig @@ -5301,7 +5086,7 @@ TEST_F(HeartbeatResponseTest, ReconfigBetweenHeartbeatRequestAndRepsonse) { hb.setElectionTime(election.getTimestamp()); StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - now()++, Milliseconds(0), HostAndPort("host3"), hbResponse, lastOpTimeApplied); + now()++, Milliseconds(0), HostAndPort("host3"), hbResponse); // now primary should be host3, index 1, and we should perform NoAction in response ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -5316,20 +5101,12 @@ TEST_F(HeartbeatResponseTest, ReconfigNodeRemovedBetweenHeartbeatRequestAndRepso // all three members up and secondaries setSelfMemberState(MemberState::RS_SECONDARY); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // now request from host3 and receive after host2 has been removed via reconfig @@ -5352,7 +5129,7 @@ TEST_F(HeartbeatResponseTest, ReconfigNodeRemovedBetweenHeartbeatRequestAndRepso hb.setElectionTime(election.getTimestamp()); StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - now()++, Milliseconds(0), HostAndPort("host3"), hbResponse, lastOpTimeApplied); + now()++, Milliseconds(0), HostAndPort("host3"), hbResponse); // primary should not be set and we should perform NoAction in response ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); @@ -5364,7 +5141,7 @@ TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenMemberNotInConfig) { // "host4" since "host4" is absent from the config of version 10. ReplSetMetadata metadata(0, OpTime(), OpTime(), 10, OID(), -1, -1); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host4"), OpTime(), metadata, makeOplogQueryMetadata(), now())); + HostAndPort("host4"), metadata, makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenMemberHasYetToHeartbeatUs) { @@ -5372,7 +5149,7 @@ TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenMemberHasYetToHeartbeatU // "host2" since we do not yet have a heartbeat (and as a result do not yet have an optime) // for "host2" ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsFreshByHeartbeatButNotMetadata) { @@ -5384,26 +5161,17 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsFreshByHeartbea // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied), now())); @@ -5412,11 +5180,8 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsFreshByHeartbea // set up complete, time for actual check startCapturingLogMessages(); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); } @@ -5430,27 +5195,18 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsStaleByHeartbea // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_FALSE( getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(fresherLastOpTimeApplied), now())); @@ -5459,12 +5215,8 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenNodeIsStaleByHeartbea // set up complete, time for actual check startCapturingLogMessages(); - ASSERT_FALSE( - getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(fresherLastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(fresherLastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); } @@ -5477,26 +5229,18 @@ TEST_F(HeartbeatResponseTest, ShouldChangeSyncSourceWhenFresherMemberExists) { // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -5511,37 +5255,29 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhileFresherMemberIsBlack // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); getTopoCoord().blacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100)); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); // unblacklist with too early a time (node should remained blacklisted) getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(90)); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); // unblacklist and it should succeed getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100)); startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -5555,27 +5291,19 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberIsDown) // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberIsNotReadable) { @@ -5587,25 +5315,17 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberIsNotRea // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_ROLLBACK, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_ROLLBACK, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberDoesNotBuildIndexes) { @@ -5633,24 +5353,16 @@ TEST_F(HeartbeatResponseTest, ShouldNotChangeSyncSourceWhenFresherMemberDoesNotB << "priority" << 0))), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTest, @@ -5683,25 +5395,17 @@ TEST_F(HeartbeatResponseTest, << "priority" << 0))), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -5723,7 +5427,7 @@ TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileAwareOfPrimary) { heartbeatFromMember( HostAndPort("h2"), "rs0", MemberState::RS_PRIMARY, OpTime(Timestamp(1, 0), 0)); - const auto status = getTopoCoord().checkShouldStandForElection(now()++, OpTime()); + const auto status = getTopoCoord().checkShouldStandForElection(now()++); ASSERT_EQ(ErrorCodes::NodeNotElectable, status); ASSERT_STRING_CONTAINS(status.reason(), "there is a Primary"); } @@ -5745,8 +5449,7 @@ TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileTooStale) { heartbeatFromMember( HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(10000, 0), 0)); - const auto status = - getTopoCoord().checkShouldStandForElection(now()++, OpTime(Timestamp(100, 0), 0)); + const auto status = getTopoCoord().checkShouldStandForElection(now()++); ASSERT_EQ(ErrorCodes::NodeNotElectable, status); ASSERT_STRING_CONTAINS(status.reason(), "my last optime is"); } @@ -5787,8 +5490,7 @@ TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) { } TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) { - const auto status = - getTopoCoord().checkShouldStandForElection(now()++, OpTime(Timestamp(10, 0), 0)); + const auto status = getTopoCoord().checkShouldStandForElection(now()++); ASSERT_EQ(ErrorCodes::NodeNotElectable, status); ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config"); } @@ -5822,16 +5524,14 @@ TEST_F(TopoCoordTest, ShouldNotStandForElectionWhenAPositiveResponseWasGivenInTh now() += Seconds(30); BSONObjBuilder electResponseBuilder; Status result = Status(ErrorCodes::InternalError, "status not set by prepareElectResponse"); - getTopoCoord().prepareElectResponse( - electArgs, now()++, OpTime(Timestamp(100, 0), 0), &electResponseBuilder, &result); + getTopoCoord().prepareElectResponse(electArgs, now()++, &electResponseBuilder, &result); BSONObj response = electResponseBuilder.obj(); ASSERT_OK(result); std::cout << response; ASSERT_EQUALS(1, response["vote"].Int()); ASSERT_EQUALS(remoteRound, response["round"].OID()); - const auto status = - getTopoCoord().checkShouldStandForElection(now()++, OpTime(Timestamp(10, 0), 0)); + const auto status = getTopoCoord().checkShouldStandForElection(now()++); ASSERT_EQ(ErrorCodes::NodeNotElectable, status); ASSERT_STRING_CONTAINS(status.reason(), "I recently voted for "); } @@ -5863,9 +5563,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -5884,7 +5583,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { ReplSetRequestVotesResponse response2; // different candidate same term, should be a problem - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -5919,9 +5618,8 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -5942,7 +5640,7 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response2; - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("", response2.getReason()); ASSERT_TRUE(response2.getVoteGranted()); } @@ -5977,9 +5675,8 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -6000,7 +5697,7 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response2; - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -6033,9 +5730,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6068,9 +5764,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6115,9 +5810,8 @@ TEST_F(TopoCoordTest, ArbiterDoesNotGrantVoteWhenItCanSeeAHealthyPrimaryOfEqualO << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("can see a healthy primary of equal or greater priority", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6154,9 +5848,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); ASSERT_EQUALS(2, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -6191,9 +5884,10 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, + Date_t()); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -6229,10 +5923,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -6253,7 +5945,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -6290,10 +5982,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -6314,7 +6004,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -6351,10 +6041,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -6374,7 +6062,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -6411,10 +6099,8 @@ TEST_F(TopoCoordTest, GrantDryRunVoteEvenWhenTermHasBeenSeen) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -6435,7 +6121,7 @@ TEST_F(TopoCoordTest, GrantDryRunVoteEvenWhenTermHasBeenSeen) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_TRUE(response.getVoteGranted()); @@ -6472,10 +6158,8 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -6495,9 +6179,10 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, + Date_t()); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); diff --git a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp index f1026f6110c..60445499198 100644 --- a/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp +++ b/src/mongo/db/repl/topology_coordinator_impl_v1_test.cpp @@ -129,10 +129,7 @@ protected: // Update config and set selfIndex // If "now" is passed in, set _now to now+1 - void updateConfig(BSONObj cfg, - int selfIndex, - Date_t now = Date_t::fromMillisSinceEpoch(-1), - const OpTime& lastOp = OpTime()) { + void updateConfig(BSONObj cfg, int selfIndex, Date_t now = Date_t::fromMillisSinceEpoch(-1)) { ReplSetConfig config; // Use Protocol version 1 by default. ASSERT_OK(config.initialize(cfg, true)); @@ -141,11 +138,11 @@ protected: _selfIndex = selfIndex; if (now == Date_t::fromMillisSinceEpoch(-1)) { - getTopoCoord().updateConfig(config, selfIndex, _now, lastOp); + getTopoCoord().updateConfig(config, selfIndex, _now); _now += Milliseconds(1); } else { invariant(now > _now); - getTopoCoord().updateConfig(config, selfIndex, now, lastOp); + getTopoCoord().updateConfig(config, selfIndex, now); _now = now + Milliseconds(1); } @@ -178,22 +175,19 @@ protected: const std::string& setName, MemberState memberState, const OpTime& electionTime, - const OpTime& lastOpTimeSender, - const OpTime& lastOpTimeReceiver) { + const OpTime& lastOpTimeSender) { return _receiveHeartbeatHelper(Status::OK(), member, setName, memberState, electionTime.getTimestamp(), lastOpTimeSender, - lastOpTimeReceiver, Milliseconds(1)); } HeartbeatResponseAction receiveDownHeartbeat( const HostAndPort& member, const std::string& setName, - const OpTime& lastOpTimeReceiver, ErrorCodes::Error errcode = ErrorCodes::HostUnreachable) { // timed out heartbeat to mark a node as down @@ -204,7 +198,6 @@ protected: MemberState::RS_UNKNOWN, Timestamp(), OpTime(), - lastOpTimeReceiver, roundTripTime); } @@ -219,7 +212,6 @@ protected: memberState, Timestamp(), lastOpTimeSender, - OpTime(), roundTripTime); } @@ -230,7 +222,6 @@ private: MemberState memberState, Timestamp electionTime, const OpTime& lastOpTimeSender, - const OpTime& lastOpTimeReceiver, Milliseconds roundTripTime) { ReplSetHeartbeatResponse hb; hb.setConfigVersion(1); @@ -246,8 +237,7 @@ private: getTopoCoord().prepareHeartbeatRequestV1(now(), setName, member); now() += roundTripTime; - return getTopoCoord().processHeartbeatResponse( - now(), roundTripTime, member, hbResponse, lastOpTimeReceiver); + return getTopoCoord().processHeartbeatResponse(now(), roundTripTime, member, hbResponse); } private: @@ -326,7 +316,7 @@ TEST_F(TopoCoordTest, NodeReturnsSecondaryWithMostRecentDataAsSyncSource) { ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress()); // h3 goes down - receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h3"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress()); @@ -479,33 +469,33 @@ TEST_F(TopoCoordTest, NodeReturnsClosestValidSyncSourceAsSyncSource) { ASSERT_EQUALS(HostAndPort("h4"), getTopoCoord().getSyncSourceAddress()); // h4 goes down; should choose h1 - receiveDownHeartbeat(HostAndPort("h4"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h4"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h1"), getTopoCoord().getSyncSourceAddress()); // Primary and h1 go down; should choose h6 - receiveDownHeartbeat(HostAndPort("h1"), "rs0", OpTime()); - receiveDownHeartbeat(HostAndPort("hprimary"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h1"), "rs0"); + receiveDownHeartbeat(HostAndPort("hprimary"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h6"), getTopoCoord().getSyncSourceAddress()); // h6 goes down; should choose h5 - receiveDownHeartbeat(HostAndPort("h6"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h6"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h5"), getTopoCoord().getSyncSourceAddress()); // h5 goes down; should choose h3 - receiveDownHeartbeat(HostAndPort("h5"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h5"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h3"), getTopoCoord().getSyncSourceAddress()); // h3 goes down; no sync source candidates remain - receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h3"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT(getTopoCoord().getSyncSourceAddress().empty()); @@ -556,7 +546,7 @@ TEST_F(TopoCoordTest, NodeWontChooseSyncSourceFromOlderTerm) { ASSERT_EQUALS(HostAndPort("h1"), getTopoCoord().getSyncSourceAddress()); // h1 goes down; no sync source candidates remain - receiveDownHeartbeat(HostAndPort("h1"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h1"), "rs0"); getTopoCoord().chooseNewSyncSource( now()++, lastOpTimeWeApplied, TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT(getTopoCoord().getSyncSourceAddress().empty()); @@ -782,20 +772,14 @@ TEST_F(TopoCoordTest, ChooseRequestedSyncSourceOnlyTheFirstTimeAfterTheSyncSourc getTopoCoord().setForceSyncSourceIndex(1); // force should cause shouldChangeSyncSource() to return true // even if the currentSource is the force target - ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("h2"), - OpTime(), - makeReplSetMetadata(), - makeOplogQueryMetadata(oldOpTime), - now())); - ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("h3"), - OpTime(), - makeReplSetMetadata(), - makeOplogQueryMetadata(newOpTime), - now())); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("h2"), OpTime(), makeReplSetMetadata(oldOpTime), boost::none, now())); + HostAndPort("h2"), makeReplSetMetadata(), makeOplogQueryMetadata(oldOpTime), now())); + ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("h3"), makeReplSetMetadata(), makeOplogQueryMetadata(newOpTime), now())); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("h3"), OpTime(), makeReplSetMetadata(newOpTime), boost::none, now())); + HostAndPort("h2"), makeReplSetMetadata(oldOpTime), boost::none, now())); + ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("h3"), makeReplSetMetadata(newOpTime), boost::none, now())); getTopoCoord().chooseNewSyncSource( now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h2"), getTopoCoord().getSyncSourceAddress()); @@ -965,8 +949,8 @@ TEST_F(TopoCoordTest, NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp) { // Good state setup done // Mark nodes down, ensure that we have no source and are secondary - receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime(), ErrorCodes::NetworkTimeout); - receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime(), ErrorCodes::NetworkTimeout); + receiveDownHeartbeat(HostAndPort("h2"), "rs0", ErrorCodes::NetworkTimeout); + receiveDownHeartbeat(HostAndPort("h3"), "rs0", ErrorCodes::NetworkTimeout); ASSERT_TRUE(getTopoCoord() .chooseNewSyncSource(now()++, OpTime(), @@ -975,8 +959,8 @@ TEST_F(TopoCoordTest, NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp) { ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s); // Mark nodes down + unauth, ensure that we have no source and are secondary - receiveDownHeartbeat(HostAndPort("h2"), "rs0", OpTime(), ErrorCodes::Unauthorized); - receiveDownHeartbeat(HostAndPort("h3"), "rs0", OpTime(), ErrorCodes::Unauthorized); + receiveDownHeartbeat(HostAndPort("h2"), "rs0", ErrorCodes::Unauthorized); + receiveDownHeartbeat(HostAndPort("h3"), "rs0", ErrorCodes::Unauthorized); ASSERT_TRUE(getTopoCoord() .chooseNewSyncSource(now()++, OpTime(), @@ -985,12 +969,10 @@ TEST_F(TopoCoordTest, NodeChangesToRecoveringWhenOnlyUnauthorizedNodesAreUp) { ASSERT_EQUALS(MemberState::RS_RECOVERING, getTopoCoord().getMemberState().s); // Having an auth error but with another node up should bring us out of RECOVERING - HeartbeatResponseAction action = receiveUpHeartbeat(HostAndPort("h2"), - "rs0", - MemberState::RS_SECONDARY, - OpTime(), - OpTime(Timestamp(2, 0), 0), - OpTime(Timestamp(2, 0), 0)); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(OpTime(Timestamp(2, 0), 0), + Date_t()); + HeartbeatResponseAction action = receiveUpHeartbeat( + HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(), OpTime(Timestamp(2, 0), 0)); ASSERT_EQUALS(MemberState::RS_SECONDARY, getTopoCoord().getMemberState().s); // Test that the heartbeat that brings us from RECOVERING to SECONDARY doesn't initiate // an election (SERVER-17164) @@ -1019,22 +1001,16 @@ TEST_F(TopoCoordTest, NodeDoesNotActOnHeartbeatsWhenAbsentFromConfig) { } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunPriorToHavingAConfig) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; // if we do not have an index in the config, we should get ErrorCodes::NotSecondary - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("Removed and uninitialized nodes do not sync", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1053,15 +1029,12 @@ TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstArbiter) { << "h1"))), 0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("arbiters don't sync", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1097,16 +1070,13 @@ TEST_F(TopoCoordTest, NodeReturnsNotSecondaryWhenSyncFromIsRunAgainstPrimary) { makeSelfPrimary(); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); getTopoCoord()._setCurrentPrimaryForTest(0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h3"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h3"), &response, &result); ASSERT_EQUALS(ErrorCodes::NotSecondary, result); ASSERT_EQUALS("primaries don't sync", result.reason()); ASSERT_EQUALS("h3:27017", response.obj()["syncFromRequested"].String()); } TEST_F(TopoCoordTest, NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfig) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1138,16 +1108,12 @@ TEST_F(TopoCoordTest, NodeReturnsNodeNotFoundWhenSyncFromRequestsANodeNotInConfi 0); setSelfMemberState(MemberState::RS_SECONDARY); - getTopoCoord().prepareSyncFromResponse( - HostAndPort("fakemember"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("fakemember"), &response, &result); ASSERT_EQUALS(ErrorCodes::NodeNotFound, result); ASSERT_EQUALS("Could not find member \"fakemember:27017\" in replica set", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1180,15 +1146,12 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsSelf) { setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from self - getTopoCoord().prepareSyncFromResponse(HostAndPort("hself"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("hself"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("I cannot sync from myself", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1222,15 +1185,12 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsArbiter) { // Try to sync from an arbiter - getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h1"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h1:27017\" because it is an arbiter", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbuilder) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1263,16 +1223,13 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenSyncFromRequestsAnIndexNonbui setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from a node that doesn't build indexes - getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it does not build indexes", result.reason()); } TEST_F(TopoCoordTest, NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode) { - OpTime staleOpTime(Timestamp(1, 1), 0); - OpTime ourOpTime(Timestamp(staleOpTime.getSecs() + 11, 1), 0); - Status result = Status::OK(); BSONObjBuilder response; @@ -1305,9 +1262,9 @@ TEST_F(TopoCoordTest, NodeReturnsHostUnreachableWhenSyncFromRequestsADownNode) { setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from a member that is down - receiveDownHeartbeat(HostAndPort("h4"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h4"), "rs0"); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h4"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h4"), &response, &result); ASSERT_EQUALS(ErrorCodes::HostUnreachable, result); ASSERT_EQUALS("I cannot reach the requested member: h4:27017", result.reason()); } @@ -1351,7 +1308,8 @@ TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAStaleNode) { heartbeatFromMember( HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, staleOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_OK(result); ASSERT_EQUALS("requested member \"h5:27017\" is more than 10 seconds behind us", response.obj()["warning"].String()); @@ -1399,7 +1357,8 @@ TEST_F(TopoCoordTest, ChooseRequestedNodeWhenSyncFromRequestsAValidNode) { heartbeatFromMember( HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result); ASSERT_OK(result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); @@ -1448,10 +1407,11 @@ TEST_F(TopoCoordTest, HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); // node goes down between forceSync and chooseNewSyncSource - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response, &result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); - receiveDownHeartbeat(HostAndPort("h6"), "rs0", OpTime()); + receiveDownHeartbeat(HostAndPort("h6"), "rs0"); HostAndPort syncSource = getTopoCoord().chooseNewSyncSource( now()++, OpTime(), TopologyCoordinator::ChainingPreference::kUseConfiguration); ASSERT_EQUALS(HostAndPort("h6"), syncSource); @@ -1493,9 +1453,10 @@ TEST_F(TopoCoordTest, NodeReturnsUnauthorizedWhenSyncFromRequestsANodeWeAreNotAu setSelfMemberState(MemberState::RS_SECONDARY); // Try to sync from a member that is unauth'd - receiveDownHeartbeat(HostAndPort("h5"), "rs0", OpTime(), ErrorCodes::Unauthorized); + receiveDownHeartbeat(HostAndPort("h5"), "rs0", ErrorCodes::Unauthorized); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_NOT_OK(result); ASSERT_EQUALS(ErrorCodes::Unauthorized, result.code()); ASSERT_EQUALS("not authorized to communicate with h5:27017", result.reason()); @@ -1508,6 +1469,7 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenAskedToSyncFromANonVoterAsAVo Status result = Status::OK(); BSONObjBuilder response; + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); // Test trying to sync from another node updateConfig(fromjson("{_id:'rs0', version:1, members:[" "{_id:0, host:'self'}," @@ -1516,7 +1478,7 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidOptionsWhenAskedToSyncFromANonVoterAsAVo "]}"), 0); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), ourOpTime, &response, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h2"), &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidOptions, result); ASSERT_EQUALS("Cannot sync from \"h2:27017\" because it is not a voter", result.reason()); } @@ -1562,7 +1524,8 @@ TEST_F(TopoCoordTest, heartbeatFromMember( HostAndPort("h5"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); - getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), ourOpTime, &response, &result); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(ourOpTime, Date_t()); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h5"), &response, &result); ASSERT_OK(result); BSONObj responseObj = response.obj(); ASSERT_FALSE(responseObj.hasField("warning")); @@ -1575,7 +1538,7 @@ TEST_F(TopoCoordTest, HostAndPort("h6"), "rs0", MemberState::RS_SECONDARY, ourOpTime, Milliseconds(100)); // Sync successfully from another up-to-date member. - getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), ourOpTime, &response2, &result); + getTopoCoord().prepareSyncFromResponse(HostAndPort("h6"), &response2, &result); BSONObj response2Obj = response2.obj(); ASSERT_FALSE(response2Obj.hasField("warning")); ASSERT_EQUALS(HostAndPort("h5").toString(), response2Obj["prevSyncTarget"].String()); @@ -1629,7 +1592,7 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { HostAndPort member = HostAndPort("test0:1234"); getTopoCoord().prepareHeartbeatRequestV1(startupTime + Milliseconds(1), setName, member); getTopoCoord().processHeartbeatResponse( - startupTime + Milliseconds(2), Milliseconds(1), member, hbResponseGood, OpTime()); + startupTime + Milliseconds(2), Milliseconds(1), member, hbResponseGood); getTopoCoord().prepareHeartbeatRequestV1(startupTime + Milliseconds(3), setName, member); Date_t timeoutTime = startupTime + Milliseconds(3) + ReplSetConfig::kDefaultHeartbeatTimeoutPeriod; @@ -1638,13 +1601,16 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { StatusWith<ReplSetHeartbeatResponse>(Status(ErrorCodes::HostUnreachable, "")); getTopoCoord().processHeartbeatResponse( - timeoutTime, Milliseconds(5000), member, hbResponseDown, OpTime()); + timeoutTime, Milliseconds(5000), member, hbResponseDown); member = HostAndPort("test1:1234"); getTopoCoord().prepareHeartbeatRequestV1(startupTime + Milliseconds(2), setName, member); getTopoCoord().processHeartbeatResponse( - heartbeatTime, Milliseconds(4000), member, hbResponseGood, OpTime()); + heartbeatTime, Milliseconds(4000), member, hbResponseGood); makeSelfPrimary(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(oplogProgress, startupTime); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(oplogDurable, startupTime); + getTopoCoord().advanceLastCommittedOpTime(lastCommittedOpTime); // Now node 0 is down, node 1 is up, and for node 2 we have no heartbeat data yet. BSONObjBuilder statusBuilder; @@ -1653,9 +1619,6 @@ TEST_F(TopoCoordTest, ReplSetGetStatus) { TopologyCoordinator::ReplSetStatusArgs{ curTime, static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), - oplogProgress, - oplogDurable, - lastCommittedOpTime, readConcernMajorityOpTime, initialSyncStatus}, &statusBuilder, @@ -1771,9 +1734,6 @@ TEST_F(TopoCoordTest, NodeReturnsInvalidReplicaSetConfigInResponseToGetStatusWhe TopologyCoordinator::ReplSetStatusArgs{ curTime, static_cast<unsigned>(durationCount<Seconds>(uptimeSecs)), - oplogProgress, - oplogProgress, - OpTime(), OpTime(), BSONObj()}, &statusBuilder, @@ -1794,12 +1754,8 @@ TEST_F(TopoCoordTest, HeartbeatFrequencyShouldBeHalfElectionTimeoutWhenArbiter) Date_t requestDate = now(); std::pair<ReplSetHeartbeatArgs, Milliseconds> uppingRequest = getTopoCoord().prepareHeartbeatRequest(requestDate, "myset", target); - auto action = - getTopoCoord().processHeartbeatResponse(requestDate, - Milliseconds(0), - target, - makeStatusWith<ReplSetHeartbeatResponse>(), - OpTime(Timestamp(0, 0), 0)); + auto action = getTopoCoord().processHeartbeatResponse( + requestDate, Milliseconds(0), target, makeStatusWith<ReplSetHeartbeatResponse>()); Date_t expected(now() + Milliseconds(2500)); ASSERT_EQUALS(expected, action.getNextHeartbeatStartDate()); } @@ -1826,11 +1782,9 @@ public: } void prepareHeartbeatResponseV1(const ReplSetHeartbeatArgsV1& args, - OpTime lastOpApplied, ReplSetHeartbeatResponse* response, Status* result) { - *result = getTopoCoord().prepareHeartbeatResponseV1( - now()++, args, "rs0", lastOpApplied, lastOpApplied, response); + *result = getTopoCoord().prepareHeartbeatResponseV1(now()++, args, "rs0", response); } }; @@ -1843,7 +1797,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); startCapturingLogMessages(); - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); stopCapturingLogMessages(); ASSERT_EQUALS(ErrorCodes::InconsistentReplicaSetNames, result); ASSERT(result.reason().find("repl set names do not match")) << "Actual string was \"" @@ -1875,7 +1829,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, args.setSenderId(20); ReplSetHeartbeatResponse response; Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_EQUALS(ErrorCodes::InvalidReplicaSetConfig, result); ASSERT(result.reason().find("replica set configuration is invalid or does not include us")) << "Actual string was \"" << result.reason() << '"'; @@ -1890,7 +1844,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, NodeReturnsBadValueWhenAHeartbeatRequestI args.setSenderId(10); ReplSetHeartbeatResponse response; Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_EQUALS(ErrorCodes::BadValue, result); ASSERT(result.reason().find("from member with the same member ID as our self")) << "Actual string was \"" << result.reason() << '"'; @@ -1905,8 +1859,7 @@ TEST_F(TopoCoordTest, SetConfigVersionToNegativeTwoInHeartbeatResponseWhenNoConf args.setSenderId(20); ReplSetHeartbeatResponse response; // prepare response and check the results - Status result = getTopoCoord().prepareHeartbeatResponseV1( - now()++, args, "rs0", OpTime(), OpTime(), &response); + Status result = getTopoCoord().prepareHeartbeatResponseV1(now()++, args, "rs0", &response); ASSERT_OK(result); // this change to true because we can now see a majority, unlike in the previous cases ASSERT_EQUALS("rs0", response.getReplicaSetName()); @@ -1927,7 +1880,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_EQUALS("rs0", response.getReplicaSetName()); ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s); @@ -1947,7 +1900,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_EQUALS("rs0", response.getReplicaSetName()); ASSERT_EQUALS(MemberState::RS_SECONDARY, response.getState().s); @@ -1967,7 +1920,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_TRUE(response.hasConfig()); ASSERT_EQUALS("rs0", response.getReplicaSetName()); @@ -1988,7 +1941,7 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(), &response, &result); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_FALSE(response.hasConfig()); ASSERT_EQUALS("rs0", response.getReplicaSetName()); @@ -2009,7 +1962,10 @@ TEST_F(PrepareHeartbeatResponseV1Test, SetStatePrimaryInHeartbeatResponseWhenPri Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(Timestamp(11, 0), 0), &response, &result); + OpTime lastOpTime(Timestamp(11, 0), 0); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTime, Date_t()); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(lastOpTime, Date_t()); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_FALSE(response.hasConfig()); ASSERT_EQUALS("rs0", response.getReplicaSetName()); @@ -2042,7 +1998,10 @@ TEST_F(PrepareHeartbeatResponseV1Test, Status result(ErrorCodes::InternalError, "prepareHeartbeatResponse didn't set result"); // prepare response and check the results - prepareHeartbeatResponseV1(args, OpTime(Timestamp(100, 0), 0), &response, &result); + OpTime lastOpTime(Timestamp(100, 0), 0); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTime, Date_t()); + getTopoCoord().getMyMemberHeartbeatData()->setLastDurableOpTime(lastOpTime, Date_t()); + prepareHeartbeatResponseV1(args, &response, &result); ASSERT_OK(result); ASSERT_FALSE(response.hasConfig()); ASSERT_EQUALS("rs0", response.getReplicaSetName()); @@ -2087,7 +2046,7 @@ TEST_F(TopoCoordTest, BecomeCandidateWhenReconfigToBeElectableInSingleNodeSet) { << "hself" << "priority" << 0)))); - getTopoCoord().updateConfig(cfg, 0, now()++, OpTime()); + getTopoCoord().updateConfig(cfg, 0, now()++); ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s); ASSERT_FALSE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); @@ -2122,7 +2081,7 @@ TEST_F(TopoCoordTest, NodeDoesNotBecomeCandidateWhenBecomingSecondaryInSingleNod << "priority" << 0)))); - getTopoCoord().updateConfig(cfg, 0, now()++, OpTime()); + getTopoCoord().updateConfig(cfg, 0, now()++); ASSERT_EQUALS(MemberState::RS_STARTUP2, getTopoCoord().getMemberState().s); // despite being the only node, we are unelectable, so we should not become a candidate @@ -2315,8 +2274,7 @@ TEST_F(TopoCoordTest, NodeMaintainsPrimaryStateAcrossReconfigIfNodeRemainsElecta << BSON("_id" << 2 << "host" << "host3:27017"))), 0, - Date_t::fromMillisSinceEpoch(-1), - OpTime(Timestamp(10, 0), 0)); + Date_t::fromMillisSinceEpoch(-1)); ASSERT_TRUE(TopologyCoordinator::Role::leader == getTopoCoord().getRole()); ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s); @@ -2340,8 +2298,7 @@ TEST_F(TopoCoordTest, NodeMaintainsPrimaryStateAcrossReconfigIfNodeRemainsElecta << "rack" << "rack1")))), 0, - Date_t::fromMillisSinceEpoch(-1), - OpTime(Timestamp(10, 0), 0)); + Date_t::fromMillisSinceEpoch(-1)); ASSERT_TRUE(TopologyCoordinator::Role::leader == getTopoCoord().getRole()); ASSERT_EQUALS(MemberState::RS_PRIMARY, getTopoCoord().getMemberState().s); } @@ -2396,7 +2353,7 @@ TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileAwareOfPrimary) { heartbeatFromMember( HostAndPort("h2"), "rs0", MemberState::RS_PRIMARY, OpTime(Timestamp(1, 0), 0)); - ASSERT_NOT_OK(getTopoCoord().checkShouldStandForElection(now()++, OpTime())); + ASSERT_NOT_OK(getTopoCoord().checkShouldStandForElection(now()++)); } TEST_F(TopoCoordTest, ShouldStandForElectionDespiteNotCloseEnoughToLastOptime) { @@ -2414,9 +2371,11 @@ TEST_F(TopoCoordTest, ShouldStandForElectionDespiteNotCloseEnoughToLastOptime) { 0); setSelfMemberState(MemberState::RS_SECONDARY); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(OpTime(Timestamp(100, 0), 0), + Date_t()); heartbeatFromMember( HostAndPort("h2"), "rs0", MemberState::RS_SECONDARY, OpTime(Timestamp(10000, 0), 0)); - ASSERT_OK(getTopoCoord().checkShouldStandForElection(now()++, OpTime(Timestamp(100, 0), 0))); + ASSERT_OK(getTopoCoord().checkShouldStandForElection(now()++)); } TEST_F(TopoCoordTest, VoteForMyselfFailsWhileNotCandidate) { @@ -2455,8 +2414,7 @@ TEST_F(TopoCoordTest, NodeReturnsArbiterWhenGetMemberStateRunsAgainstArbiter) { } TEST_F(TopoCoordTest, ShouldNotStandForElectionWhileRemovedFromTheConfig) { - const auto status = - getTopoCoord().checkShouldStandForElection(now()++, OpTime(Timestamp(10, 0), 0)); + const auto status = getTopoCoord().checkShouldStandForElection(now()++); ASSERT_NOT_OK(status); ASSERT_STRING_CONTAINS(status.reason(), "not a member of a valid replica set config"); } @@ -2488,9 +2446,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -2509,7 +2466,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVotesToTwoDifferentNodesInTheSameTerm) { ReplSetRequestVotesResponse response2; // different candidate same term, should be a problem - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -2544,9 +2501,8 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -2567,7 +2523,7 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response2; - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("", response2.getReason()); ASSERT_TRUE(response2.getVoteGranted()); @@ -2588,7 +2544,7 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response3; - getTopoCoord().processReplSetRequestVotes(args3, &response3, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args3, &response3); ASSERT_EQUALS("", response3.getReason()); ASSERT_TRUE(response3.getVoteGranted()); @@ -2609,7 +2565,7 @@ TEST_F(TopoCoordTest, DryRunVoteRequestShouldNotPreventSubsequentDryRunsForThatT << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response4; - getTopoCoord().processReplSetRequestVotes(args4, &response4, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args4, &response4); ASSERT_EQUALS("already voted for another candidate this term", response4.getReason()); ASSERT_FALSE(response4.getVoteGranted()); } @@ -2644,9 +2600,8 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_TRUE(response.getVoteGranted()); @@ -2667,7 +2622,7 @@ TEST_F(TopoCoordTest, VoteRequestShouldNotPreventDryRunsForThatTerm) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response2; - getTopoCoord().processReplSetRequestVotes(args2, &response2, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args2, &response2); ASSERT_EQUALS("already voted for another candidate this term", response2.getReason()); ASSERT_FALSE(response2.getVoteGranted()); } @@ -2700,9 +2655,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenReplSetNameDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2735,9 +2689,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenConfigVersionDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2774,9 +2727,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenTermIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); ASSERT_EQUALS(2, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -2811,9 +2763,10 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, + Date_t()); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); ASSERT_FALSE(response.getVoteGranted()); } @@ -2849,10 +2802,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -2873,7 +2824,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenReplSetNameDoesNotMatch) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's set name differs from mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -2910,10 +2861,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -2934,7 +2883,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenConfigVersionDoesNotMatch) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's config version differs from mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -2971,10 +2920,8 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -2994,7 +2941,7 @@ TEST_F(TopoCoordTest, NodeDoesNotGrantDryRunVoteWhenTermIsStale) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's term is lower than mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -3031,10 +2978,8 @@ TEST_F(TopoCoordTest, GrantDryRunVoteEvenWhenTermHasBeenSeen) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -3055,7 +3000,7 @@ TEST_F(TopoCoordTest, GrantDryRunVoteEvenWhenTermHasBeenSeen) { << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_TRUE(response.getVoteGranted()); @@ -3092,10 +3037,8 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse responseForRealVote; - OpTime lastAppliedOpTime; - getTopoCoord().processReplSetRequestVotes( - argsForRealVote, &responseForRealVote, lastAppliedOpTime); + getTopoCoord().processReplSetRequestVotes(argsForRealVote, &responseForRealVote); ASSERT_EQUALS("", responseForRealVote.getReason()); ASSERT_TRUE(responseForRealVote.getVoteGranted()); @@ -3115,9 +3058,10 @@ TEST_F(TopoCoordTest, DoNotGrantDryRunVoteWhenOpTimeIsStale) { << "lastCommittedOp" << BSON("ts" << Timestamp(10, 0) << "term" << 0LL))); ReplSetRequestVotesResponse response; - OpTime lastAppliedOpTime2 = {Timestamp(20, 0), 0}; - getTopoCoord().processReplSetRequestVotes(args, &response, lastAppliedOpTime2); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime({Timestamp(20, 0), 0}, + Date_t()); + getTopoCoord().processReplSetRequestVotes(args, &response); ASSERT_EQUALS("candidate's data is staler than mine", response.getReason()); ASSERT_EQUALS(1, response.getTerm()); ASSERT_FALSE(response.getVoteGranted()); @@ -3233,25 +3177,17 @@ TEST_F(HeartbeatResponseTestV1, << "priority" << 0))), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied), now())); @@ -3260,11 +3196,8 @@ TEST_F(HeartbeatResponseTestV1, // set up complete, time for actual check startCapturingLogMessages(); - ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -3275,7 +3208,7 @@ TEST_F(HeartbeatResponseTestV1, NodeReturnsBadValueWhenProcessingPV0ElectionComm Status internalErrorStatus(ErrorCodes::InternalError, "didn't set status"); BSONObjBuilder responseBuilder; Status status = internalErrorStatus; - getTopoCoord().prepareFreshResponse(freshArgs, Date_t(), OpTime(), &responseBuilder, &status); + getTopoCoord().prepareFreshResponse(freshArgs, Date_t(), &responseBuilder, &status); ASSERT_EQUALS(ErrorCodes::BadValue, status); ASSERT_EQUALS("replset: incompatible replset protocol version: 1", status.reason()); ASSERT_TRUE(responseBuilder.obj().isEmpty()); @@ -3283,8 +3216,7 @@ TEST_F(HeartbeatResponseTestV1, NodeReturnsBadValueWhenProcessingPV0ElectionComm BSONObjBuilder electResponseBuilder; ReplicationCoordinator::ReplSetElectArgs electArgs; status = internalErrorStatus; - getTopoCoord().prepareElectResponse( - electArgs, Date_t(), OpTime(), &electResponseBuilder, &status); + getTopoCoord().prepareElectResponse(electArgs, Date_t(), &electResponseBuilder, &status); ASSERT_EQUALS(ErrorCodes::BadValue, status); ASSERT_EQUALS("replset: incompatible replset protocol version: 1", status.reason()); ASSERT_TRUE(electResponseBuilder.obj().isEmpty()); @@ -3297,76 +3229,51 @@ TEST_F(HeartbeatResponseTestV1, OpTime election = OpTime(); OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // Show we like host2 while it is primary. ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied, 1), now())); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, - makeReplSetMetadata(lastOpTimeApplied, 1), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied, 1), boost::none, now())); // Show that we also like host2 while it has a sync source. - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_FALSE( getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied, 2, 2), now())); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, - makeReplSetMetadata(lastOpTimeApplied, 2, 2), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied, 2, 2), boost::none, now())); // Show that we do not like it when it is not PRIMARY and lacks a sync source and lacks progress // beyond our own. - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied), now())); - ASSERT(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now())); // Sometimes the heartbeat is stale and the metadata says it's the primary. Trust the metadata. ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(), makeOplogQueryMetadata( lastOpTimeApplied, 1 /* host2 is primary */, -1 /* no sync source */), now())); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(lastOpTimeApplied, 1 /* host2 is primary */, -1 /* no sync source */), boost::none, now())); @@ -3377,21 +3284,15 @@ TEST_F(HeartbeatResponseTestV1, "rs0", MemberState::RS_SECONDARY, election, - newerThanLastOpTimeApplied, newerThanLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_FALSE( getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, makeReplSetMetadata(), makeOplogQueryMetadata(newerThanLastOpTimeApplied), now())); - ASSERT_FALSE( - getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - lastOpTimeApplied, - makeReplSetMetadata(newerThanLastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(newerThanLastOpTimeApplied), boost::none, now())); } TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberIsDown) { @@ -3399,40 +3300,38 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberIsDown // "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind // "host3", since "host3" is down OpTime election = OpTime(); + // Our last op time applied must be behind host2, or we'll hit the case where we change + // sync sources due to the sync source being behind, without a sync source, and not primary. OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0); + OpTime syncSourceOpTime = OpTime(Timestamp(400, 1), 0); // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, syncSourceOpTime); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); + // while the host is up, we should want to change to its sync source + ASSERT(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), + makeReplSetMetadata(), + makeOplogQueryMetadata(syncSourceOpTime), + now())); + // set up complete, time for actual check - nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), + makeOplogQueryMetadata(syncSourceOpTime), now())); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); } TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhileFresherMemberIsBlackListed) { @@ -3441,69 +3340,53 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhileFresherMemberIsBla // "host3", since "host3" is blacklisted // Then, confirm that unblacklisting only works if time has passed the blacklist time. OpTime election = OpTime(); + // Our last op time applied must be behind host2, or we'll hit the case where we change + // sync sources due to the sync source being behind, without a sync source, and not primary. OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0); + OpTime syncSourceOpTime = OpTime(Timestamp(400, 1), 0); // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, syncSourceOpTime); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); getTopoCoord().blacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100)); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), - now())); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, + makeOplogQueryMetadata(syncSourceOpTime), now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); // unblacklist with too early a time (node should remained blacklisted) getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(90)); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), - now())); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, + makeOplogQueryMetadata(syncSourceOpTime), now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); // unblacklist and it should succeed getTopoCoord().unblacklistSyncSource(HostAndPort("host3"), now() + Milliseconds(100)); startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), + makeOplogQueryMetadata(syncSourceOpTime), now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); startCapturingLogMessages(); - ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -3517,26 +3400,18 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceIfNodeIsFreshByHeartbea // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied), now())); @@ -3544,11 +3419,8 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceIfNodeIsFreshByHeartbea ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); startCapturingLogMessages(); - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); } @@ -3562,27 +3434,19 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceIfNodeIsStaleByHeartbea // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_FALSE( getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(fresherLastOpTimeApplied), now())); @@ -3590,12 +3454,8 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceIfNodeIsStaleByHeartbea ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); // set up complete, time for actual check startCapturingLogMessages(); - ASSERT_FALSE( - getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(fresherLastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(fresherLastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(0, countLogLinesContaining("Choosing new sync source")); } @@ -3608,26 +3468,18 @@ TEST_F(HeartbeatResponseTestV1, ShouldChangeSyncSourceWhenFresherMemberExists) { // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check startCapturingLogMessages(); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(lastOpTimeApplied), now())); @@ -3636,11 +3488,8 @@ TEST_F(HeartbeatResponseTestV1, ShouldChangeSyncSourceWhenFresherMemberExists) { // set up complete, time for actual check startCapturingLogMessages(); - ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(lastOpTimeApplied), boost::none, now())); stopCapturingLogMessages(); ASSERT_EQUALS(1, countLogLinesContaining("Choosing new sync source")); } @@ -3649,7 +3498,7 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenMemberHasYetToHeart // In this test, the TopologyCoordinator should not tell us to change sync sources away from // "host2" since we do not use the member's heartbeatdata in pv1. ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host2"), OpTime(), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); + HostAndPort("host2"), makeReplSetMetadata(), makeOplogQueryMetadata(), now())); } TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenMemberNotInConfig) { @@ -3657,7 +3506,7 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenMemberNotInConfig) // "host4" since "host4" is absent from the config of version 10. ReplSetMetadata replMetadata(0, OpTime(), OpTime(), 10, OID(), -1, -1); ASSERT_TRUE(getTopoCoord().shouldChangeSyncSource( - HostAndPort("host4"), OpTime(), replMetadata, makeOplogQueryMetadata(), now())); + HostAndPort("host4"), replMetadata, makeOplogQueryMetadata(), now())); } // TODO(dannenberg) figure out what this is trying to test.. @@ -3668,20 +3517,13 @@ TEST_F(HeartbeatResponseTestV1, ReconfigNodeRemovedBetweenHeartbeatRequestAndRep // all three members up and secondaries setSelfMemberState(MemberState::RS_SECONDARY); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // now request from host3 and receive after host2 has been removed via reconfig @@ -3706,7 +3548,7 @@ TEST_F(HeartbeatResponseTestV1, ReconfigNodeRemovedBetweenHeartbeatRequestAndRep hb.setElectionTime(election.getTimestamp()); StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - now()++, Milliseconds(0), HostAndPort("host3"), hbResponse, lastOpTimeApplied); + now()++, Milliseconds(0), HostAndPort("host3"), hbResponse); // primary should not be set and we should perform NoAction in response ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); @@ -3721,20 +3563,13 @@ TEST_F(HeartbeatResponseTestV1, ReconfigBetweenHeartbeatRequestAndRepsonse) { // all three members up and secondaries setSelfMemberState(MemberState::RS_SECONDARY); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // now request from host3 and receive after host2 has been removed via reconfig @@ -3758,8 +3593,9 @@ TEST_F(HeartbeatResponseTestV1, ReconfigBetweenHeartbeatRequestAndRepsonse) { hb.setDurableOpTime(lastOpTimeApplied); hb.setElectionTime(election.getTimestamp()); StatusWith<ReplSetHeartbeatResponse> hbResponse = StatusWith<ReplSetHeartbeatResponse>(hb); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - now()++, Milliseconds(0), HostAndPort("host3"), hbResponse, lastOpTimeApplied); + now()++, Milliseconds(0), HostAndPort("host3"), hbResponse); // now primary should be host3, index 1, and we should perform NoAction in response ASSERT_EQUALS(1, getCurrentPrimaryIndex()); @@ -3771,12 +3607,9 @@ TEST_F(HeartbeatResponseTestV1, NodeDoesNotUpdateHeartbeatDataIfNodeIsAbsentFrom OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host9"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host9"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -3798,10 +3631,9 @@ TEST_F(HeartbeatResponseTestV1, RelinquishPrimaryWhenMajorityOfVotersIsNoLongerV // Lose that awareness, but we are not going to step down, because stepdown only // depends on liveness. - HeartbeatResponseAction nextAction = - receiveDownHeartbeat(HostAndPort("host2"), "rs0", OpTime(Timestamp(100, 0), 0)); + HeartbeatResponseAction nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", OpTime(Timestamp(100, 0), 0)); + nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::leader == getTopoCoord().getRole()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); @@ -3835,12 +3667,9 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(HeartbeatResponseAction::PriorityTakeover, nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); } @@ -3875,22 +3704,15 @@ TEST_F(HeartbeatResponseTestV1, UpdateHeartbeatDataTermPreventsPriorityTakeover) // Host 2 is the current primary in term 1. getTopoCoord().updateTerm(1, now()); ASSERT_EQUALS(getTopoCoord().getTerm(), 1); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(HeartbeatResponseAction::PriorityTakeover, nextAction.getAction()); ASSERT_EQUALS(2, getCurrentPrimaryIndex()); // Heartbeat from a secondary node shouldn't schedule a priority takeover. - nextAction = receiveUpHeartbeat(HostAndPort("host1"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host1"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(2, getCurrentPrimaryIndex()); @@ -3901,12 +3723,8 @@ TEST_F(HeartbeatResponseTestV1, UpdateHeartbeatDataTermPreventsPriorityTakeover) // This heartbeat shouldn't schedule priority takeover, because the current primary // host 1 is not in my term. - nextAction = receiveUpHeartbeat(HostAndPort("host1"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host1"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, nextAction.getAction()); ASSERT_EQUALS(2, getCurrentPrimaryIndex()); } @@ -3960,39 +3778,32 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); // Make sure all non-voting nodes are down, that way we do not have a majority of nodes // but do have a majority of votes since one of two voting members is up and so are we. - nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host3"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host4"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host4"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host5"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host5"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host6"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host6"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host7"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host7"), "rs0", MemberState::RS_SECONDARY, election, lastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), lastOpTimeApplied, false)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), false)); ASSERT_TRUE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); } @@ -4003,28 +3814,21 @@ TEST_F(HeartbeatResponseTestV1, ScheduleElectionWhenPrimaryIsMarkedDownAndWeAreE OpTime lastOpTimeApplied = OpTime(Timestamp(399, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // We are electable now. - ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), lastOpTimeApplied, false)); + ASSERT_OK(getTopoCoord().becomeCandidateIfElectable(now(), false)); ASSERT_TRUE(TopologyCoordinator::Role::candidate == getTopoCoord().getRole()); } @@ -4050,25 +3854,18 @@ TEST_F(HeartbeatResponseTestV1, OpTime election = OpTime(Timestamp(400, 0), 0); OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4082,28 +3879,21 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); // Freeze node to set stepdown wait. BSONObjBuilder response; getTopoCoord().prepareFreezeResponse(now()++, 20, &response); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4117,16 +3907,13 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4140,24 +3927,17 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4188,25 +3968,18 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4220,16 +3993,13 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(300, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4244,24 +4014,16 @@ TEST_F(HeartbeatResponseTestV1, NodeDoesNotStepDownSelfWhenRemoteNodeWasElectedM OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); - receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - election, - lastOpTimeApplied); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::leader == getTopoCoord().getRole()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); // If the other PRIMARY falls down, this node should set its primaryIndex to itself. - nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0", lastOpTimeApplied); + nextAction = receiveDownHeartbeat(HostAndPort("host2"), "rs0"); ASSERT_TRUE(TopologyCoordinator::Role::leader == getTopoCoord().getRole()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); @@ -4296,12 +4058,13 @@ TEST_F(HeartbeatResponseTestV1, OpTime stale = OpTime(); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election, election); + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); - nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, stale, election); + nextAction = + receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, stale); ASSERT_NO_ACTION(nextAction.getAction()); } @@ -4335,8 +4098,9 @@ TEST_F(HeartbeatResponseTestV1, makeSelfPrimary(election.getTimestamp()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, staleTime, election); + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, staleTime); ASSERT_NO_ACTION(nextAction.getAction()); } @@ -4370,8 +4134,9 @@ TEST_F(HeartbeatResponseTestV1, makeSelfPrimary(election.getTimestamp()); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(election, Date_t()); HeartbeatResponseAction nextAction = receiveUpHeartbeat( - HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election, election); + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, election); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_EQUALS(-1, nextAction.getPrimaryConfigIndex()); } @@ -4406,20 +4171,16 @@ TEST_F(HeartbeatResponseTestV1, OpTime slightlyLessFreshLastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, lastOpTimeApplied); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); nextAction = receiveUpHeartbeat(HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, - slightlyLessFreshLastOpTimeApplied, - lastOpTimeApplied); + slightlyLessFreshLastOpTimeApplied); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, nextAction.getAction()); } @@ -4431,12 +4192,9 @@ TEST_F(HeartbeatResponseTestV1, NodeDoesNotStepDownSelfWhenRemoteNodeWasElectedL OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(0, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); } @@ -4448,21 +4206,14 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election2, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election); // Second primary does not change primary index. ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); @@ -4476,21 +4227,14 @@ TEST_F(HeartbeatResponseTestV1, OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_PRIMARY, - election2, - election, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_PRIMARY, election2, election); // Second primary does not change primary index. ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); @@ -4502,12 +4246,9 @@ TEST_F(HeartbeatResponseTestV1, UpdatePrimaryIndexWhenAHeartbeatMakesNodeAwareOf OpTime lastOpTimeApplied = OpTime(Timestamp(3, 0), 0); ASSERT_EQUALS(-1, getCurrentPrimaryIndex()); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_PRIMARY, - election, - election, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_PRIMARY, election, election); ASSERT_EQUALS(1, getCurrentPrimaryIndex()); ASSERT_NO_ACTION(nextAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4532,8 +4273,7 @@ TEST_F(HeartbeatResponseTestV1, NodeDoesNotRetryHeartbeatIfTheFirstFailureTakesT // no retry allowed. Milliseconds(4990), // Spent 4.99 of the 5 seconds in the network. target, - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4548,7 +4288,10 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberDoesNo // from "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind // "host3", since "host3" does not build indexes OpTime election = OpTime(); - OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0); + // Our last op time applied must be behind host2, or we'll hit the case where we change + // sync sources due to the sync source being behind, without a sync source, and not primary. + OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0); + OpTime syncSourceOpTime = OpTime(Timestamp(400, 1), 0); // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); @@ -4570,33 +4313,22 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberDoesNo << "protocolVersion" << 1), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, syncSourceOpTime); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_SECONDARY, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_SECONDARY, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), + makeOplogQueryMetadata(syncSourceOpTime), now())); // set up complete, time for actual check - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); } TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberIsNotReadable) { @@ -4604,39 +4336,31 @@ TEST_F(HeartbeatResponseTestV1, ShouldNotChangeSyncSourceWhenFresherMemberIsNotR // from "host2" and to "host3" despite "host2" being more than maxSyncSourceLagSecs(30) behind // "host3", since "host3" is in a non-readable mode (RS_ROLLBACK) OpTime election = OpTime(); - OpTime lastOpTimeApplied = OpTime(Timestamp(4, 0), 0); + // Our last op time applied must be behind host2, or we'll hit the case where we change + // sync sources due to the sync source being behind, without a sync source, and not primary. + OpTime lastOpTimeApplied = OpTime(Timestamp(400, 0), 0); + OpTime syncSourceOpTime = OpTime(Timestamp(400, 1), 0); // ahead by more than maxSyncSourceLagSecs (30) OpTime fresherLastOpTimeApplied = OpTime(Timestamp(3005, 0), 0); - HeartbeatResponseAction nextAction = receiveUpHeartbeat(HostAndPort("host2"), - "rs0", - MemberState::RS_SECONDARY, - election, - lastOpTimeApplied, - lastOpTimeApplied); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); + HeartbeatResponseAction nextAction = receiveUpHeartbeat( + HostAndPort("host2"), "rs0", MemberState::RS_SECONDARY, election, syncSourceOpTime); ASSERT_NO_ACTION(nextAction.getAction()); - nextAction = receiveUpHeartbeat(HostAndPort("host3"), - "rs0", - MemberState::RS_ROLLBACK, - election, - fresherLastOpTimeApplied, - lastOpTimeApplied); + nextAction = receiveUpHeartbeat( + HostAndPort("host3"), "rs0", MemberState::RS_ROLLBACK, election, fresherLastOpTimeApplied); ASSERT_NO_ACTION(nextAction.getAction()); // set up complete, time for actual check ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), makeReplSetMetadata(), - makeOplogQueryMetadata(lastOpTimeApplied), + makeOplogQueryMetadata(syncSourceOpTime), now())); // set up complete, time for actual check - ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource(HostAndPort("host2"), - OpTime(), - makeReplSetMetadata(lastOpTimeApplied), - boost::none, - now())); + ASSERT_FALSE(getTopoCoord().shouldChangeSyncSource( + HostAndPort("host2"), makeReplSetMetadata(syncSourceOpTime), boost::none, now())); } class HeartbeatResponseTestOneRetryV1 : public HeartbeatResponseTestV1 { @@ -4650,11 +4374,7 @@ public: std::pair<ReplSetHeartbeatArgsV1, Milliseconds> uppingRequest = getTopoCoord().prepareHeartbeatRequestV1(_upRequestDate, "rs0", _target); HeartbeatResponseAction upAction = getTopoCoord().processHeartbeatResponse( - _upRequestDate, - Milliseconds(0), - _target, - makeStatusWith<ReplSetHeartbeatResponse>(), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + _upRequestDate, Milliseconds(0), _target, makeStatusWith<ReplSetHeartbeatResponse>()); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, upAction.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4673,8 +4393,8 @@ public: _firstRequestDate + Seconds(4), // 4 seconds elapsed, retry allowed. Milliseconds(3990), // Spent 3.99 of the 4 seconds in the network. _target, - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>( + ErrorCodes::ExceededTimeLimit, "Took too long")); // We've never applied anything. ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4691,13 +4411,8 @@ public: BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{_firstRequestDate + Milliseconds(4000), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + _firstRequestDate + Milliseconds(4000), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -4736,8 +4451,8 @@ TEST_F(HeartbeatResponseTestOneRetryV1, // no retry allowed. Milliseconds(1000), // Spent 1 of the 1.01 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, "Took too long"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::ExceededTimeLimit, + "Took too long")); // We've never applied anything. ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4757,8 +4472,7 @@ public: // could retry. Milliseconds(400), // Spent 0.4 of the 0.5 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // Because the first retry failed without timing out, we expect to retry immediately. @@ -4775,13 +4489,8 @@ public: BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Seconds(4), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Seconds(4), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -4810,8 +4519,7 @@ TEST_F(HeartbeatResponseTestTwoRetriesV1, NodeDoesNotRetryHeartbeatsAfterFailing // could still retry. Milliseconds(100), // Spent 0.1 of the 0.3 seconds in the network. target(), - StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?"), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(ErrorCodes::NodeNotFound, "Bad DNS?")); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); // Because this is the second retry, rather than retry again, we expect to wait for half @@ -4824,13 +4532,8 @@ TEST_F(HeartbeatResponseTestTwoRetriesV1, NodeDoesNotRetryHeartbeatsAfterFailing BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Milliseconds(4900), - 10, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Milliseconds(4900), 10, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -4853,12 +4556,11 @@ TEST_F(HeartbeatResponseTestTwoRetriesV1, HeartbeatThreeNonconsecutiveFailures) response.setConfigVersion(5); // successful response (third response due to the two failures in setUp()) - HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( - firstRequestDate() + Milliseconds(4500), - Milliseconds(400), - target(), - StatusWith<ReplSetHeartbeatResponse>(response), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + HeartbeatResponseAction action = + getTopoCoord().processHeartbeatResponse(firstRequestDate() + Milliseconds(4500), + Milliseconds(400), + target(), + StatusWith<ReplSetHeartbeatResponse>(response)); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4875,8 +4577,7 @@ TEST_F(HeartbeatResponseTestTwoRetriesV1, HeartbeatThreeNonconsecutiveFailures) firstRequestDate() + Milliseconds(7100), Milliseconds(400), target(), - StatusWith<ReplSetHeartbeatResponse>(Status{ErrorCodes::HostUnreachable, ""}), - OpTime(Timestamp(0, 0), 0)); // We've never applied anything. + StatusWith<ReplSetHeartbeatResponse>(Status{ErrorCodes::HostUnreachable, ""})); ASSERT_EQUALS(HeartbeatResponseAction::NoAction, action.getAction()); ASSERT_TRUE(TopologyCoordinator::Role::follower == getTopoCoord().getRole()); @@ -4885,13 +4586,8 @@ TEST_F(HeartbeatResponseTestTwoRetriesV1, HeartbeatThreeNonconsecutiveFailures) BSONObjBuilder statusBuilder; Status resultStatus(ErrorCodes::InternalError, "prepareStatusResponse didn't set result"); getTopoCoord().prepareStatusResponse( - TopologyCoordinator::ReplSetStatusArgs{firstRequestDate() + Milliseconds(7000), - 600, - OpTime(Timestamp(100, 0), 0), - OpTime(Timestamp(100, 0), 0), - OpTime(), - OpTime(), - BSONObj()}, + TopologyCoordinator::ReplSetStatusArgs{ + firstRequestDate() + Milliseconds(7000), 600, OpTime(), BSONObj()}, &statusBuilder, &resultStatus); ASSERT_OK(resultStatus); @@ -4932,12 +4628,12 @@ TEST_F(HeartbeatResponseHighVerbosityTestV1, UpdateHeartbeatDataOldConfig) { believesWeAreDownResponse.setElectable(true); believesWeAreDownResponse.noteStateDisagreement(); startCapturingLogMessages(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down")); @@ -4979,12 +4675,12 @@ TEST_F(HeartbeatResponseHighVerbosityTestV1, UpdateHeartbeatDataSameConfig) { sameConfigResponse.setConfigVersion(2); sameConfigResponse.setConfig(originalConfig); startCapturingLogMessages(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(sameConfigResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(sameConfigResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, @@ -5007,12 +4703,12 @@ TEST_F(HeartbeatResponseHighVerbosityTestV1, memberMissingResponse.setElectable(true); memberMissingResponse.noteStateDisagreement(); startCapturingLogMessages(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host5"), - StatusWith<ReplSetHeartbeatResponse>(memberMissingResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(memberMissingResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("Could not find host5:27017 in current config")); @@ -5033,12 +4729,12 @@ TEST_F(HeartbeatResponseHighVerbosityTestV1, believesWeAreDownResponse.setElectable(true); believesWeAreDownResponse.noteStateDisagreement(); startCapturingLogMessages(); + getTopoCoord().getMyMemberHeartbeatData()->setLastAppliedOpTime(lastOpTimeApplied, Date_t()); HeartbeatResponseAction action = getTopoCoord().processHeartbeatResponse( now()++, // Time is left. Milliseconds(400), // Spent 0.4 of the 0.5 second in the network. HostAndPort("host2"), - StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse), - lastOpTimeApplied); + StatusWith<ReplSetHeartbeatResponse>(believesWeAreDownResponse)); stopCapturingLogMessages(); ASSERT_NO_ACTION(action.getAction()); ASSERT_EQUALS(1, countLogLinesContaining("host2:27017 thinks that we are down")); |