SERVER-18280 ReplicaSetMonitor should use electionId to avoid talking to old primaries

4 files changed, 214 insertions, 4 deletions

diff --git a/src/mongo/client/replica_set_monitor.cpp b/src/mongo/client/replica_set_monitor.cpp
index 9cd07cc0716..0b92969acc5 100644
--- a/src/mongo/client/replica_set_monitor.cpp
+++ b/src/mongo/client/replica_set_monitor.cpp
@@ -545,8 +545,16 @@ namespace {
             return;
         }
 
-        if (reply.isMaster)
-            receivedIsMasterFromMaster(reply);
+        if (reply.isMaster) {
+            const bool stalePrimary = !receivedIsMasterFromMaster(reply);
+            if (stalePrimary) {
+                log() << "node " << from << " believes it is primary, but its election id of "
+                      << reply.electionId << " is older than the most recent election id"
+                      << " for this set, " << _set->maxElectionId;
+                failedHost(from);
+                return;
+            }
+        }
 
         if (_scan->foundUpMaster) {
             // We only update a Node if a master has confirmed it is in the set.
@@ -622,9 +630,16 @@ namespace {
         return scan;
     }
 
-    void Refresher::receivedIsMasterFromMaster(const IsMasterReply& reply) {
+    bool Refresher::receivedIsMasterFromMaster(const IsMasterReply& reply) {
         invariant(reply.isMaster);
 
+        if (reply.electionId.isSet()) {
+            if (_set->maxElectionId.isSet() && _set->maxElectionId.compare(reply.electionId) > 0) {
+                return false;
+            }
+            _set->maxElectionId = reply.electionId;
+        }
+
         // Mark all nodes as not master. We will mark ourself as master before releasing the lock.
         // NOTE: we use a "last-wins" policy if multiple hosts claim to be master.
         for (size_t i = 0; i < _set->nodes.size(); i++) {
@@ -694,6 +709,8 @@ namespace {
 
         _scan->foundUpMaster = true;
         _set->lastSeenMaster = reply.host;
+
+        return true;
     }
 
     void Refresher::receivedIsMasterBeforeFoundMaster(const IsMasterReply& reply) {
@@ -787,6 +804,10 @@ namespace {
             // hidden nodes can't be master, even if they claim to be.
             isMaster = !hidden && raw["ismaster"].trueValue();
 
+            if (isMaster && raw.hasField("electionId")) {
+                electionId = raw["electionId"].OID();
+            }
+
             const string primaryString = raw["primary"].str();
             primary = primaryString.empty() ? HostAndPort() : HostAndPort(primaryString);
 
diff --git a/src/mongo/client/replica_set_monitor.h b/src/mongo/client/replica_set_monitor.h
index 0b06520f5b4..64b7a89274e 100644
--- a/src/mongo/client/replica_set_monitor.h
+++ b/src/mongo/client/replica_set_monitor.h
@@ -306,12 +306,17 @@ namespace mongo {
         static ScanStatePtr startNewScan(const SetState* set);
 
     private:
+
         /**
+         * First, checks that the "reply" is not from a stale primary by
+         * comparing the electionId of "reply" to the maxElectionId recorded by the SetState.
+         * Returns true if "reply" belongs to a non-stale primary.
+         *
          * Updates _set and _scan based on set-membership information from a master.
          * Applies _scan->unconfirmedReplies to confirmed nodes.
          * Does not update this host's node in _set->nodes.
          */
-        void receivedIsMasterFromMaster(const IsMasterReply& reply);
+        bool receivedIsMasterFromMaster(const IsMasterReply& reply);
 
         /**
          * Adjusts the _scan work queue based on information from this host.
diff --git a/src/mongo/client/replica_set_monitor_internal.h b/src/mongo/client/replica_set_monitor_internal.h
index 7f434c302e7..ba19ad1eb82 100644
--- a/src/mongo/client/replica_set_monitor_internal.h
+++ b/src/mongo/client/replica_set_monitor_internal.h
@@ -69,6 +69,7 @@ namespace mongo {
         bool isMaster;
         bool secondary;
         bool hidden;
+        OID electionId; // Set if this isMaster reply is from the primary
         HostAndPort primary; // empty if not present
         std::set<HostAndPort> normalHosts; // both "hosts" and "passives"
         BSONObj tags;
@@ -177,6 +178,7 @@ namespace mongo {
         const std::string name; // safe to read outside lock since it is const
         int consecutiveFailedScans;
         std::set<HostAndPort> seedNodes; // updated whenever a master reports set membership changes
+        OID maxElectionId; // largest election id observed by this ReplicaSetMonitor
         HostAndPort lastSeenMaster; // empty if we have never seen a master. can be same as current
         Nodes nodes; // maintained sorted and unique by host
         ScanStatePtr currentScan; // NULL if no scan in progress
diff --git a/src/mongo/client/replica_set_monitor_test.cpp b/src/mongo/client/replica_set_monitor_test.cpp
index b6ebf7ab23c..736e59f3f57 100644
--- a/src/mongo/client/replica_set_monitor_test.cpp
+++ b/src/mongo/client/replica_set_monitor_test.cpp
@@ -782,4 +782,186 @@ namespace {
         }
     }
 
+    // Newly elected primary with electionId >= maximum electionId seen by the Refresher
+    TEST(ReplicaSetMonitorTests, NewPrimaryWithMaxElectionId) {
+        SetStatePtr state = boost::make_shared<SetState>("name", basicSeedsSet);
+        Refresher refresher(state);
+
+        set<HostAndPort> seen;
+
+        // get all hosts to contact first
+        for (size_t i = 0; i != basicSeeds.size(); ++i) {
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST);
+            ASSERT(basicSeedsSet.count(ns.host));
+            ASSERT(!seen.count(ns.host));
+            seen.insert(ns.host);
+        }
+
+        const ReadPreferenceSetting primaryOnly(ReadPreference::PrimaryOnly, TagSet());
+
+        // mock all replies
+        for (size_t i = 0; i != basicSeeds.size(); ++i) {
+            // All hosts to talk to are already dispatched, but no reply has been received
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::WAIT);
+            ASSERT(ns.host.empty());
+
+            refresher.receivedIsMaster(basicSeeds[i],
+                                       -1,
+                                       BSON("setName" << "name"
+                                            << "ismaster" << true
+                                            << "secondary" << false
+                                            << "hosts" << BSON_ARRAY("a" << "b" << "c")
+                                            << "electionId" << OID::gen()
+                                            << "ok" << true));
+
+            // Ensure the set primary is the host we just got a reply from
+            HostAndPort currentPrimary = state->getMatchingHost(primaryOnly);
+            ASSERT_EQUALS(currentPrimary.host(), basicSeeds[i].host());
+            ASSERT_EQUALS(state->nodes.size(), basicSeeds.size());
+
+            // Check the state of each individual node
+            for (size_t j = 0; j != basicSeeds.size(); ++j) {
+                Node* node = state->findNode(basicSeeds[j]);
+                ASSERT(node);
+                ASSERT_EQUALS(node->host.toString(), basicSeeds[j].toString());
+                ASSERT_EQUALS(node->isUp, j <= i);
+                ASSERT_EQUALS(node->isMaster, j == i);
+                ASSERT(node->tags.isEmpty());
+            }
+        }
+
+        // Now all hosts have returned data
+        NextStep ns = refresher.getNextStep();
+        ASSERT_EQUALS(ns.step, NextStep::DONE);
+        ASSERT(ns.host.empty());
+    }
+
+    // Ignore electionId of secondaries
+    TEST(ReplicaSetMonitorTests, IgnoreElectionIdFromSecondaries) {
+        SetStatePtr state = boost::make_shared<SetState>("name", basicSeedsSet);
+        Refresher refresher(state);
+
+        set<HostAndPort> seen;
+
+        const OID primaryElectionId = OID::gen();
+
+        // mock all replies
+        for (size_t i = 0; i != basicSeeds.size(); ++i) {
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST);
+            ASSERT(basicSeedsSet.count(ns.host));
+            ASSERT(!seen.count(ns.host));
+            seen.insert(ns.host);
+
+            // mock a reply
+            const bool primary = ns.host.host() == "a";
+            refresher.receivedIsMaster(ns.host,
+                                       -1,
+                                       BSON("setName" << "name"
+                                            << "ismaster" << primary
+                                            << "secondary" << !primary
+                                            << "electionId" << (primary ?
+                                                                primaryElectionId : OID::gen())
+                                            << "hosts" << BSON_ARRAY("a" << "b" << "c")
+                                            << "ok" << true));
+        }
+
+        // check that the SetState's maxElectionId == primary's electionId
+        ASSERT_EQUALS(state->maxElectionId, primaryElectionId);
+
+        // Now all hosts have returned data
+        NextStep ns = refresher.getNextStep();
+        ASSERT_EQUALS(ns.step, NextStep::DONE);
+        ASSERT(ns.host.empty());
+    }
+
+    // Stale Primary with obsolete electionId
+    TEST(ReplicaSetMonitorTests, StalePrimaryWithObsoleteElectionId) {
+        SetStatePtr state = boost::make_shared<SetState>("name", basicSeedsSet);
+        Refresher refresher(state);
+
+        const OID firstElectionId = OID::gen();
+        const OID secondElectionId = OID::gen();
+
+        set<HostAndPort> seen;
+
+        // contact first host claiming to be primary with greater electionId
+        {
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST);
+            ASSERT(basicSeedsSet.count(ns.host));
+            ASSERT(!seen.count(ns.host));
+            seen.insert(ns.host);
+
+            refresher.receivedIsMaster(ns.host,
+                                       -1,
+                                       BSON("setName" << "name"
+                                            << "ismaster" << true
+                                            << "secondary" << false
+                                            << "electionId" << secondElectionId
+                                            << "hosts" << BSON_ARRAY("a" << "b" << "c")
+                                            << "ok" << true));
+
+            Node* node = state->findNode(ns.host);
+            ASSERT(node);
+            ASSERT_TRUE(node->isMaster);
+            ASSERT_EQUALS(state->maxElectionId, secondElectionId);
+        }
+
+        // contact second host claiming to be primary with smaller electionId
+        {
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST);
+            ASSERT(basicSeedsSet.count(ns.host));
+            ASSERT(!seen.count(ns.host));
+            seen.insert(ns.host);
+
+            refresher.receivedIsMaster(ns.host,
+                                       -1,
+                                       BSON("setName" << "name"
+                                            << "ismaster" << true
+                                            << "secondary" << false
+                                            << "electionId" << firstElectionId
+                                            << "hosts" << BSON_ARRAY("a" << "b" << "c")
+                                            << "ok" << true));
+
+            Node* node = state->findNode(ns.host);
+            ASSERT(node);
+            // The SetState shouldn't see this host as master
+            ASSERT_FALSE(node->isMaster);
+            // the max electionId should remain the same
+            ASSERT_EQUALS(state->maxElectionId, secondElectionId);
+        }
+
+        // third host is a secondary
+        {
+            NextStep ns = refresher.getNextStep();
+            ASSERT_EQUALS(ns.step, NextStep::CONTACT_HOST);
+            ASSERT(basicSeedsSet.count(ns.host));
+            ASSERT(!seen.count(ns.host));
+            seen.insert(ns.host);
+
+            refresher.receivedIsMaster(ns.host,
+                                       -1,
+                                       BSON("setName" << "name"
+                                            << "ismaster" << false
+                                            << "secondary" << true
+                                            << "hosts" << BSON_ARRAY("a" << "b" << "c")
+                                            << "ok" << true));
+
+            Node* node = state->findNode(ns.host);
+            ASSERT(node);
+            ASSERT_FALSE(node->isMaster);
+            // the max electionId should remain the same
+            ASSERT_EQUALS(state->maxElectionId, secondElectionId);
+        }
+
+        // Now all hosts have returned data
+        NextStep ns = refresher.getNextStep();
+        ASSERT_EQUALS(ns.step, NextStep::DONE);
+        ASSERT(ns.host.empty());
+    }
+
 } // namespace