SERVER-24676 implement support for maxStalenessMS in ReplicaSetMonitor

1 files changed, 428 insertions, 0 deletions

diff --git a/src/mongo/client/replica_set_monitor_test.cpp b/src/mongo/client/replica_set_monitor_test.cpp
index a27c3e00692..28b6980c523 100644
--- a/src/mongo/client/replica_set_monitor_test.cpp
+++ b/src/mongo/client/replica_set_monitor_test.cpp
@@ -1308,4 +1308,432 @@ TEST(ReplicaSetMonitorTests, TwoPrimaries2ndHasOlderConfigVersion) {
     ASSERT_EQUALS(state->configVersion, 2);
 }
 
+/**
+ * Success finding node matching maxStalenessMS parameter
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSMatch) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(
+        ReadPreference::SecondaryOnly, TagSet(), Milliseconds(100000));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(10);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        bool nonStale = ns.host.host() == "c";
+        nonStale |= primary;
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate" << (nonStale ? lastWriteDateNonStale
+                                                                             : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    // make sure all secondaries are in the scan
+    ASSERT(state->findNode(HostAndPort("b")));
+    ASSERT(state->findNode(HostAndPort("c")));
+
+    HostAndPort nonStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(nonStale.host(), "c");
+}
+
+/**
+ * Fail matching maxStalenessMS parameter ( all secondary nodes are stale)
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSNoMatch) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate" << (primary ? lastWriteDateNonStale
+                                                                            : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    // make sure all secondaries are in the scan
+    ASSERT(state->findNode(HostAndPort("b")));
+    ASSERT(state->findNode(HostAndPort("c")));
+
+    HostAndPort notFound = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notFound.host(), "");
+}
+
+/**
+ * Success matching maxStalenessMS parameter when there is no primary node.
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSNoPrimaryMatch) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool isNonStale = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << false
+                                        << "secondary"
+                                        << true
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate"
+                                                << (isNonStale ? lastWriteDateNonStale
+                                                               : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    // make sure all secondaries are in the scan
+    ASSERT(state->findNode(HostAndPort("a")));
+    ASSERT(state->findNode(HostAndPort("b")));
+    ASSERT(state->findNode(HostAndPort("c")));
+
+    HostAndPort notStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notStale.host(), "a");
+}
+
+
+/**
+ * Fail matching maxStalenessMS parameter when all nodes are failed
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSAllFailed) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool isNonStale = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << false
+                                        << "secondary"
+                                        << true
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate"
+                                                << (isNonStale ? lastWriteDateNonStale
+                                                               : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    // make sure all secondaries are in the scan
+    refresher.failedHost(HostAndPort("a"));
+    refresher.failedHost(HostAndPort("b"));
+    refresher.failedHost(HostAndPort("c"));
+
+    HostAndPort notStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notStale.host(), "");
+}
+
+/**
+ * Fail matching maxStalenessMS parameter when all nodes except primary are failed
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSAllButPrimaryFailed) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate" << (primary ? lastWriteDateNonStale
+                                                                            : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    // make sure the primary is in the scan
+    ASSERT(state->findNode(HostAndPort("a")));
+    refresher.failedHost(HostAndPort("b"));
+    refresher.failedHost(HostAndPort("c"));
+
+    // No match because the request needs secondaryOnly host
+    HostAndPort notStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notStale.host(), "");
+}
+
+/**
+ * Fail matching maxStalenessMS parameter one secondary failed,  one secondary is stale
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSOneSecondaryFailed) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate" << (primary ? lastWriteDateNonStale
+                                                                            : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    ASSERT(state->findNode(HostAndPort("a")));
+    ASSERT(state->findNode(HostAndPort("b")));
+    refresher.failedHost(HostAndPort("c"));
+
+    // No match because the write date is stale
+    HostAndPort notStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notStale.host(), "");
+}
+
+/**
+ * Success matching maxStalenessMS parameter when one secondary failed
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSNonStaleeSecondaryMatched) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    Date_t lastWriteDateStale = Date_t::now() - Seconds(1000);
+    Date_t lastWriteDateNonStale = Date_t::now() - Seconds(100);
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        bool isNonStale = ns.host.host() == "b";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "lastWrite"
+                                        << BSON("lastWriteDate"
+                                                << (isNonStale ? lastWriteDateNonStale
+                                                               : lastWriteDateStale))
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    refresher.failedHost(HostAndPort("a"));
+    ASSERT(state->findNode(HostAndPort("b")));
+    refresher.failedHost(HostAndPort("c"));
+
+    HostAndPort notStale = state->getMatchingHost(secondary);
+    ASSERT_EQUALS(notStale.host(), "b");
+}
+
+/**
+ * Fail matching maxStalenessMS parameter when no lastWrite in the response
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSNoLastWrite) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Seconds(200));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    ASSERT(state->findNode(HostAndPort("a")));
+    ASSERT(state->findNode(HostAndPort("b")));
+    ASSERT(state->findNode(HostAndPort("c")));
+
+    ASSERT(state->getMatchingHost(secondary).empty());
+}
+
+/**
+ * Match when maxStalenessMS=0 and no lastWrite in the response
+ */
+TEST(ReplicaSetMonitor, MaxStalenessMSZeroNoLastWrite) {
+    SetStatePtr state = std::make_shared<SetState>("name", basicSeedsSet);
+    Refresher refresher(state);
+
+    const ReadPreferenceSetting secondary(ReadPreference::SecondaryOnly, TagSet(), Milliseconds(0));
+    BSONArray hosts = BSON_ARRAY("a"
+                                 << "b"
+                                 << "c");
+
+    // mock all replies
+    NextStep ns = refresher.getNextStep();
+    while (ns.step == NextStep::CONTACT_HOST) {
+        bool primary = ns.host.host() == "a";
+        refresher.receivedIsMaster(ns.host,
+                                   -1,
+                                   BSON("setName"
+                                        << "name"
+                                        << "ismaster"
+                                        << primary
+                                        << "secondary"
+                                        << !primary
+                                        << "hosts"
+                                        << hosts
+                                        << "ok"
+                                        << true));
+        ns = refresher.getNextStep();
+    }
+
+    // Ensure that we have heard from all hosts and scan is done
+    ASSERT_EQUALS(ns.step, NextStep::DONE);
+
+    ASSERT(state->findNode(HostAndPort("a")));
+    ASSERT(state->findNode(HostAndPort("b")));
+    ASSERT(state->findNode(HostAndPort("c")));
+
+    ASSERT(!state->getMatchingHost(secondary).empty());
+}
+
+
 }  // namespace