SERVER-15535 finish freshness checker tests that were waiting on MockNetworkInterface

1 files changed, 160 insertions, 188 deletions

diff --git a/src/mongo/db/repl/freshness_checker_test.cpp b/src/mongo/db/repl/freshness_checker_test.cpp
index 459cc7e9f50..3dfb993f3a3 100644
--- a/src/mongo/db/repl/freshness_checker_test.cpp
+++ b/src/mongo/db/repl/freshness_checker_test.cpp
@@ -307,7 +307,6 @@ namespace {
                                                 "set" << "rs0" <<
                                                 "who" << "h1" <<
                                                 "cfgver" << 1 <<
-                                                "fresher" << true <<
                                                 "opTime" << Date_t(OpTime(10,0).asDate())),
                                            Milliseconds(8))));
         }
@@ -318,7 +317,6 @@ namespace {
 
         stopCapturingLogMessages();
         ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound);
-        ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest"));
     }
 
     TEST_F(FreshnessCheckerTest, ElectWrongTypeInFreshnessResponse) {
@@ -482,99 +480,84 @@ namespace {
         ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest"));
     }
 
-    // TODO(dannenberg) re-enable this test once we can control message order
-
-    // TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTimeManyNodes) {
-    //     // one other responds with a later optime than ours
-    //     startCapturingLogMessages();
-    //     ReplicaSetConfig config = assertMakeRSConfig(
-    //             BSON("_id" << "rs0" <<
-    //                  "version" << 1 <<
-    //                  "members" << BSON_ARRAY(
-    //                      BSON("_id" << 1 << "host" << "h0") <<
-    //                      BSON("_id" << 2 << "host" << "h1") <<
-    //                      BSON("_id" << 3 << "host" << "h2") <<
-    //                      BSON("_id" << 4 << "host" << "h3") <<
-    //                      BSON("_id" << 5 << "host" << "h4"))));
+    TEST_F(FreshnessCheckerTest, ElectNotElectingSelfWeAreNotFreshestOpTimeManyNodes) {
+        // one other responds with a later optime than ours
+        startCapturingLogMessages();
+        ReplicaSetConfig config = assertMakeRSConfig(
+                BSON("_id" << "rs0" <<
+                     "version" << 1 <<
+                     "members" << BSON_ARRAY(
+                         BSON("_id" << 1 << "host" << "h0") <<
+                         BSON("_id" << 2 << "host" << "h1") <<
+                         BSON("_id" << 3 << "host" << "h2") <<
+                         BSON("_id" << 4 << "host" << "h3") <<
+                         BSON("_id" << 5 << "host" << "h4"))));
         
-    //     StatusWith<ReplicationExecutor::EventHandle> evh = _executor->makeEvent();
-    //     ASSERT_OK(evh.getStatus());
-
-    //     Date_t now(0);
-    //     std::vector<HostAndPort> hosts;
-    //     for (ReplicaSetConfig::MemberIterator mem = config.membersBegin();
-    //             mem != config.membersEnd();
-    //             ++mem) {
-    //         hosts.push_back(mem->getHostAndPort());
-    //     }
-
-    //     const BSONObj freshRequest = makeFreshRequest(config, OpTime(0,0), 0);
-
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h1"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 2 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h1" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "fresher" << true <<
-    //                                                "opTime" << Date_t(OpTime(10,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h2"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 3 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h2" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h3"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 4 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h3" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h4"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 5 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h4" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-
-    //     FreshnessChecker checker;
-    //     StatusWith<ReplicationExecutor::CallbackHandle> cbh = 
-    //         _executor->scheduleWork(
-    //             stdx::bind(&FreshnessCheckerTest::freshnessCheckerRunner,
-    //                        this,
-    //                        stdx::placeholders::_1,
-    //                        &checker,
-    //                        evh.getValue(),
-    //                        OpTime(0,0),
-    //                        config,
-    //                        0,
-    //                        hosts));
-    //     ASSERT_OK(cbh.getStatus());
-    //     _executor->wait(cbh.getValue());
-
-    //     ASSERT_OK(_lastStatus);
-
-    //     _executor->waitForEvent(evh.getValue());
-    //     stopCapturingLogMessages();
-
-    //     bool weAreFreshest(true);
-    //     bool tied(true);
-    //     checker.getResults(&weAreFreshest, &tied);
-    //     ASSERT_FALSE(weAreFreshest);
-    //     ASSERT_FALSE(tied);
-    //     ASSERT_EQUALS(1, countLogLinesContaining("not electing self, we are not freshest"));
-    // }
+        std::vector<HostAndPort> hosts;
+        for (ReplicaSetConfig::MemberIterator mem = config.membersBegin();
+                mem != config.membersEnd();
+                ++mem) {
+            if (HostAndPort("h0") == mem->getHostAndPort()) {
+                continue;
+            }
+            hosts.push_back(mem->getHostAndPort());
+        }
+
+        const BSONObj freshRequest = makeFreshRequest(config, OpTime(10,0), 0);
+
+        startTest(OpTime(10, 0), config, 0, hosts);
+        const Date_t startDate = _net->now();
+        unordered_set<HostAndPort> seen;
+        _net->enterNetwork();
+
+        for (size_t i = 0; i < hosts.size(); ++i) {
+            const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+            const HostAndPort target = noi->getRequest().target;
+            ASSERT_EQUALS("admin", noi->getRequest().dbname);
+            ASSERT_EQUALS(freshRequest, noi->getRequest().cmdObj);
+            ASSERT(seen.insert(target).second) << "Already saw " << target;
+            BSONObjBuilder responseBuilder;
+            if (target.host() == "h4") {
+                responseBuilder <<
+                    "ok" << 1 <<
+                    "id" << findIdForMember(config, target) <<
+                    "set" << "rs0" <<
+                    "who" << target.toString() <<
+                    "cfgver" << 1 <<
+                    "opTime" << Date_t(OpTime(20,0).asDate());
+                _net->scheduleResponse(
+                        noi,
+                        startDate + 20,
+                        ResponseStatus(ReplicationExecutor::RemoteCommandResponse(
+                                               responseBuilder.obj(),
+                                               Milliseconds(8))));
+            }
+            else {
+                responseBuilder <<
+                    "ok" << 1 <<
+                    "id" << findIdForMember(config, target) <<
+                    "set" << "rs0" <<
+                    "who" << target.toString() <<
+                    "cfgver" << 1 <<
+                    "opTime" << Date_t(OpTime(10,0).asDate());
+                _net->scheduleResponse(
+                        noi,
+                        startDate + 10,
+                        ResponseStatus(ReplicationExecutor::RemoteCommandResponse(
+                                               responseBuilder.obj(),
+                                               Milliseconds(8))));
+            }
+        }
+        _net->runUntil(startDate + 10);
+        ASSERT_EQUALS(startDate + 10, _net->now());
+        ASSERT_EQUALS(0, countLogLinesContaining("not electing self, we are not freshest"));
+        _net->runUntil(startDate + 20);
+        ASSERT_EQUALS(startDate + 20, _net->now());
+        _net->exitNetwork();
+        waitOnChecker();
+        stopCapturingLogMessages();
+        ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound);
+    }
 
     TEST_F(FreshnessCheckerTest, ElectWrongTypeInFreshnessResponseManyNodes) {
         // one other responds with "opTime" field of non-Date value, causing not freshest
@@ -698,100 +681,89 @@ namespace {
                                                  "errmsg: \"I'd rather you didn't\"'"));
     }
 
-    // TODO(dannenberg) re-enable this test once we can control message order
-
-    // TEST_F(FreshnessCheckerTest, ElectVetoedAndTiedFreshnessManyNodes) {
-    //     // one other responds with veto and another responds with tie
-    //     startCapturingLogMessages();
-    //     ReplicaSetConfig config = assertMakeRSConfig(
-    //             BSON("_id" << "rs0" <<
-    //                  "version" << 1 <<
-    //                  "members" << BSON_ARRAY(
-    //                      BSON("_id" << 1 << "host" << "h0") <<
-    //                      BSON("_id" << 2 << "host" << "h1") <<
-    //                      BSON("_id" << 3 << "host" << "h2") <<
-    //                      BSON("_id" << 4 << "host" << "h3") <<
-    //                      BSON("_id" << 5 << "host" << "h4"))));
-    //     StatusWith<ReplicationExecutor::EventHandle> evh = _executor->makeEvent();
-    //     ASSERT_OK(evh.getStatus());
-
-    //     Date_t now(0);
-    //     std::vector<HostAndPort> hosts;
-    //     for (ReplicaSetConfig::MemberIterator mem = config.membersBegin();
-    //             mem != config.membersEnd();
-    //             ++mem) {
-    //         hosts.push_back(mem->getHostAndPort());
-    //     }
-
-    //     const BSONObj freshRequest = makeFreshRequest(config, OpTime(10,0), 0);
-
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h1"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 2 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h1" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "veto" << true <<
-    //                                                "errmsg" << "I'd rather you didn't" <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h2"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 3 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h2" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(10,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h3"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 4 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h3" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-    //     _net->addResponse(RemoteCommandRequest(HostAndPort("h4"),
-    //                                            "admin",
-    //                                            freshRequest),
-    //                       StatusWith<BSONObj>(BSON("ok" << 1 <<
-    //                                                "id" << 5 <<
-    //                                                "set" << "rs0" <<
-    //                                                "who" << "h4" <<
-    //                                                "cfgver" << 1 <<
-    //                                                "opTime" << Date_t(OpTime(0,0).asDate()))));
-
-    //     FreshnessChecker checker;
-    //     StatusWith<ReplicationExecutor::CallbackHandle> cbh = 
-    //         _executor->scheduleWork(
-    //             stdx::bind(&FreshnessCheckerTest::freshnessCheckerRunner,
-    //                        this,
-    //                        stdx::placeholders::_1,
-    //                        &checker,
-    //                        evh.getValue(),
-    //                        OpTime(10,0),
-    //                        config,
-    //                        0,
-    //                        hosts));
-    //     ASSERT_OK(cbh.getStatus());
-    //     _executor->wait(cbh.getValue());
-
-    //     ASSERT_OK(_lastStatus);
-
-    //     _executor->waitForEvent(evh.getValue());
-    //     stopCapturingLogMessages();
-
-    //     bool weAreFreshest(true);
-    //     bool tied(true);
-    //     checker.getResults(&weAreFreshest, &tied);
-    //     ASSERT_FALSE(weAreFreshest);
-    //     ASSERT_FALSE(tied);
-    //     ASSERT_EQUALS(1, countLogLinesContaining("not electing self, h1:27017 would veto with '"
-    //                                              "errmsg: \"I'd rather you didn't\"'"));
-    // }
+    TEST_F(FreshnessCheckerTest, ElectVetoedAndTiedFreshnessManyNodes) {
+        // one other responds with veto and another responds with tie
+        startCapturingLogMessages();
+        ReplicaSetConfig config = assertMakeRSConfig(
+                BSON("_id" << "rs0" <<
+                     "version" << 1 <<
+                     "members" << BSON_ARRAY(
+                         BSON("_id" << 1 << "host" << "h0") <<
+                         BSON("_id" << 2 << "host" << "h1") <<
+                         BSON("_id" << 3 << "host" << "h2") <<
+                         BSON("_id" << 4 << "host" << "h3") <<
+                         BSON("_id" << 5 << "host" << "h4"))));
+        
+        std::vector<HostAndPort> hosts;
+        for (ReplicaSetConfig::MemberIterator mem = config.membersBegin();
+                mem != config.membersEnd();
+                ++mem) {
+            if (HostAndPort("h0") == mem->getHostAndPort()) {
+                continue;
+            }
+            hosts.push_back(mem->getHostAndPort());
+        }
+
+        const BSONObj freshRequest = makeFreshRequest(config, OpTime(10,0), 0);
+
+        startTest(OpTime(10, 0), config, 0, hosts);
+        const Date_t startDate = _net->now();
+        unordered_set<HostAndPort> seen;
+        _net->enterNetwork();
+
+        for (size_t i = 0; i < hosts.size(); ++i) {
+            const NetworkInterfaceMock::NetworkOperationIterator noi = _net->getNextReadyRequest();
+            const HostAndPort target = noi->getRequest().target;
+            ASSERT_EQUALS("admin", noi->getRequest().dbname);
+            ASSERT_EQUALS(freshRequest, noi->getRequest().cmdObj);
+            ASSERT(seen.insert(target).second) << "Already saw " << target;
+            BSONObjBuilder responseBuilder;
+            if (target.host() == "h4") {
+                responseBuilder <<
+                    "ok" << 1 <<
+                    "id" << findIdForMember(config, target) <<
+                    "set" << "rs0" <<
+                    "who" << target.toString() <<
+                    "cfgver" << 1 <<
+                    "veto" << true <<
+                    "errmsg" << "I'd rather you didn't" <<
+                    "opTime" << Date_t(OpTime(10,0).asDate());
+                _net->scheduleResponse(
+                        noi,
+                        startDate + 20,
+                        ResponseStatus(ReplicationExecutor::RemoteCommandResponse(
+                                               responseBuilder.obj(),
+                                               Milliseconds(8))));
+            }
+            else {
+                responseBuilder <<
+                    "ok" << 1 <<
+                    "id" << findIdForMember(config, target) <<
+                    "set" << "rs0" <<
+                    "who" << target.toString() <<
+                    "cfgver" << 1 <<
+                    "opTime" << Date_t(OpTime(10,0).asDate());
+                _net->scheduleResponse(
+                        noi,
+                        startDate + 10,
+                        ResponseStatus(ReplicationExecutor::RemoteCommandResponse(
+                                               responseBuilder.obj(),
+                                               Milliseconds(8))));
+            }
+        }
+        _net->runUntil(startDate + 10);
+        ASSERT_EQUALS(startDate + 10, _net->now());
+        ASSERT_EQUALS(0, countLogLinesContaining("not electing self, h4:27017 would veto with '"
+                                                 "errmsg: \"I'd rather you didn't\"'"));
+        _net->runUntil(startDate + 20);
+        ASSERT_EQUALS(startDate + 20, _net->now());
+        _net->exitNetwork();
+        waitOnChecker();
+        stopCapturingLogMessages();
+        ASSERT_EQUALS(shouldAbortElection(), FreshnessChecker::FresherNodeFound);
+        ASSERT_EQUALS(1, countLogLinesContaining("not electing self, h4:27017 would veto with '"
+                                                 "errmsg: \"I'd rather you didn't\"'"));
+    }
 
     TEST_F(FreshnessCheckerTest, ElectManyNodesNotAllRespond) {
         ReplicaSetConfig config = assertMakeRSConfig(