SERVER-16217 improve replSetReconfig and replSetInitiate error messages

4 files changed, 36 insertions, 17 deletions

diff --git a/src/mongo/db/repl/check_quorum_for_config_change.cpp b/src/mongo/db/repl/check_quorum_for_config_change.cpp
index 2a6804027ae..7064b9a473d 100644
--- a/src/mongo/db/repl/check_quorum_for_config_change.cpp
+++ b/src/mongo/db/repl/check_quorum_for_config_change.cpp
@@ -122,12 +122,18 @@ namespace repl {
             _finalStatus = _vetoStatus;
             return;
         }
-        if (_rsConfig->getConfigVersion() == 1 && !_down.empty()) {
+        if (_rsConfig->getConfigVersion() == 1 && !_badResponses.empty()) {
             str::stream message;
-            message << "Could not contact the following nodes during replica set initiation: " <<
-                _down.front().toString();
-            for (size_t i = 1; i < _down.size(); ++i) {
-                message << ", " << _down[i].toString();
+            message << "replSetInitiate quorum check failed because not all proposed set members "
+                       "responded affirmatively: ";
+            for (std::vector<std::pair<HostAndPort, Status> >::const_iterator it =
+                        _badResponses.begin();
+                    it != _badResponses.end();
+                    ++it) {
+                if (it != _badResponses.begin()) {
+                    message << ", ";
+                }
+                message << it->first.toString() << " failed with " << it->second.reason();
             }
             _finalStatus = Status(ErrorCodes::NodeNotFound, message);
             return;
@@ -153,6 +159,18 @@ namespace repl {
                     message << ", " << _voters[i].toString();
                 }
             }
+            if (!_badResponses.empty()) {
+                message << "; the following nodes did not respond affirmatively: ";
+                for (std::vector<std::pair<HostAndPort, Status> >::const_iterator it =
+                            _badResponses.begin();
+                        it != _badResponses.end();
+                        ++it) {
+                    if (it != _badResponses.begin()) {
+                        message << ", ";
+                    }
+                    message << it->first.toString() << " failed with " << it->second.reason();
+                }
+            }
             _finalStatus = Status(ErrorCodes::NodeNotFound, message);
             return;
         }
@@ -167,7 +185,7 @@ namespace repl {
         if (!response.isOK()) {
             warning() << "Failed to complete heartbeat request to " << request.target <<
                 "; " << response.getStatus();
-            _down.push_back(request.target);
+            _badResponses.push_back(std::make_pair(request.target, response.getStatus()));
             return;
         }
 
@@ -187,7 +205,7 @@ namespace repl {
             warning() << "Got error (" << hbStatus
                       << ") response on heartbeat request to " << request.target
                       << "; " << hbResp;
-            _down.push_back(request.target);
+            _badResponses.push_back(std::make_pair(request.target, hbStatus));
             return;
         }
 
diff --git a/src/mongo/db/repl/check_quorum_for_config_change.h b/src/mongo/db/repl/check_quorum_for_config_change.h
index 04f3f87b340..396ac4dea39 100644
--- a/src/mongo/db/repl/check_quorum_for_config_change.h
+++ b/src/mongo/db/repl/check_quorum_for_config_change.h
@@ -90,12 +90,12 @@ namespace repl {
         // Index of the local node's member configuration in _rsConfig.
         const int _myIndex;
 
-        // List of nodes believed to be down.
-        std::vector<HostAndPort> _down;
-
         // List of voting nodes that have responded affirmatively.
         std::vector<HostAndPort> _voters;
 
+        // List of nodes with bad responses and the bad response status they returned.
+        std::vector<std::pair<HostAndPort, Status> > _badResponses;
+
         // Total number of responses and timeouts processed.
         int _numResponses;
 
diff --git a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
index 22b30cfeda4..49d87e9cf4f 100644
--- a/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
+++ b/src/mongo/db/repl/check_quorum_for_config_change_test.cpp
@@ -201,7 +201,7 @@ namespace {
         Status status = waitForQuorumCheck();
         ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
         ASSERT_REASON_CONTAINS(
-                status, "Could not contact the following nodes during replica set initiation");
+                status, "replSetInitiate quorum check failed because not all proposed set members");
         ASSERT_REASON_CONTAINS(status, "h1:1");
         ASSERT_REASON_CONTAINS(status, "h2:1");
         ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
@@ -308,7 +308,7 @@ namespace {
         Status status = waitForQuorumCheck();
         ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
         ASSERT_REASON_CONTAINS(
-                status, "Could not contact the following nodes during replica set initiation");
+                status, "replSetInitiate quorum check failed because not all proposed set members");
         ASSERT_NOT_REASON_CONTAINS(status, "h1:1");
         ASSERT_REASON_CONTAINS(status, "h2:1");
         ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
@@ -696,8 +696,8 @@ namespace {
         ASSERT_EQUALS(ErrorCodes::NodeNotFound, status);
         ASSERT_REASON_CONTAINS(status, "not enough voting nodes responded; required 2 but only");
         ASSERT_REASON_CONTAINS(status, "h1:1");
-        ASSERT_NOT_REASON_CONTAINS(status, "h2:1");
-        ASSERT_NOT_REASON_CONTAINS(status, "h3:1");
+        ASSERT_REASON_CONTAINS(status, "h2:1 failed with");
+        ASSERT_REASON_CONTAINS(status, "h3:1 failed with");
         ASSERT_NOT_REASON_CONTAINS(status, "h4:1");
         ASSERT_NOT_REASON_CONTAINS(status, "h5:1");
     }
diff --git a/src/mongo/db/repl/replset_commands.cpp b/src/mongo/db/repl/replset_commands.cpp
index 52a8bc819f5..40fc8f8b095 100644
--- a/src/mongo/db/repl/replset_commands.cpp
+++ b/src/mongo/db/repl/replset_commands.cpp
@@ -598,11 +598,12 @@ namespace {
                 sleepsecs(data["delay"].numberInt());
             }
 
+            Status status = Status(ErrorCodes::InternalError, "status not set in heartbeat code");
             /* we don't call ReplSetCommand::check() here because heartbeat
                checks many things that are pre-initialization. */
             if (!getGlobalReplicationCoordinator()->getSettings().usingReplSets()) {
-                errmsg = "not running with --replSet";
-                return false;
+                status = Status(ErrorCodes::NoReplicationEnabled, "not running with --replSet");
+                return appendCommandStatus(result, status);
             }
 
             /* we want to keep heartbeat connections open when relinquishing primary.
@@ -614,7 +615,7 @@ namespace {
             }
 
             ReplSetHeartbeatArgs args;
-            Status status = args.initialize(cmdObj);
+            status = args.initialize(cmdObj);
             if (!status.isOK()) {
                 return appendCommandStatus(result, status);
             }