SERVER-30768 Mark hosts as down on NetworkInterfaceExceededTimeLimit instead of ExceededTimeLimit

(cherry picked from commit 06b3488461d820c9081e5d7fca0fc028470922ea)

11 files changed, 17 insertions, 25 deletions

diff --git a/jstests/sharding/lagged_config_secondary.js b/jstests/sharding/lagged_config_secondary.js
index 0525e24aaae..63dc9a8c842 100644
--- a/jstests/sharding/lagged_config_secondary.js
+++ b/jstests/sharding/lagged_config_secondary.js
@@ -36,7 +36,8 @@
         testDB.user.find({}).maxTimeMS(15000).itcount();
     });
 
-    assert.eq(ErrorCodes.ExceededTimeLimit, exception.code);
+    assert(exception.code == ErrorCodes.ExceededTimeLimit ||
+           exception.code == ErrorCodes.NetworkInterfaceExceededTimeLimit);
 
     var msg = 'Command on database config timed out waiting for read concern to be satisfied.';
     assert.soon(function() {
diff --git a/src/mongo/base/error_codes.err b/src/mongo/base/error_codes.err
index 0604e9b4cf7..5ace2beae56 100644
--- a/src/mongo/base/error_codes.err
+++ b/src/mongo/base/error_codes.err
@@ -244,3 +244,5 @@ error_class("WriteConcernError", ["WriteConcernFailed",
                                   "UnknownReplWriteConcern",
                                   "CannotSatisfyWriteConcern"])
 error_class("ShutdownError", ["ShutdownInProgress", "InterruptedAtShutdown"])
+
+error_class("ExceededTimeLimitError", ["ExceededTimeLimit", "NetworkInterfaceExceededTimeLimit"])
diff --git a/src/mongo/db/s/balancer/migration_manager_test.cpp b/src/mongo/db/s/balancer/migration_manager_test.cpp
index 1c5ead4acbf..03001c3fecb 100644
--- a/src/mongo/db/s/balancer/migration_manager_test.cpp
+++ b/src/mongo/db/s/balancer/migration_manager_test.cpp
@@ -1014,7 +1014,7 @@ TEST_F(MigrationManagerTest, RemoteCallErrorConversionToOperationFailed) {
         chunk2,
         kShardId3,
         false,
-        Status(ErrorCodes::ExceededTimeLimit,
+        Status(ErrorCodes::NetworkInterfaceExceededTimeLimit,
                "RemoteCallErrorConversionToOperationFailedCheck generated error."));
 
     // Run the MigrationManager code.
diff --git a/src/mongo/executor/connection_pool_asio_integration_test.cpp b/src/mongo/executor/connection_pool_asio_integration_test.cpp
index 6c60cd78ecd..049b3d9d8d8 100644
--- a/src/mongo/executor/connection_pool_asio_integration_test.cpp
+++ b/src/mongo/executor/connection_pool_asio_integration_test.cpp
@@ -175,7 +175,7 @@ TEST(ConnectionPoolASIO, ConnSetupTimeout) {
         deferred.emplace(std::move(resp));
     });
 
-    ASSERT_EQ(deferred.get().status.code(), ErrorCodes::ExceededTimeLimit);
+    ASSERT_EQ(deferred.get().status.code(), ErrorCodes::NetworkInterfaceExceededTimeLimit);
 }
 
 /**
diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp
index 9056bd1a570..6bf719eb793 100644
--- a/src/mongo/executor/network_interface_asio.cpp
+++ b/src/mongo/executor/network_interface_asio.cpp
@@ -290,10 +290,7 @@ Status NetworkInterfaceASIO::startCommand(const TaskExecutor::CallbackHandle& cb
             Status status = wasPreviouslyCanceled
                 ? Status(ErrorCodes::CallbackCanceled, "Callback canceled")
                 : swConn.getStatus();
-            if (status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) {
-                status = Status(ErrorCodes::ExceededTimeLimit, status.reason());
-            }
-            if (status.code() == ErrorCodes::ExceededTimeLimit) {
+            if (ErrorCodes::isExceededTimeLimitError(status.code())) {
                 _numTimedOutOps.fetchAndAdd(1);
             }
             if (status.code() != ErrorCodes::CallbackCanceled) {
diff --git a/src/mongo/executor/network_interface_asio_command.cpp b/src/mongo/executor/network_interface_asio_command.cpp
index 52d523e7d97..55bcb34c3b4 100644
--- a/src/mongo/executor/network_interface_asio_command.cpp
+++ b/src/mongo/executor/network_interface_asio_command.cpp
@@ -287,8 +287,7 @@ void NetworkInterfaceASIO::_completeOperation(AsyncOp* op, ResponseStatus resp)
         op->_timeoutAlarm->cancel();
     }
 
-    if (resp.status.code() == ErrorCodes::ExceededTimeLimit ||
-        resp.status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) {
+    if (ErrorCodes::isExceededTimeLimitError(resp.status.code())) {
         _numTimedOutOps.fetchAndAdd(1);
     }
 
diff --git a/src/mongo/executor/network_interface_asio_integration_test.cpp b/src/mongo/executor/network_interface_asio_integration_test.cpp
index 39282fb7fcd..79688a60d19 100644
--- a/src/mongo/executor/network_interface_asio_integration_test.cpp
+++ b/src/mongo/executor/network_interface_asio_integration_test.cpp
@@ -65,7 +65,7 @@ TEST_F(NetworkInterfaceASIOIntegrationFixture, Timeouts) {
                                             << "none"
                                             << "secs"
                                             << 10),
-                               ErrorCodes::ExceededTimeLimit,
+                               ErrorCodes::NetworkInterfaceExceededTimeLimit,
                                Milliseconds(100));
 
     // Run a sleep command that should return before we hit the ASIO timeout.
@@ -179,7 +179,7 @@ TEST_F(NetworkInterfaceASIOIntegrationFixture, StressTest) {
             expectedResults[i] = ErrorCodes::OK;
             StressTestOp::runCompleteOp(this, cb);
         } else if (r < .99) {
-            expectedResults[i] = ErrorCodes::ExceededTimeLimit;
+            expectedResults[i] = ErrorCodes::NetworkInterfaceExceededTimeLimit;
             StressTestOp::runTimeoutOp(this, cb);
         } else {
             // Just a sprinkling of long ops, to mitigate connection pool contention
@@ -228,7 +228,7 @@ TEST_F(NetworkInterfaceASIOIntegrationFixture, HookHangs) {
     startNet(std::move(options));
 
     assertCommandFailsOnClient(
-        "admin", BSON("ping" << 1), ErrorCodes::ExceededTimeLimit, Seconds(1));
+        "admin", BSON("ping" << 1), ErrorCodes::NetworkInterfaceExceededTimeLimit, Seconds(1));
 }
 
 }  // namespace
diff --git a/src/mongo/executor/network_interface_asio_operation.cpp b/src/mongo/executor/network_interface_asio_operation.cpp
index b300080ce76..87d4b740ec6 100644
--- a/src/mongo/executor/network_interface_asio_operation.cpp
+++ b/src/mongo/executor/network_interface_asio_operation.cpp
@@ -198,13 +198,6 @@ void NetworkInterfaceASIO::AsyncOp::finish(ResponseStatus&& rs) {
     LOG(2) << "Request " << _request.id << " finished with response: "
            << redact(rs.isOK() ? rs.data.toString() : rs.status.toString());
 
-    // Our own internally generated time outs have a different error code to allow us to retry
-    // internal timeouts.  But the outside world (and our tests) still expect the one true
-    // ExceededTimeLimit, so we convert back here so they get what they expect.
-    if (!rs.isOK() && rs.status.code() == ErrorCodes::NetworkInterfaceExceededTimeLimit) {
-        rs.status = Status(ErrorCodes::ExceededTimeLimit, rs.status.reason());
-    }
-
     // Calling the completion handler may invalidate state in this op, so do it last.
     _onFinish(rs);
 }
diff --git a/src/mongo/executor/network_interface_asio_test.cpp b/src/mongo/executor/network_interface_asio_test.cpp
index 2564b9611e5..23ef151d92f 100644
--- a/src/mongo/executor/network_interface_asio_test.cpp
+++ b/src/mongo/executor/network_interface_asio_test.cpp
@@ -325,7 +325,7 @@ TEST_F(NetworkInterfaceASIOTest, TimeoutWithNetworkError) {
 
     // Wait for op to complete, assert that timeout had precedence.
     auto& result = deferred.get();
-    ASSERT_EQ(ErrorCodes::ExceededTimeLimit, result.status);
+    ASSERT_EQ(ErrorCodes::NetworkInterfaceExceededTimeLimit, result.status);
     ASSERT(result.elapsedMillis);
     assertNumOps(0u, 1u, 1u, 0u);
 }
@@ -394,7 +394,7 @@ TEST_F(NetworkInterfaceASIOTest, AsyncOpTimeout) {
     }
 
     auto& result = deferred.get();
-    ASSERT_EQ(ErrorCodes::ExceededTimeLimit, result.status);
+    ASSERT_EQ(ErrorCodes::NetworkInterfaceExceededTimeLimit, result.status);
     ASSERT(result.elapsedMillis);
     assertNumOps(0u, 1u, 1u, 0u);
 }
diff --git a/src/mongo/s/client/shard.cpp b/src/mongo/s/client/shard.cpp
index be61c500604..006af71f3dc 100644
--- a/src/mongo/s/client/shard.cpp
+++ b/src/mongo/s/client/shard.cpp
@@ -97,7 +97,7 @@ bool Shard::shouldErrorBePropagated(ErrorCodes::Error code) {
     return std::find(RemoteCommandRetryScheduler::kAllRetriableErrors.begin(),
                      RemoteCommandRetryScheduler::kAllRetriableErrors.end(),
                      code) == RemoteCommandRetryScheduler::kAllRetriableErrors.end() &&
-        code != ErrorCodes::ExceededTimeLimit;
+        code != ErrorCodes::NetworkInterfaceExceededTimeLimit;
 }
 
 Shard::Shard(const ShardId& id) : _id(id) {}
diff --git a/src/mongo/s/client/shard_remote.cpp b/src/mongo/s/client/shard_remote.cpp
index 00c9141f2d6..1f0538a19f5 100644
--- a/src/mongo/s/client/shard_remote.cpp
+++ b/src/mongo/s/client/shard_remote.cpp
@@ -142,7 +142,7 @@ void ShardRemote::updateReplSetMonitor(const HostAndPort& remoteHost,
         _targeter->markHostUnreachable(remoteHost, remoteCommandStatus);
     } else if (remoteCommandStatus == ErrorCodes::NotMasterOrSecondary) {
         _targeter->markHostUnreachable(remoteHost, remoteCommandStatus);
-    } else if (remoteCommandStatus == ErrorCodes::ExceededTimeLimit) {
+    } else if (remoteCommandStatus == ErrorCodes::NetworkInterfaceExceededTimeLimit) {
         _targeter->markHostUnreachable(remoteHost, remoteCommandStatus);
     }
 }
@@ -228,7 +228,7 @@ Shard::HostWithResponse ShardRemote::_runCommand(OperationContext* txn,
     updateReplSetMonitor(host.getValue(), swResponse.status);
 
     if (!swResponse.isOK()) {
-        if (swResponse.status.compareCode(ErrorCodes::ExceededTimeLimit)) {
+        if (ErrorCodes::isExceededTimeLimitError(swResponse.status.code())) {
             LOG(0) << "Operation timed out with status " << redact(swResponse.status);
         }
         return Shard::HostWithResponse(host.getValue(), swResponse.status);
@@ -362,7 +362,7 @@ StatusWith<Shard::QueryResponse> ShardRemote::_exhaustiveFindOnConfig(
     updateReplSetMonitor(host.getValue(), status);
 
     if (!status.isOK()) {
-        if (status.compareCode(ErrorCodes::ExceededTimeLimit)) {
+        if (ErrorCodes::isExceededTimeLimitError(status.code())) {
             LOG(0) << "Operation timed out " << causedBy(status);
         }
         return status;