SERVER-41679 Add counters in serverStatus for slow network actions

4 files changed, 43 insertions, 2 deletions

diff --git a/src/mongo/db/stats/counters.cpp b/src/mongo/db/stats/counters.cpp
index f9fd472eebe..0db49e44aca 100644
--- a/src/mongo/db/stats/counters.cpp
+++ b/src/mongo/db/stats/counters.cpp
@@ -148,6 +148,14 @@ void NetworkCounter::hitLogicalOut(long long bytes) {
     }
 }
 
+void NetworkCounter::incrementNumSlowDNSOperations() {
+    _numSlowDNSOperations.fetchAndAdd(1);
+}
+
+void NetworkCounter::incrementNumSlowSSLOperations() {
+    _numSlowSSLOperations.fetchAndAdd(1);
+}
+
 void NetworkCounter::acceptedTFOIngress() {
     _tfo.accepted.fetchAndAddRelaxed(1);
 }
@@ -157,6 +165,8 @@ void NetworkCounter::append(BSONObjBuilder& b) {
     b.append("bytesOut", static_cast<long long>(_logicalBytesOut.loadRelaxed()));
     b.append("physicalBytesIn", static_cast<long long>(_physicalBytesIn.loadRelaxed()));
     b.append("physicalBytesOut", static_cast<long long>(_physicalBytesOut.loadRelaxed()));
+    b.append("numSlowDNSOperations", static_cast<long long>(_numSlowDNSOperations.loadRelaxed()));
+    b.append("numSlowSSLOperations", static_cast<long long>(_numSlowSSLOperations.loadRelaxed()));
     b.append("numRequests", static_cast<long long>(_together.requests.loadRelaxed()));
 
     BSONObjBuilder tfo;
diff --git a/src/mongo/db/stats/counters.h b/src/mongo/db/stats/counters.h
index 36037052cc4..7a5a2aa8f02 100644
--- a/src/mongo/db/stats/counters.h
+++ b/src/mongo/db/stats/counters.h
@@ -123,6 +123,12 @@ public:
     void hitLogicalIn(long long bytes);
     void hitLogicalOut(long long bytes);
 
+    // Increment the counter for the number of slow dns resolution operations.
+    void incrementNumSlowDNSOperations();
+
+    // Increment the counter for the number of slow ssl handshake operations.
+    void incrementNumSlowSSLOperations();
+
     // TFO Counters and Status;
     void acceptedTFOIngress();
 
@@ -156,6 +162,9 @@ private:
 
     CacheAligned<AtomicWord<long long>> _logicalBytesOut{0};
 
+    CacheAligned<AtomicWord<long long>> _numSlowDNSOperations{0};
+    CacheAligned<AtomicWord<long long>> _numSlowSSLOperations{0};
+
     struct TFO {
         // Counter of inbound connections at runtime.
         AtomicWord<std::int64_t> accepted{0};
diff --git a/src/mongo/transport/transport_layer_asio.cpp b/src/mongo/transport/transport_layer_asio.cpp
index a22ffdd7e2e..4b78a05e9e4 100644
--- a/src/mongo/transport/transport_layer_asio.cpp
+++ b/src/mongo/transport/transport_layer_asio.cpp
@@ -454,7 +454,13 @@ StatusWith<SessionHandle> TransportLayerASIO::connect(HostAndPort peer,
     GenericSocket sock(*_egressReactor);
     WrappedResolver resolver(*_egressReactor);
 
+    Date_t timeBefore = Date_t::now();
     auto swEndpoints = resolver.resolve(peer, _listenerOptions.enableIPv6);
+    Date_t timeAfter = Date_t::now();
+    if (timeAfter - timeBefore > kSlowOperationThreshold) {
+        networkCounter.incrementNumSlowDNSOperations();
+    }
+
     if (!swEndpoints.isOK()) {
         return swEndpoints.getStatus();
     }
@@ -484,7 +490,13 @@ StatusWith<SessionHandle> TransportLayerASIO::connect(HostAndPort peer,
         (sslMode == kGlobalSSLMode &&
          ((globalSSLMode == SSLParams::SSLMode_preferSSL) ||
           (globalSSLMode == SSLParams::SSLMode_requireSSL)))) {
+        Date_t timeBefore = Date_t::now();
         auto sslStatus = session->handshakeSSLForEgress(peer).getNoThrow();
+        Date_t timeAfter = Date_t::now();
+        if (timeAfter - timeBefore > kSlowOperationThreshold) {
+            networkCounter.incrementNumSlowSSLOperations();
+        }
+
         if (!sslStatus.isOK()) {
             return sslStatus;
         }
@@ -616,12 +628,13 @@ Future<SessionHandle> TransportLayerASIO::asyncConnect(HostAndPort peer,
         .then([connector, timeBefore](WrappedResolver::EndpointVector results) {
             try {
                 Date_t timeAfter = Date_t::now();
-                if (timeAfter - timeBefore > Seconds(1)) {
+                if (timeAfter - timeBefore > kSlowOperationThreshold) {
                     LOGV2_WARNING(23019,
                                   "DNS resolution while connecting to {connector_peer} took "
                                   "{timeAfter_timeBefore}",
                                   "connector_peer"_attr = connector->peer,
                                   "timeAfter_timeBefore"_attr = timeAfter - timeBefore);
+                    networkCounter.incrementNumSlowDNSOperations();
                 }
 
                 stdx::lock_guard<Latch> lk(connector->mutex);
@@ -658,9 +671,16 @@ Future<SessionHandle> TransportLayerASIO::asyncConnect(HostAndPort peer,
                 (sslMode == kGlobalSSLMode &&
                  ((globalSSLMode == SSLParams::SSLMode_preferSSL) ||
                   (globalSSLMode == SSLParams::SSLMode_requireSSL)))) {
+                Date_t timeBefore = Date_t::now();
                 return connector->session
                     ->handshakeSSLForEgressWithLock(std::move(lk), connector->peer)
-                    .then([connector] { return Status::OK(); });
+                    .then([connector, timeBefore] {
+                        Date_t timeAfter = Date_t::now();
+                        if (timeAfter - timeBefore > kSlowOperationThreshold) {
+                            networkCounter.incrementNumSlowSSLOperations();
+                        }
+                        return Status::OK();
+                    });
             }
 #endif
             return Status::OK();
diff --git a/src/mongo/transport/transport_layer_asio.h b/src/mongo/transport/transport_layer_asio.h
index da3b978863c..f551dac5420 100644
--- a/src/mongo/transport/transport_layer_asio.h
+++ b/src/mongo/transport/transport_layer_asio.h
@@ -84,6 +84,8 @@ class TransportLayerASIO final : public TransportLayer {
     TransportLayerASIO& operator=(const TransportLayerASIO&) = delete;
 
 public:
+    constexpr static auto kSlowOperationThreshold = Seconds(1);
+
     struct Options {
         constexpr static auto kIngress = 0x1;
         constexpr static auto kEgress = 0x10;