SERVER-63263 Add metric for connection establishment once MongoDB accepts() a new connection on a socket

8 files changed, 62 insertions, 1 deletions

diff --git a/jstests/sharding/listener_processing_server_status_metrics.js b/jstests/sharding/listener_processing_server_status_metrics.js
new file mode 100644
index 00000000000..536bb51b0d7
--- /dev/null
+++ b/jstests/sharding/listener_processing_server_status_metrics.js
@@ -0,0 +1,33 @@
+/**
+ *  @tags: [requires_fcv_60, featureFlagConnHealthMetrics]
+ *
+ * Tests that listener processing times for connections are properly reported in server status
+ * metrics. With each new connection to the same host, the value of the metric should always
+ * be strictly increasing to reflect its rolling sum nature.
+ */
+
+(function() {
+"use strict";
+
+const numConnections = 10;
+const st = new ShardingTest({shards: 1, mongos: 1});
+const admin = st.s.getDB("admin");
+
+assert(admin.adminCommand({getParameter: 1, featureFlagConnHealthMetrics: 1})
+           .featureFlagConnHealthMetrics.value,
+       'featureFlagConnHealthMetrics should be enabled for this test');
+
+const uri = "mongodb://" + admin.getMongo().host;
+const testDB = "listenerProcessingTest";
+
+let previous = 0;
+for (var i = 0; i < numConnections; i++) {
+    const conn = new Mongo(uri);
+    const t =
+        assert.commandWorked(admin.serverStatus()).network.listenerProcessingTime["durationMicros"];
+    assert.gt(t, previous);
+    previous = t;
+}
+
+st.stop();
+})();
diff --git a/src/mongo/db/commands/server_status_servers.cpp b/src/mongo/db/commands/server_status_servers.cpp
index 34814f3a3df..80cdd8ec25d 100644
--- a/src/mongo/db/commands/server_status_servers.cpp
+++ b/src/mongo/db/commands/server_status_servers.cpp
@@ -86,10 +86,10 @@ public:
         networkCounter.append(b);
         appendMessageCompressionStats(&b);
 
+        auto svcCtx = opCtx->getServiceContext();
         {
             BSONObjBuilder section = b.subobjStart("serviceExecutors");
 
-            auto svcCtx = opCtx->getServiceContext();
             if (auto executor = transport::ServiceExecutorSynchronous::get(svcCtx)) {
                 executor->appendStats(&section);
             }
@@ -102,6 +102,8 @@ public:
                 executor->appendStats(&section);
             }
         }
+        if (auto tl = svcCtx->getTransportLayer())
+            tl->appendStats(&b);
 
         return b.obj();
     }
diff --git a/src/mongo/transport/SConscript b/src/mongo/transport/SConscript
index 02ff0750d99..8043daa2ecd 100644
--- a/src/mongo/transport/SConscript
+++ b/src/mongo/transport/SConscript
@@ -69,6 +69,8 @@ tlEnv.Library(
         'transport_layer_common',
     ],
     LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/db/server_feature_flags',
+        '$BUILD_DIR/mongo/idl/feature_flag',
         '$BUILD_DIR/mongo/idl/server_parameter',
         '$BUILD_DIR/mongo/util/net/ssl_manager',
         '$BUILD_DIR/mongo/util/options_parser/options_parser',
diff --git a/src/mongo/transport/transport_layer.h b/src/mongo/transport/transport_layer.h
index 0398da05442..4d0441b1aff 100644
--- a/src/mongo/transport/transport_layer.h
+++ b/src/mongo/transport/transport_layer.h
@@ -121,6 +121,9 @@ public:
      */
     virtual Status setup() = 0;
 
+    /** Allows a `TransportLayer` to contribute to a stats summary (e.g. `serverStatus`). */
+    virtual void appendStats(BSONObjBuilder* bob) const {}
+
     enum WhichReactor { kIngress, kEgress, kNewReactor };
     virtual ReactorHandle getReactor(WhichReactor which) = 0;
 
diff --git a/src/mongo/transport/transport_layer_asio.cpp b/src/mongo/transport/transport_layer_asio.cpp
index 85b5e816930..3c93c105dad 100644
--- a/src/mongo/transport/transport_layer_asio.cpp
+++ b/src/mongo/transport/transport_layer_asio.cpp
@@ -44,6 +44,7 @@
 #include "mongo/config.h"
 
 #include "mongo/base/system_error.h"
+#include "mongo/db/server_feature_flags_gen.h"
 #include "mongo/db/server_options.h"
 #include "mongo/db/service_context.h"
 #include "mongo/db/stats/counters.h"
@@ -1228,6 +1229,11 @@ Status TransportLayerASIO::setup() {
     return Status::OK();
 }
 
+void TransportLayerASIO::appendStats(BSONObjBuilder* bob) const {
+    if (gFeatureFlagConnHealthMetrics.isEnabledAndIgnoreFCV())
+        bob->append("listenerProcessingTime", _listenerProcessingTime.load().toBSON());
+}
+
 void TransportLayerASIO::_runListener() noexcept {
     setThreadName("listener");
 
@@ -1360,6 +1366,7 @@ ReactorHandle TransportLayerASIO::getReactor(WhichReactor which) {
 void TransportLayerASIO::_acceptConnection(GenericAcceptor& acceptor) {
     auto acceptCb = [this, &acceptor](const std::error_code& ec,
                                       ASIOSession::GenericSocket peerSocket) mutable {
+        Timer timer;
         transportLayerASIOhangBeforeAccept.pauseWhileSet();
 
         if (auto lk = stdx::lock_guard(_mutex); _isShutdown) {
@@ -1415,6 +1422,9 @@ void TransportLayerASIO::_acceptConnection(GenericAcceptor& acceptor) {
                           "error"_attr = e);
         }
 
+        // _acceptConnection() is accessed by only one thread (i.e. the listener thread), so an
+        // atomic increment is not required here
+        _listenerProcessingTime.store(_listenerProcessingTime.load() + timer.elapsed());
         _acceptConnection(acceptor);
     };
 
diff --git a/src/mongo/transport/transport_layer_asio.h b/src/mongo/transport/transport_layer_asio.h
index c0ecf4b9b80..e03425ea02f 100644
--- a/src/mongo/transport/transport_layer_asio.h
+++ b/src/mongo/transport/transport_layer_asio.h
@@ -191,6 +191,8 @@ public:
 
     void shutdown() final;
 
+    void appendStats(BSONObjBuilder* bob) const override;
+
     int listenerPort() const {
         return _listenerPort;
     }
@@ -297,6 +299,10 @@ private:
     bool _isShutdown = false;
 
     const std::unique_ptr<TimerService> _timerService;
+
+    // Tracks the cumulative time the listener spends between accepting incoming connections to
+    // handing them off to dedicated connection threads.
+    AtomicWord<Microseconds> _listenerProcessingTime;
 };
 
 }  // namespace transport
diff --git a/src/mongo/transport/transport_layer_manager.cpp b/src/mongo/transport/transport_layer_manager.cpp
index d92142685b0..0605d2c6dbd 100644
--- a/src/mongo/transport/transport_layer_manager.cpp
+++ b/src/mongo/transport/transport_layer_manager.cpp
@@ -113,6 +113,10 @@ Status TransportLayerManager::setup() {
     return Status::OK();
 }
 
+void TransportLayerManager::appendStats(BSONObjBuilder* bob) const {
+    _foreach([&](const TransportLayer* tl) { tl->appendStats(bob); });
+}
+
 Status TransportLayerManager::addAndStartTransportLayer(std::unique_ptr<TransportLayer> tl) {
     auto ptr = tl.get();
     {
diff --git a/src/mongo/transport/transport_layer_manager.h b/src/mongo/transport/transport_layer_manager.h
index 8bcd645ff5d..8e2b949b704 100644
--- a/src/mongo/transport/transport_layer_manager.h
+++ b/src/mongo/transport/transport_layer_manager.h
@@ -80,6 +80,7 @@ public:
     Status start() override;
     void shutdown() override;
     Status setup() override;
+    void appendStats(BSONObjBuilder* bob) const override;
 
     ReactorHandle getReactor(WhichReactor which) override;