SERVER-56931 instrumentation for Asio setOption failures

3 files changed, 80 insertions, 15 deletions

diff --git a/src/mongo/transport/asio_utils.h b/src/mongo/transport/asio_utils.h
index 48a12138432..e903843d43c 100644
--- a/src/mongo/transport/asio_utils.h
+++ b/src/mongo/transport/asio_utils.h
@@ -29,21 +29,23 @@
 
 #pragma once
 
+#ifndef _WIN32
+#include <sys/poll.h>
+#endif
+
+#include <asio.hpp>
+
 #include "mongo/base/status.h"
+#include "mongo/base/string_data.h"
 #include "mongo/base/system_error.h"
 #include "mongo/config.h"
 #include "mongo/util/errno_util.h"
 #include "mongo/util/future.h"
+#include "mongo/util/hex.h"
 #include "mongo/util/net/hostandport.h"
 #include "mongo/util/net/sockaddr.h"
 #include "mongo/util/net/ssl_manager.h"
 
-#ifndef _WIN32
-#include <sys/poll.h>
-#endif  // ndef _WIN32
-
-#include <asio.hpp>
-
 namespace mongo {
 namespace transport {
 
@@ -322,6 +324,58 @@ boost::optional<std::array<std::uint8_t, 7>> checkTLSRequest(const Buffer& buffe
 #endif
 
 /**
+ * Calls Asio `socket.set_option(opt)` with better failure diagnostics.
+ * To be used instead of Asio `socket.set_option``, because errors are hard to diagnose.
+ * Emits a log message about what option was attempted and what went wrong with
+ * it. The `note` string should uniquely identify the source of the call.
+ *
+ * Two overloads are provided, matching the Asio `socket.set_option` overloads.
+ *
+ *     setSocketOption(socket, opt, note)
+ *     setSocketOption(socket, opt, ec, note)
+ *
+ * If an `ec` is provided, errors are reported by mutating it.
+ * Otherwise, the Asio `std::system_error` exception is rethrown.
+ */
+template <typename Socket, typename Option>
+void setSocketOption(Socket& socket, const Option& opt, StringData note) {
+    try {
+        socket.set_option(opt);
+    } catch (const std::system_error& ex) {
+        LOGV2_INFO(5693100,
+                   "Asio socket.set_option failed with std::system_error",
+                   "note"_attr = note,
+                   "option"_attr =
+                       [&opt, p = socket.local_endpoint().protocol()] {
+                           return BSONObjBuilder{}
+                               .append("level", opt.level(p))
+                               .append("name", opt.name(p))
+                               .append("data", hexdump(opt.data(p), opt.size(p)))
+                               .obj();
+                       }(),
+                   "error"_attr =
+                       [&ex] {
+                           return BSONObjBuilder{}
+                               .append("what", ex.what())
+                               .append("message", ex.code().message())
+                               .append("category", ex.code().category().name())
+                               .append("value", ex.code().value())
+                               .obj();
+                       }());
+        throw;
+    }
+}
+
+template <typename Socket, typename Option>
+void setSocketOption(Socket& socket, const Option& opt, std::error_code& ec, StringData note) {
+    try {
+        setSocketOption(socket, opt, note);
+    } catch (const std::system_error& ex) {
+        ec = ex.code();
+    }
+}
+
+/**
  * Pass this to asio functions in place of a callback to have them return a Future<T>. This behaves
  * similarly to asio::use_future_t, however it returns a mongo::Future<T> rather than a
  * std::future<T>.
diff --git a/src/mongo/transport/session_asio.h b/src/mongo/transport/session_asio.h
index 8007aa3d781..a91e710f6f9 100644
--- a/src/mongo/transport/session_asio.h
+++ b/src/mongo/transport/session_asio.h
@@ -94,8 +94,8 @@ public:
           _isIngressSession(isIngressSession) {
         auto family = endpointToSockAddr(_socket.local_endpoint()).getType();
         if (family == AF_INET || family == AF_INET6) {
-            _socket.set_option(asio::ip::tcp::no_delay(true));
-            _socket.set_option(asio::socket_base::keep_alive(true));
+            setSocketOption(_socket, asio::ip::tcp::no_delay(true), "session no delay");
+            setSocketOption(_socket, asio::socket_base::keep_alive(true), "session keep alive");
             setSocketKeepAliveParams(_socket.native_handle());
         }
 
@@ -357,12 +357,18 @@ protected:
             // Change boost::none (which means no timeout) into a zero value for the socket option,
             // which also means no timeout.
             auto timeout = _configuredTimeout.value_or(Milliseconds{0});
-            getSocket().set_option(ASIOSocketTimeoutOption<SO_SNDTIMEO>(timeout), ec);
+            setSocketOption(getSocket(),
+                            ASIOSocketTimeoutOption<SO_SNDTIMEO>(timeout),
+                            ec,
+                            "session send timeout");
             if (auto status = errorCodeToStatus(ec); !status.isOK()) {
                 tasserted(5342000, status.reason());
             }
 
-            getSocket().set_option(ASIOSocketTimeoutOption<SO_RCVTIMEO>(timeout), ec);
+            setSocketOption(getSocket(),
+                            ASIOSocketTimeoutOption<SO_RCVTIMEO>(timeout),
+                            ec,
+                            "session receive timeout");
             if (auto status = errorCodeToStatus(ec); !status.isOK()) {
                 tasserted(5342001, status.reason());
             }
diff --git a/src/mongo/transport/transport_layer_asio.cpp b/src/mongo/transport/transport_layer_asio.cpp
index 205a3066949..291d2dc2108 100644
--- a/src/mongo/transport/transport_layer_asio.cpp
+++ b/src/mongo/transport/transport_layer_asio.cpp
@@ -538,7 +538,8 @@ StatusWith<TransportLayerASIO::ASIOSessionHandle> TransportLayerASIO::_doSyncCon
 #ifdef TCP_FASTOPEN_CONNECT
     const auto family = protocol.family();
     if ((family == AF_INET) || (family == AF_INET6)) {
-        sock.set_option(TCPFastOpenConnect(gTCPFastOpenClient), ec);
+        setSocketOption(
+            sock, TCPFastOpenConnect(gTCPFastOpenClient), ec, "connect (sync) TCP fast open");
         if (tcpFastOpenIsConfigured) {
             return errorCodeToStatus(ec);
         }
@@ -689,7 +690,10 @@ Future<SessionHandle> TransportLayerASIO::asyncConnect(
 
 #ifdef TCP_FASTOPEN_CONNECT
             std::error_code ec;
-            connector->socket.set_option(TCPFastOpenConnect(gTCPFastOpenClient), ec);
+            setSocketOption(connector->socket,
+                            TCPFastOpenConnect(gTCPFastOpenClient),
+                            ec,
+                            "connect (async) TCP fast open");
             if (tcpFastOpenIsConfigured) {
                 return futurize(ec);
             }
@@ -984,12 +988,13 @@ Status TransportLayerASIO::setup() {
 
             throw;
         }
-        acceptor.set_option(GenericAcceptor::reuse_address(true));
+        setSocketOption(acceptor, GenericAcceptor::reuse_address(true), "acceptor reuse address");
 
         std::error_code ec;
 #ifdef TCP_FASTOPEN
         if (gTCPFastOpenServer && ((addr.family() == AF_INET) || (addr.family() == AF_INET6))) {
-            acceptor.set_option(TCPFastOpen(gTCPFastOpenQueueSize), ec);
+            setSocketOption(
+                acceptor, TCPFastOpen(gTCPFastOpenQueueSize), ec, "acceptor TCP fast open");
             if (tcpFastOpenIsConfigured) {
                 return errorCodeToStatus(ec);
             }
@@ -997,7 +1002,7 @@ Status TransportLayerASIO::setup() {
         }
 #endif
         if (addr.family() == AF_INET6) {
-            acceptor.set_option(asio::ip::v6_only(true));
+            setSocketOption(acceptor, asio::ip::v6_only(true), "acceptor v6 only");
         }
 
         acceptor.non_blocking(true, ec);