diff options
author | Blake Oler <blake.oler@mongodb.com> | 2021-10-14 20:40:06 +0000 |
---|---|---|
committer | Evergreen Agent <no-reply@evergreen.mongodb.com> | 2022-03-16 19:39:43 +0000 |
commit | ea8eccd0a2210d84493b516e470ebb3df2beca8b (patch) | |
tree | 313bd87b9a61ee8271a084ae3d91cb4220c2c85d /src/mongo/transport | |
parent | ca1a96490687e5bff52c35bb5eec8408e0912291 (diff) | |
download | mongo-ea8eccd0a2210d84493b516e470ebb3df2beca8b.tar.gz |
SERVER-57466 Swallow connection reset-related errors received during ASIO session establishment
(cherry picked from commit ffededcdc670d83f4fdf5819f5ab1d9c5750c67c)
SERVER-57466 fix windows compile
(cherry picked from commit 37595d5ee16f06c964b9d416e5847b99c1a537d8)
Diffstat (limited to 'src/mongo/transport')
-rw-r--r-- | src/mongo/transport/asio_utils.cpp | 26 | ||||
-rw-r--r-- | src/mongo/transport/asio_utils.h | 36 | ||||
-rw-r--r-- | src/mongo/transport/session_asio.cpp | 23 | ||||
-rw-r--r-- | src/mongo/transport/transport_layer_asio.cpp | 40 | ||||
-rw-r--r-- | src/mongo/transport/transport_layer_asio_test.cpp | 76 |
5 files changed, 136 insertions, 65 deletions
diff --git a/src/mongo/transport/asio_utils.cpp b/src/mongo/transport/asio_utils.cpp index c954dc829eb..8e8368ff586 100644 --- a/src/mongo/transport/asio_utils.cpp +++ b/src/mongo/transport/asio_utils.cpp @@ -303,19 +303,19 @@ boost::optional<std::array<std::uint8_t, 7>> checkTLSRequest(const asio::const_b void failedSetSocketOption(const std::system_error& ex, StringData note, - BSONObj optionDescription) { - LOGV2_INFO(5693100, - "Asio socket.set_option failed with std::system_error", - "note"_attr = note, - "option"_attr = optionDescription, - "error"_attr = [&ex] { - return BSONObjBuilder{} - .append("what", ex.what()) - .append("message", ex.code().message()) - .append("category", ex.code().category().name()) - .append("value", ex.code().value()) - .obj(); - }()); + BSONObj optionDescription, + logv2::LogSeverity errorLogSeverity) { + LOGV2_DEBUG(5693100, + errorLogSeverity.toInt(), + "Asio socket.set_option failed with std::system_error", + "note"_attr = note, + "option"_attr = optionDescription, + "error"_attr = BSONObjBuilder{} + .append("what", ex.what()) + .append("message", ex.code().message()) + .append("category", ex.code().category().name()) + .append("value", ex.code().value()) + .obj()); } } // namespace mongo::transport diff --git a/src/mongo/transport/asio_utils.h b/src/mongo/transport/asio_utils.h index 2475e90154b..fc63d800492 100644 --- a/src/mongo/transport/asio_utils.h +++ b/src/mongo/transport/asio_utils.h @@ -39,6 +39,7 @@ #include "mongo/base/string_data.h" #include "mongo/base/system_error.h" #include "mongo/config.h" +#include "mongo/logv2/log_severity.h" #include "mongo/stdx/type_traits.h" #include "mongo/util/errno_util.h" #include "mongo/util/future.h" @@ -89,7 +90,10 @@ boost::optional<std::array<std::uint8_t, 7>> checkTLSRequest(const asio::const_b * setSocketOption failed. Log the error. * This is in the .cpp file just to keep LOGV2 out of this header. */ -void failedSetSocketOption(const std::system_error& ex, StringData note, BSONObj optionDescription); +void failedSetSocketOption(const std::system_error& ex, + StringData note, + BSONObj optionDescription, + logv2::LogSeverity errorLogSeverity); /** * Calls Asio `socket.set_option(opt)` with better failure diagnostics. @@ -97,16 +101,20 @@ void failedSetSocketOption(const std::system_error& ex, StringData note, BSONObj * Emits a log message about what option was attempted and what went wrong with * it. The `note` string should uniquely identify the source of the call. * - * Two overloads are provided, matching the Asio `socket.set_option` overloads. + * Two overloads are provided matching the Asio `socket.set_option` overloads, with an additional + * parameter to indicate the level at which the failure diagnostics should logged. * - * setSocketOption(socket, opt, note) - * setSocketOption(socket, opt, ec, note) + * setSocketOption(socket, opt, note, errorLogSeverity) + * setSocketOption(socket, opt, note, errorLogSeverity, ec) * * If an `ec` is provided, errors are reported by mutating it. * Otherwise, the Asio `std::system_error` exception is rethrown. */ template <typename Socket, typename Option> -void setSocketOption(Socket& socket, const Option& opt, StringData note) { +void setSocketOption(Socket& socket, + const Option& opt, + StringData note, + logv2::LogSeverity errorLogSeverity) { try { socket.set_option(opt); } catch (const std::system_error& ex) { @@ -117,15 +125,27 @@ void setSocketOption(Socket& socket, const Option& opt, StringData note) { .append("data", hexdump(opt.data(p), opt.size(p))) .obj(); }(); - failedSetSocketOption(ex, note, optionDescription); + auto&& p = socket.local_endpoint().protocol(); + failedSetSocketOption(ex, + note, + BSONObjBuilder{} + .append("level", opt.level(p)) + .append("name", opt.name(p)) + .append("data", hexdump(opt.data(p), opt.size(p))) + .obj(), + errorLogSeverity); throw; } } template <typename Socket, typename Option> -void setSocketOption(Socket& socket, const Option& opt, std::error_code& ec, StringData note) { +void setSocketOption(Socket& socket, + const Option& opt, + StringData note, + logv2::LogSeverity errorLogSeverity, + std::error_code& ec) { try { - setSocketOption(socket, opt, note); + setSocketOption(socket, opt, note, errorLogSeverity); } catch (const std::system_error& ex) { ec = ex.code(); } diff --git a/src/mongo/transport/session_asio.cpp b/src/mongo/transport/session_asio.cpp index b4f27331874..80f9e87aac9 100644 --- a/src/mongo/transport/session_asio.cpp +++ b/src/mongo/transport/session_asio.cpp @@ -95,10 +95,11 @@ TransportLayerASIO::ASIOSession::ASIOSession( _tl(tl), _isIngressSession(isIngressSession) { auto family = endpointToSockAddr(_socket.local_endpoint()).getType(); + auto sev = logv2::LogSeverity::Debug(3); if (family == AF_INET || family == AF_INET6) { - setSocketOption(_socket, asio::ip::tcp::no_delay(true), "session no delay"); - setSocketOption(_socket, asio::socket_base::keep_alive(true), "session keep alive"); - setSocketKeepAliveParams(_socket.native_handle()); + setSocketOption(_socket, asio::ip::tcp::no_delay(true), "session no delay", sev); + setSocketOption(_socket, asio::socket_base::keep_alive(true), "session keep alive", sev); + setSocketKeepAliveParams(_socket.native_handle(), sev); } _localAddr = endpointToSockAddr(_socket.local_endpoint()); @@ -129,8 +130,8 @@ TransportLayerASIO::ASIOSession::ASIOSession( #endif } catch (const DBException&) { throw; -} catch (const asio::system_error& error) { - uasserted(ErrorCodes::SocketException, error.what()); +} catch (const asio::system_error&) { + throw; } catch (...) { uasserted(50797, str::stream() << "Unknown exception while configuring socket."); } @@ -324,16 +325,20 @@ void TransportLayerASIO::ASIOSession::ensureSync() { // Change boost::none (which means no timeout) into a zero value for the socket option, // which also means no timeout. auto timeout = _configuredTimeout.value_or(Milliseconds{0}); - setSocketOption( - getSocket(), ASIOSocketTimeoutOption<SO_SNDTIMEO>(timeout), ec, "session send timeout"); + setSocketOption(getSocket(), + ASIOSocketTimeoutOption<SO_SNDTIMEO>(timeout), + "session send timeout", + logv2::LogSeverity::Info(), + ec); if (auto status = errorCodeToStatus(ec); !status.isOK()) { tasserted(5342000, status.reason()); } setSocketOption(getSocket(), ASIOSocketTimeoutOption<SO_RCVTIMEO>(timeout), - ec, - "session receive timeout"); + "session receive timeout", + logv2::LogSeverity::Info(), + ec); if (auto status = errorCodeToStatus(ec); !status.isOK()) { tasserted(5342001, status.reason()); } diff --git a/src/mongo/transport/transport_layer_asio.cpp b/src/mongo/transport/transport_layer_asio.cpp index 1a8304aeca8..6047ffa7561 100644 --- a/src/mongo/transport/transport_layer_asio.cpp +++ b/src/mongo/transport/transport_layer_asio.cpp @@ -101,6 +101,7 @@ boost::optional<Status> maybeTcpFastOpenStatus; } // namespace MONGO_FAIL_POINT_DEFINE(transportLayerASIOasyncConnectTimesOut); +MONGO_FAIL_POINT_DEFINE(transportLayerASIOhangBeforeAccept); #ifdef MONGO_CONFIG_SSL SSLConnectionContext::~SSLConnectionContext() = default; @@ -567,8 +568,11 @@ StatusWith<TransportLayerASIO::ASIOSessionHandle> TransportLayerASIO::_doSyncCon #ifdef TCP_FASTOPEN_CONNECT const auto family = protocol.family(); if ((family == AF_INET) || (family == AF_INET6)) { - setSocketOption( - sock, TCPFastOpenConnect(gTCPFastOpenClient), ec, "connect (sync) TCP fast open"); + setSocketOption(sock, + TCPFastOpenConnect(gTCPFastOpenClient), + "connect (sync) TCP fast open", + logv2::LogSeverity::Info(), + ec); if (tcpFastOpenIsConfigured) { return errorCodeToStatus(ec); } @@ -721,8 +725,9 @@ Future<SessionHandle> TransportLayerASIO::asyncConnect( std::error_code ec; setSocketOption(connector->socket, TCPFastOpenConnect(gTCPFastOpenClient), - ec, - "connect (async) TCP fast open"); + "connect (async) TCP fast open", + logv2::LogSeverity::Info(), + ec); if (tcpFastOpenIsConfigured) { return futurize(ec); } @@ -1017,13 +1022,19 @@ Status TransportLayerASIO::setup() { throw; } - setSocketOption(acceptor, GenericAcceptor::reuse_address(true), "acceptor reuse address"); + setSocketOption(acceptor, + GenericAcceptor::reuse_address(true), + "acceptor reuse address", + logv2::LogSeverity::Info()); std::error_code ec; #ifdef TCP_FASTOPEN if (gTCPFastOpenServer && ((addr.family() == AF_INET) || (addr.family() == AF_INET6))) { - setSocketOption( - acceptor, TCPFastOpen(gTCPFastOpenQueueSize), ec, "acceptor TCP fast open"); + setSocketOption(acceptor, + TCPFastOpen(gTCPFastOpenQueueSize), + "acceptor TCP fast open", + logv2::LogSeverity::Info(), + ec); if (tcpFastOpenIsConfigured) { return errorCodeToStatus(ec); } @@ -1031,7 +1042,8 @@ Status TransportLayerASIO::setup() { } #endif if (addr.family() == AF_INET6) { - setSocketOption(acceptor, asio::ip::v6_only(true), "acceptor v6 only"); + setSocketOption( + acceptor, asio::ip::v6_only(true), "acceptor v6 only", logv2::LogSeverity::Info()); } acceptor.non_blocking(true, ec); @@ -1216,6 +1228,8 @@ ReactorHandle TransportLayerASIO::getReactor(WhichReactor which) { void TransportLayerASIO::_acceptConnection(GenericAcceptor& acceptor) { auto acceptCb = [this, &acceptor](const std::error_code& ec, ASIOSession::GenericSocket peerSocket) mutable { + transportLayerASIOhangBeforeAccept.pauseWhileSet(); + if (auto lk = stdx::lock_guard(_mutex); _isShutdown) { return; } @@ -1243,6 +1257,16 @@ void TransportLayerASIO::_acceptConnection(GenericAcceptor& acceptor) { std::shared_ptr<ASIOSession> session( new ASIOSession(this, std::move(peerSocket), true)); _sep->startSession(std::move(session)); + } catch (const asio::system_error& e) { + // Swallow connection reset errors. Connection reset errors classically present as + // asio::error::eof, but can bubble up as asio::error::invalid_argument when calling + // into socket.set_option(). + if (e.code() != asio::error::eof && e.code() != asio::error::invalid_argument) { + LOGV2_WARNING(5746600, + "Error accepting new connection: {error}", + "Error accepting new connection", + "error"_attr = e.code().message()); + } } catch (const DBException& e) { LOGV2_WARNING(23023, "Error accepting new connection: {error}", diff --git a/src/mongo/transport/transport_layer_asio_test.cpp b/src/mongo/transport/transport_layer_asio_test.cpp index 3d422f9c8df..7bfc2e3f5c4 100644 --- a/src/mongo/transport/transport_layer_asio_test.cpp +++ b/src/mongo/transport/transport_layer_asio_test.cpp @@ -104,9 +104,8 @@ class SimpleConnectionThread { public: explicit SimpleConnectionThread(int port) : _port(port) { _thr = stdx::thread{[&] { - Socket s; auto sa = SockAddr::create("localhost", _port, AF_INET); - s.connect(sa); + _s.connect(sa); LOGV2(23034, "connection: port {port}", "port"_attr = _port); stdx::unique_lock<Latch> lk(_mutex); _cv.wait(lk, [&] { return _stop; }); @@ -114,6 +113,19 @@ public: }}; } + void forceCloseSocket() { + // Setting linger on to a zero-value timeout causes the socket to send an RST packet to the + // recipient side when closing the connection. + struct linger sl = {1, 0}; +#ifdef _WIN32 + char* pval = reinterpret_cast<char*>(&sl); +#else + void* pval = &sl; +#endif + ASSERT(!setsockopt(_s.rawFD(), SOL_SOCKET, SO_LINGER, pval, sizeof(sl))); + _s.close(); + } + void stop() { { stdx::unique_lock<Latch> lk(_mutex); @@ -130,12 +142,12 @@ private: stdx::condition_variable _cv; stdx::thread _thr; bool _stop = false; + + Socket _s; int _port; }; -TEST(TransportLayerASIO, PortZeroConnect) { - ServiceEntryPointUtil sepu; - +std::unique_ptr<transport::TransportLayerASIO> makeAndStartTL(ServiceEntryPoint* sep) { auto options = [] { ServerGlobalParams params; params.noUnixSocket = true; @@ -147,20 +159,46 @@ TEST(TransportLayerASIO, PortZeroConnect) { return opts; }(); - transport::TransportLayerASIO tla(options, &sepu); - sepu.setTransportLayer(&tla); + auto tla = std::make_unique<transport::TransportLayerASIO>(options, sep); + ASSERT_OK(tla->setup()); + ASSERT_OK(tla->start()); + + return tla; +} + +TEST(TransportLayerASIO, PortZeroConnect) { + ServiceEntryPointUtil sepu; + auto tla = makeAndStartTL(&sepu); - ASSERT_OK(tla.setup()); - ASSERT_OK(tla.start()); - int port = tla.listenerPort(); + int port = tla->listenerPort(); ASSERT_GT(port, 0); LOGV2(23038, "TransportLayerASIO.listenerPort() is {port}", "port"_attr = port); SimpleConnectionThread connect_thread(port); sepu.waitForConnect(); + + ASSERT_EQ(sepu.numOpenSessions(), 1); connect_thread.stop(); sepu.endAllSessions({}); - tla.shutdown(); + tla->shutdown(); +} + +TEST(TransportLayerASIO, TCPResetAfterConnectionIsSilentlySwallowed) { + ServiceEntryPointUtil sepu; + auto tla = makeAndStartTL(&sepu); + + auto hangBeforeAcceptFp = globalFailPointRegistry().find("transportLayerASIOhangBeforeAccept"); + auto timesEntered = hangBeforeAcceptFp->setMode(FailPoint::alwaysOn); + + SimpleConnectionThread connect_thread(tla->listenerPort()); + hangBeforeAcceptFp->waitForTimesEntered(timesEntered + 1); + + connect_thread.forceCloseSocket(); + hangBeforeAcceptFp->setMode(FailPoint::off); + + ASSERT_EQ(sepu.numOpenSessions(), 0); + connect_thread.stop(); + tla->shutdown(); } class TimeoutSEP : public ServiceEntryPoint { @@ -291,22 +329,6 @@ private: asio::ip::tcp::endpoint _endpoint; }; -std::unique_ptr<transport::TransportLayerASIO> makeAndStartTL(ServiceEntryPoint* sep) { - auto options = [] { - ServerGlobalParams params; - params.noUnixSocket = true; - transport::TransportLayerASIO::Options opts(¶ms); - opts.port = 0; - return opts; - }(); - - auto tla = std::make_unique<transport::TransportLayerASIO>(options, sep); - ASSERT_OK(tla->setup()); - ASSERT_OK(tla->start()); - - return tla; -} - /* check that timeouts actually time out */ TEST(TransportLayerASIO, SourceSyncTimeoutTimesOut) { TimeoutSyncSEP sep(TimeoutSyncSEP::kShouldTimeout); |