SERVER-26560 Properly handle or avoid making throwing calls into ASIO within NetworkInterfaceASIO

4 files changed, 57 insertions, 8 deletions

diff --git a/src/mongo/executor/async_timer_asio.cpp b/src/mongo/executor/async_timer_asio.cpp
index 293ab3e137a..d0445f053c3 100644
--- a/src/mongo/executor/async_timer_asio.cpp
+++ b/src/mongo/executor/async_timer_asio.cpp
@@ -26,9 +26,14 @@
  *    it in the license file.
  */
 
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kASIO
+
+#include "mongo/platform/basic.h"
+
 #include "mongo/executor/async_timer_asio.h"
 
 #include "mongo/stdx/memory.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 namespace executor {
@@ -37,7 +42,11 @@ AsyncTimerASIO::AsyncTimerASIO(asio::io_service::strand* strand, Milliseconds ex
     : _strand(strand), _timer(_strand->get_io_service(), expiration) {}
 
 void AsyncTimerASIO::cancel() {
-    _timer.cancel();
+    std::error_code ec;
+    _timer.cancel(ec);
+    if (ec) {
+        log() << "Failed to cancel timer: " << ec.message();
+    }
 }
 
 void AsyncTimerASIO::asyncWait(AsyncTimerInterface::Handler handler) {
diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index fb88bef8577..7b0e206469f 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -495,8 +495,15 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock<stdx::mute
 
     // While all of our inflight connections are less than our target
     while (_readyPool.size() + _processingPool.size() + _checkedOutPool.size() < target()) {
-        // make a new connection and put it in processing
-        auto handle = _parent->_factory->makeConnection(hostAndPort, _generation);
+        std::unique_ptr<ConnectionPool::ConnectionInterface> handle;
+        try {
+            // make a new connection and put it in processing
+            handle = _parent->_factory->makeConnection(hostAndPort, _generation);
+        } catch (std::system_error& e) {
+            severe() << "Failed to construct a new connection object: " << e.what();
+            fassertFailed(40336);
+        }
+
         auto connPtr = handle.get();
         _processingPool[connPtr] = std::move(handle);
 
diff --git a/src/mongo/executor/connection_pool_asio.cpp b/src/mongo/executor/connection_pool_asio.cpp
index 45be3537007..8a7bc826355 100644
--- a/src/mongo/executor/connection_pool_asio.cpp
+++ b/src/mongo/executor/connection_pool_asio.cpp
@@ -25,6 +25,8 @@
  *    it in the license file.
  */
 
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kASIO
+
 #include "mongo/platform/basic.h"
 
 #include "mongo/executor/connection_pool_asio.h"
@@ -37,6 +39,7 @@
 #include "mongo/rpc/legacy_request_builder.h"
 #include "mongo/rpc/reply_interface.h"
 #include "mongo/stdx/memory.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 namespace executor {
@@ -57,7 +60,13 @@ void ASIOTimer::setTimeout(Milliseconds timeout, TimeoutCallback cb) {
         _cb = std::move(cb);
 
         cancelTimeout();
-        _impl.expires_after(timeout);
+
+        std::error_code ec;
+        _impl.expires_after(timeout, ec);
+        if (ec) {
+            severe() << "Failed to set connection pool timer: " << ec.message();
+            fassertFailed(40333);
+        }
 
         decltype(_callbackSharedState->id) id;
         decltype(_callbackSharedState) sharedState;
@@ -102,7 +111,12 @@ void ASIOTimer::cancelTimeout() {
         stdx::lock_guard<stdx::mutex> lk(sharedState->mutex);
         if (sharedState->id != id)
             return;
-        _impl.cancel();
+
+        std::error_code ec;
+        _impl.cancel(ec);
+        if (ec) {
+            log() << "Failed to cancel connection pool timer: " << ec.message();
+        }
     });
 }
 
diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp
index d70acd649a5..3900b92f4e5 100644
--- a/src/mongo/executor/network_interface_asio.cpp
+++ b/src/mongo/executor/network_interface_asio.cpp
@@ -113,7 +113,12 @@ void NetworkInterfaceASIO::startup() {
             try {
                 LOG(2) << "The NetworkInterfaceASIO worker thread is spinning up";
                 asio::io_service::work work(_io_service);
-                _io_service.run();
+                std::error_code ec;
+                _io_service.run(ec);
+                if (ec) {
+                    severe() << "Failure in _io_service.run(): " << ec.message();
+                    fassertFailed(40335);
+                }
             } catch (...) {
                 severe() << "Uncaught exception in NetworkInterfaceASIO IO "
                             "worker thread of type: " << exceptionToStatus();
@@ -272,7 +277,13 @@ void NetworkInterfaceASIO::startCommand(const TaskExecutor::CallbackHandle& cbHa
                 const auto adjustedTimeout = op->_request.timeout - getConnectionDuration;
                 const auto requestId = op->_request.id;
 
-                op->_timeoutAlarm = op->_owner->_timerFactory->make(&op->_strand, adjustedTimeout);
+                try {
+                    op->_timeoutAlarm =
+                        op->_owner->_timerFactory->make(&op->_strand, adjustedTimeout);
+                } catch (std::system_error& e) {
+                    severe() << "Failed to construct timer for AsyncOp: " << e.what();
+                    fassertFailed(40334);
+                }
 
                 std::shared_ptr<AsyncOp::AccessControl> access;
                 std::size_t generation;
@@ -350,7 +361,15 @@ void NetworkInterfaceASIO::cancelAllCommands() {
 
 void NetworkInterfaceASIO::setAlarm(Date_t when, const stdx::function<void()>& action) {
     // "alarm" must stay alive until it expires, hence the shared_ptr.
-    auto alarm = std::make_shared<asio::steady_timer>(_io_service, when - now());
+    std::shared_ptr<asio::steady_timer> alarm;
+
+    try {
+        alarm = std::make_shared<asio::steady_timer>(_io_service, when - now());
+    } catch (std::system_error& e) {
+        severe() << "setAlarm() could not construct a timer" << e.what();
+        fassertFailed(40337);
+    }
+
     alarm->async_wait([alarm, this, action](std::error_code ec) {
         if (!ec) {
             return action();