SERVER-26560 Properly handle or avoid making throwing calls into ASIO within NetworkInterfaceASIO

4 files changed, 57 insertions, 9 deletions

diff --git a/src/mongo/executor/async_timer_asio.cpp b/src/mongo/executor/async_timer_asio.cpp
index d8951d22d43..3f2adf90dac 100644
--- a/src/mongo/executor/async_timer_asio.cpp
+++ b/src/mongo/executor/async_timer_asio.cpp
@@ -26,9 +26,14 @@
  *    it in the license file.
  */
 
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kASIO
+
+#include "mongo/platform/basic.h"
+
 #include "mongo/executor/async_timer_asio.h"
 
 #include "mongo/stdx/memory.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 namespace executor {
@@ -37,7 +42,11 @@ AsyncTimerASIO::AsyncTimerASIO(asio::io_service::strand* strand, Milliseconds ex
     : _strand(strand), _timer(_strand->get_io_service(), expiration.toSystemDuration()) {}
 
 void AsyncTimerASIO::cancel() {
-    _timer.cancel();
+    std::error_code ec;
+    _timer.cancel(ec);
+    if (ec) {
+        log() << "Failed to cancel timer: " << ec.message();
+    }
 }
 
 void AsyncTimerASIO::asyncWait(AsyncTimerInterface::Handler handler) {
diff --git a/src/mongo/executor/connection_pool.cpp b/src/mongo/executor/connection_pool.cpp
index 010a57973e7..ad5ee9f7338 100644
--- a/src/mongo/executor/connection_pool.cpp
+++ b/src/mongo/executor/connection_pool.cpp
@@ -508,8 +508,15 @@ void ConnectionPool::SpecificPool::spawnConnections(stdx::unique_lock<stdx::mute
 
     // While all of our inflight connections are less than our target
     while (_readyPool.size() + _processingPool.size() + _checkedOutPool.size() < target()) {
-        // make a new connection and put it in processing
-        auto handle = _parent->_factory->makeConnection(hostAndPort, _generation);
+        std::unique_ptr<ConnectionPool::ConnectionInterface> handle;
+        try {
+            // make a new connection and put it in processing
+            handle = _parent->_factory->makeConnection(hostAndPort, _generation);
+        } catch (std::system_error& e) {
+            severe() << "Failed to construct a new connection object: " << e.what();
+            fassertFailed(40336);
+        }
+
         auto connPtr = handle.get();
         _processingPool[connPtr] = std::move(handle);
 
diff --git a/src/mongo/executor/connection_pool_asio.cpp b/src/mongo/executor/connection_pool_asio.cpp
index 0ea121dd640..0788a67d7f6 100644
--- a/src/mongo/executor/connection_pool_asio.cpp
+++ b/src/mongo/executor/connection_pool_asio.cpp
@@ -25,6 +25,8 @@
  *    it in the license file.
  */
 
+#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kASIO
+
 #include "mongo/platform/basic.h"
 
 #include "mongo/executor/connection_pool_asio.h"
@@ -37,6 +39,7 @@
 #include "mongo/rpc/legacy_request_builder.h"
 #include "mongo/rpc/reply_interface.h"
 #include "mongo/stdx/memory.h"
+#include "mongo/util/log.h"
 
 namespace mongo {
 namespace executor {
@@ -59,7 +62,13 @@ void ASIOTimer::setTimeout(Milliseconds timeout, TimeoutCallback cb) {
         _cb = std::move(cb);
 
         cancelTimeout();
-        _impl.expires_after(std::min(kMaxTimerDuration, timeout).toSystemDuration());
+
+        std::error_code ec;
+        _impl.expires_after(std::min(kMaxTimerDuration, timeout).toSystemDuration(), ec);
+        if (ec) {
+            severe() << "Failed to set connection pool timer: " << ec.message();
+            fassertFailed(40333);
+        }
 
         decltype(_callbackSharedState->id) id;
         decltype(_callbackSharedState) sharedState;
@@ -104,7 +113,12 @@ void ASIOTimer::cancelTimeout() {
         stdx::lock_guard<stdx::mutex> lk(sharedState->mutex);
         if (sharedState->id != id)
             return;
-        _impl.cancel();
+
+        std::error_code ec;
+        _impl.cancel(ec);
+        if (ec) {
+            log() << "Failed to cancel connection pool timer: " << ec.message();
+        }
     });
 }
 
diff --git a/src/mongo/executor/network_interface_asio.cpp b/src/mongo/executor/network_interface_asio.cpp
index 517fb691f21..85bed4633bf 100644
--- a/src/mongo/executor/network_interface_asio.cpp
+++ b/src/mongo/executor/network_interface_asio.cpp
@@ -153,7 +153,12 @@ void NetworkInterfaceASIO::startup() {
             try {
                 LOG(2) << "The NetworkInterfaceASIO worker thread is spinning up";
                 asio::io_service::work work(_io_service);
-                _io_service.run();
+                std::error_code ec;
+                _io_service.run(ec);
+                if (ec) {
+                    severe() << "Failure in _io_service.run(): " << ec.message();
+                    fassertFailed(40335);
+                }
             } catch (...) {
                 severe() << "Uncaught exception in NetworkInterfaceASIO IO "
                             "worker thread of type: "
@@ -364,7 +369,13 @@ Status NetworkInterfaceASIO::startCommand(const TaskExecutor::CallbackHandle& cb
                 const auto adjustedTimeout = op->_request.timeout - getConnectionDuration;
                 const auto requestId = op->_request.id;
 
-                op->_timeoutAlarm = op->_owner->_timerFactory->make(&op->_strand, adjustedTimeout);
+                try {
+                    op->_timeoutAlarm =
+                        op->_owner->_timerFactory->make(&op->_strand, adjustedTimeout);
+                } catch (std::system_error& e) {
+                    severe() << "Failed to construct timer for AsyncOp: " << e.what();
+                    fassertFailed(40334);
+                }
 
                 std::shared_ptr<AsyncOp::AccessControl> access;
                 std::size_t generation;
@@ -448,8 +459,15 @@ Status NetworkInterfaceASIO::setAlarm(Date_t when, const stdx::function<void()>&
         return {ErrorCodes::ShutdownInProgress, "NetworkInterfaceASIO shutdown in progress"};
     }
 
-    // "alarm" must stay alive until it expires, hence the shared_ptr.
-    auto alarm = std::make_shared<asio::system_timer>(_io_service, when.toSystemTimePoint());
+    std::shared_ptr<asio::system_timer> alarm;
+
+    try {
+        // "alarm" must stay alive until it expires, hence the shared_ptr.
+        alarm = std::make_shared<asio::system_timer>(_io_service, when.toSystemTimePoint());
+    } catch (...) {
+        return exceptionToStatus();
+    }
+
     alarm->async_wait([alarm, this, action](std::error_code ec) {
         if (!ec) {
             return action();