SERVER-53065 enhance terminate handler exception dumps

6 files changed, 436 insertions, 36 deletions

diff --git a/src/mongo/unittest/unittest.cpp b/src/mongo/unittest/unittest.cpp
index b2daed62d82..df6cca3383a 100644
--- a/src/mongo/unittest/unittest.cpp
+++ b/src/mongo/unittest/unittest.cpp
@@ -56,6 +56,7 @@
 #include "mongo/logv2/plain_formatter.h"
 #include "mongo/platform/mutex.h"
 #include "mongo/util/assert_util.h"
+#include "mongo/util/signal_handlers_synchronous.h"
 #include "mongo/util/stacktrace.h"
 #include "mongo/util/timer.h"
 
@@ -635,5 +636,14 @@ ComparisonAssertion<op> ComparisonAssertion<op>::make(const char* theFile,
 // Provide definitions for common instantiations of ComparisonAssertion.
 INSTANTIATE_COMPARISON_ASSERTION_CTORS();
 
+namespace {
+// At startup, teach the terminate handler how to print TestAssertionFailureException.
+[[maybe_unused]] const auto earlyCall = [] {
+    globalActiveExceptionWitness().addHandler<TestAssertionFailureException>(
+        [](const auto& ex, std::ostream& os) { os << ex.toString() << "\n"; });
+    return 0;
+}();
+}  // namespace
+
 }  // namespace unittest
 }  // namespace mongo
diff --git a/src/mongo/util/SConscript b/src/mongo/util/SConscript
index 61e2aed1415..65325c565e9 100644
--- a/src/mongo/util/SConscript
+++ b/src/mongo/util/SConscript
@@ -625,6 +625,7 @@ icuEnv.CppUnitTest(
         'dns_name_test.cpp',
         'dns_query_test.cpp',
         'duration_test.cpp',
+        'dynamic_catch_test.cpp',
         'errno_util_test.cpp',
         'fail_point_test.cpp',
         'future_test_edge_cases.cpp',
diff --git a/src/mongo/util/dynamic_catch.h b/src/mongo/util/dynamic_catch.h
new file mode 100644
index 00000000000..4c14f1a2a3d
--- /dev/null
+++ b/src/mongo/util/dynamic_catch.h
@@ -0,0 +1,124 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+namespace mongo {
+
+/**
+ * Provides a mechanism for responding to an active exception with
+ * a dynamic registry of handlers for exception types.
+ *
+ * A handler is selected by probing the handlers in LIFO order until
+ * one of them accepts the exception. That handler invokes its registered
+ * callback, passing the exception along with any extra parameters to
+ * `doCatch(...)`. The types of these extra parameters are the `Args...` of
+ * `DynamicCatch<Args...>`.
+ *
+ *     DynamicCatch<A0, A1> dc;
+ *     dc.addCatch<Ex0>(f0);
+ *     dc.addCatch<Ex1>(f1);
+ *     dc.addCatch<Ex2>(f2);
+ *     dc.doCatch(a0, a1);
+ *
+ * Is roughly equivalent to:
+ *
+ *     try { throw; }
+ *     catch (const Ex2& ex) { f2(ex, a0, a1); }
+ *     catch (const Ex1& ex) { f1(ex, a0, a1); }
+ *     catch (const Ex0& ex) { f0(ex, a0, a1); }
+ */
+template <typename... Args>
+class DynamicCatch {
+public:
+    /**
+     * Add a probe for exception type `Ex`. If an exception `const Ex& ex` is
+     * caught by `doCatch(...args)`, then `f(ex, ...args)` will be invoked.
+     */
+    template <typename Ex, typename F>
+    void addCatch(F f) {
+        _handlers.push_back(std::make_unique<Handler<Ex, F>>(std::move(f)));
+    }
+
+    /**
+     * May only be called while an exception is active. Visits each handler
+     * starting from the back, until one accepts the exception. If no handler
+     * accepts the active exception, it will be rethrown by this function.
+     */
+    void doCatch(Args... args) {
+        for (auto iter = _handlers.rbegin(); iter != _handlers.rend(); ++iter)
+            if ((*iter)->tryRun(args...))
+                return;
+        throw;  // uncaught
+    }
+
+private:
+    /** A type-erased exception handler. */
+    struct AbstractHandler {
+        virtual ~AbstractHandler() = default;
+
+        /**
+         * Handlers must try to rethrow and catch the active exception. If it
+         * is caught, they may apply the given `args` to some action.
+         *
+         * Returns true if the exception was accepted by this handler.
+         */
+        virtual bool tryRun(Args... args) const = 0;
+    };
+
+    /**
+     * Handler that invokes `f(ex, args...)` if an exception `ex` of type `Ex`
+     * is active.
+     */
+    template <typename Ex, typename F>
+    struct Handler : AbstractHandler {
+        Handler(F f) : f(std::move(f)) {}
+
+        bool tryRun(Args... args) const override {
+            try {
+                throw;
+            } catch (const Ex& ex) {
+                f(ex, args...);
+                return true;
+            } catch (...) {
+                return false;
+            }
+        }
+
+        F f;
+    };
+
+    std::vector<std::unique_ptr<AbstractHandler>> _handlers;
+};
+
+}  // namespace mongo
diff --git a/src/mongo/util/dynamic_catch_test.cpp b/src/mongo/util/dynamic_catch_test.cpp
new file mode 100644
index 00000000000..664d1199a97
--- /dev/null
+++ b/src/mongo/util/dynamic_catch_test.cpp
@@ -0,0 +1,197 @@
+/**
+ *    Copyright (C) 2020-present MongoDB, Inc.
+ *
+ *    This program is free software: you can redistribute it and/or modify
+ *    it under the terms of the Server Side Public License, version 1,
+ *    as published by MongoDB, Inc.
+ *
+ *    This program is distributed in the hope that it will be useful,
+ *    but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *    Server Side Public License for more details.
+ *
+ *    You should have received a copy of the Server Side Public License
+ *    along with this program. If not, see
+ *    <http://www.mongodb.com/licensing/server-side-public-license>.
+ *
+ *    As a special exception, the copyright holders give permission to link the
+ *    code of portions of this program with the OpenSSL library under certain
+ *    conditions as described in each individual source file and distribute
+ *    linked combinations including the program with the OpenSSL library. You
+ *    must comply with the Server Side Public License in all respects for
+ *    all of the code used other than as permitted herein. If you modify file(s)
+ *    with this exception, you may extend this exception to your version of the
+ *    file(s), but you are not obligated to do so. If you do not wish to do so,
+ *    delete this exception statement from your version. If you delete this
+ *    exception statement from all source files in the program, then also delete
+ *    it in the license file.
+ */
+
+#include "mongo/util/dynamic_catch.h"
+
+#include <boost/exception/diagnostic_information.hpp>
+#include <boost/exception/exception.hpp>
+#include <fmt/format.h>
+#include <fmt/ranges.h>
+
+#include "mongo/base/error_codes.h"
+#include "mongo/base/status.h"
+#include "mongo/logv2/redaction.h"
+#include "mongo/platform/source_location.h"
+#include "mongo/stdx/thread.h"
+#include "mongo/unittest/death_test.h"
+#include "mongo/unittest/unittest.h"
+#include "mongo/util/assert_util.h"
+
+namespace mongo {
+namespace {
+
+using namespace fmt::literals;
+
+struct TestForeignRootException {
+    explicit TestForeignRootException(std::string what) : _what{std::move(what)} {}
+    const char* what() const noexcept {
+        return _what.c_str();
+    }
+    std::string _what;
+};
+
+class SpecificStdException : public std::exception {
+public:
+    explicit SpecificStdException(std::string what) : _what{std::move(what)} {}
+    const char* what() const noexcept override {
+        return _what.c_str();
+    }
+    std::string _what;
+};
+
+class DynamicCatchTest : public unittest::Test {
+public:
+    /** This is the type our terminate handler uses, so it gets extra testing attention. */
+    using StreamDynCatch = DynamicCatch<std::ostream&>;
+
+    struct SomeException {
+        std::string msg;
+    };
+
+    void installSomeHandlers(StreamDynCatch& dc) {
+        dc.addCatch<std::exception>(
+            [](auto&& ex, std::ostream& os) { os << "std::exception: " << redact(ex.what()); });
+        dc.addCatch<boost::exception>([](auto&& ex, std::ostream& os) {
+            os << "boost::exception: " << boost::diagnostic_information(ex);
+        });
+        dc.addCatch<DBException>(
+            [](auto&& ex, std::ostream& os) { os << "DBException::toString(): " << redact(ex); });
+        dc.addCatch<TestForeignRootException>(
+            [](auto&& ex, std::ostream& os) { os << "TestForeignRootException: " << ex.what(); });
+        dc.addCatch<SpecificStdException>(
+            [](auto&& ex, std::ostream& os) { os << "SpecificStdException: " << ex.what(); });
+    }
+};
+
+TEST_F(DynamicCatchTest, NoHandlers) {
+    try {
+        struct Uncatchable {};
+        throw Uncatchable{};
+    } catch (...) {
+        std::ostringstream os;
+        try {
+            StreamDynCatch{}.doCatch(os);
+            FAIL("expected a throw");
+        } catch (...) {
+        }
+        ASSERT_EQ(os.str(), "");
+    }
+}
+
+TEST_F(DynamicCatchTest, Nesting) {
+    // Test that later entries in the handler chain bind more tightly.
+    struct Base {};
+    struct Derived : Base {};
+    auto trial = [](const std::vector<std::function<void(StreamDynCatch&)>>& configChain) {
+        try {
+            throw Derived{};
+        } catch (...) {
+            StreamDynCatch dc;
+            for (auto&& config : configChain)
+                config(dc);
+            std::ostringstream os;
+            dc.doCatch(os);
+            return os.str();
+        }
+    };
+    auto catchDerived = [](StreamDynCatch& dc) {
+        dc.addCatch<Derived>([](auto&&, std::ostream& os) { os << "caught:Derived"; });
+    };
+    auto catchBase = [](StreamDynCatch& dc) {
+        dc.addCatch<Base>([](auto&&, std::ostream& os) { os << "caught:Base"; });
+    };
+    ASSERT_STRING_CONTAINS(trial({catchBase, catchDerived}), "caught:Derived");
+    ASSERT_STRING_CONTAINS(trial({catchDerived, catchBase}), "caught:Base");
+}
+
+TEST_F(DynamicCatchTest, RealisticScenarios) {
+    auto trial = [&](SourceLocationHolder loc, auto&& f, std::string expected) {
+        try {
+            f();
+            invariant(false, "`f` didn't throw");
+        } catch (...) {
+            std::ostringstream os;
+            StreamDynCatch dc;
+            installSomeHandlers(dc);
+            dc.doCatch(os);
+            ASSERT_STRING_SEARCH_REGEX(os.str(), expected) << " loc: " << loc;
+        }
+    };
+#define LOC MONGO_SOURCE_LOCATION()
+    trial(LOC, [] { throw TestForeignRootException{"oops"}; }, "TestForeignRootException: oops");
+    trial(LOC, [] { throw std::out_of_range{"testRange"}; }, "testRange");
+    trial(LOC, [] { throw SpecificStdException{"oops"}; }, "SpecificStdException: oops");
+    trial(LOC, [] { uasserted(ErrorCodes::UnknownError, "test"); }, "UnknownError.*test");
+#undef LOC
+}
+
+TEST_F(DynamicCatchTest, NoExtraArgs) {
+    DynamicCatch<> dc;
+    std::string capture = ">";
+    dc.addCatch<SomeException>([&](const auto& ex) { capture += "({})"_format(ex.msg); });
+    try {
+        throw SomeException{"oops"};
+    } catch (const SomeException&) {
+        dc.doCatch();
+    }
+    ASSERT_EQ(capture, ">(oops)");
+}
+
+TEST_F(DynamicCatchTest, MultipleArgs) {
+    std::vector<std::string> events;
+    DynamicCatch<std::vector<std::string>&, const std::string&> dc;
+    dc.addCatch<SomeException>(
+        [](const auto& ex, std::vector<std::string>& vec, const std::string& id) {
+            vec.push_back("{{{}:{}}}"_format(id, ex.msg));
+        });
+    try {
+        throw SomeException{"oops"};
+    } catch (const SomeException&) {
+        dc.doCatch(events, "here");
+        dc.doCatch(events, "andHere");
+    }
+    ASSERT_EQ("[{}]"_format(fmt::join(events, ",")), "[{here:oops},{andHere:oops}]");
+}
+
+DEATH_TEST_REGEX(DynamicCatchDeath,
+                 FatalTestAssertFromStdxThread,
+                 "1 == 2.*from stdx::thread.*TestAssertionFailureException") {
+    stdx::thread([] { ASSERT_EQ(1, 2) << "from stdx::thread"; }).join();
+}
+
+DEATH_TEST_REGEX(DynamicCatchDeath,
+                 NoActiveException,
+                 R"re(terminate\(\) called\. No exception is active)re") {
+    std::ostringstream os;
+    DynamicCatch<std::ostream&> dc;
+    dc.doCatch(os);
+}
+
+}  // namespace
+}  // namespace mongo
diff --git a/src/mongo/util/signal_handlers_synchronous.cpp b/src/mongo/util/signal_handlers_synchronous.cpp
index dbb2caacb4f..b197107c210 100644
--- a/src/mongo/util/signal_handlers_synchronous.cpp
+++ b/src/mongo/util/signal_handlers_synchronous.cpp
@@ -52,11 +52,13 @@
 #include "mongo/util/concurrency/thread_name.h"
 #include "mongo/util/debug_util.h"
 #include "mongo/util/debugger.h"
+#include "mongo/util/dynamic_catch.h"
 #include "mongo/util/exception_filter_win32.h"
 #include "mongo/util/exit_code.h"
 #include "mongo/util/quick_exit.h"
 #include "mongo/util/signal_handlers.h"
 #include "mongo/util/stacktrace.h"
+#include "mongo/util/static_immortal.h"
 #include "mongo/util/text.h"
 
 namespace mongo {
@@ -211,45 +213,13 @@ void printSignalAndBacktrace(int signalNum) {
 // exceptions
 void myTerminate() {
     MallocFreeOStreamGuard lk{};
-
-    // In c++11 we can recover the current exception to print it.
+    mallocFreeOStream << "terminate() called.";
     if (std::current_exception()) {
-        mallocFreeOStream << "terminate() called. An exception is active;"
-                          << " attempting to gather more information";
+        mallocFreeOStream << " An exception is active; attempting to gather more information";
         writeMallocFreeStreamToLog();
-
-        const std::type_info* typeInfo = nullptr;
-        try {
-            try {
-                throw;
-            } catch (const DBException& ex) {
-                typeInfo = &typeid(ex);
-                mallocFreeOStream << "DBException::toString(): " << redact(ex) << '\n';
-            } catch (const std::exception& ex) {
-                typeInfo = &typeid(ex);
-                mallocFreeOStream << "std::exception::what(): " << redact(ex.what()) << '\n';
-            } catch (const boost::exception& ex) {
-                typeInfo = &typeid(ex);
-                mallocFreeOStream << "boost::diagnostic_information(): "
-                                  << boost::diagnostic_information(ex) << '\n';
-            } catch (...) {
-                mallocFreeOStream << "A non-standard exception type was thrown\n";
-            }
-
-            if (typeInfo) {
-                const std::string name = demangleName(*typeInfo);
-                mallocFreeOStream << "Actual exception type: " << name << '\n';
-            }
-        } catch (...) {
-            mallocFreeOStream << "Exception while trying to print current exception.\n";
-            if (typeInfo) {
-                // It is possible that we failed during demangling. At least try to print the
-                // mangled name.
-                mallocFreeOStream << "Actual exception type: " << typeInfo->name() << '\n';
-            }
-        }
+        globalActiveExceptionWitness().describe(mallocFreeOStream);
     } else {
-        mallocFreeOStream << "terminate() called. No exception is active";
+        mallocFreeOStream << " No exception is active";
     }
     writeMallocFreeStreamToLog();
     printStackTrace();
@@ -396,4 +366,40 @@ int stackTraceSignal() {
 }
 #endif
 
+ActiveExceptionWitness::ActiveExceptionWitness() {
+    // Later entries in the catch chain will become the innermost catch blocks, so
+    // these are in order of increasing specificity. User-provided probes
+    // will be appended, so they will be considered more specific than any of
+    // these, which are essentially "fallback" handlers.
+    addHandler<boost::exception>([](auto&& ex, std::ostream& os) {
+        os << "boost::diagnostic_information(): " << boost::diagnostic_information(ex) << "\n";
+    });
+    addHandler<std::exception>([](auto&& ex, std::ostream& os) {
+        os << "std::exception::what(): " << redact(ex.what()) << "\n";
+    });
+    addHandler<DBException>([](auto&& ex, std::ostream& os) {
+        os << "DBException::toString(): " << redact(ex) << "\n";
+    });
+}
+
+void ActiveExceptionWitness::describe(std::ostream& os) {
+    CatchAndDescribe dc;
+    for (const auto& config : _configurators)
+        config(dc);
+    try {
+        dc.doCatch(os);
+    } catch (...) {
+        os << "A non-standard exception type was thrown\n";
+    }
+}
+
+void ActiveExceptionWitness::_exceptionTypeBlurb(const std::type_info& ex, std::ostream& os) {
+    os << "Actual exception type: " << demangleName(ex) << "\n";
+}
+
+ActiveExceptionWitness& globalActiveExceptionWitness() {
+    static StaticImmortal<ActiveExceptionWitness> v;
+    return *v;
+}
+
 }  // namespace mongo
diff --git a/src/mongo/util/signal_handlers_synchronous.h b/src/mongo/util/signal_handlers_synchronous.h
index ceb513c4600..990573cf307 100644
--- a/src/mongo/util/signal_handlers_synchronous.h
+++ b/src/mongo/util/signal_handlers_synchronous.h
@@ -29,6 +29,12 @@
 
 #pragma once
 
+#include <functional>
+#include <iosfwd>
+#include <vector>
+
+#include "mongo/util/dynamic_catch.h"
+
 namespace mongo {
 
 /**
@@ -69,4 +75,60 @@ int stackTraceSignal();
 #endif
 
 
+/**
+ * Analyzes the active exception, describing it to an ostream.
+ * Consults a dynamic registry of exception handlers.
+ * See `util/dynamic_catch.h`.
+ */
+class ActiveExceptionWitness {
+public:
+    /** Default constructor creates handlers for some basic exception types. */
+    ActiveExceptionWitness();
+
+    /**
+     * Called at startup to teach our std::terminate handler how to print a
+     * diagnostic for decoupled types of exceptions (e.g. in third_party, in
+     * layers above base, or outside of the server codebase).
+     *
+     * This is not thread-safe, call at startup before multithreading. The
+     * probes are evaluated in order so that later entries here will supersede
+     * earlier entries and match more tightly in the catch hierarchy.
+     */
+    template <typename Ex>
+    void addHandler(std::function<void(const Ex&, std::ostream&)> handler) {
+        _configurators.push_back([=](CatchAndDescribe& dc) {
+            dc.addCatch<Ex>([=](const Ex& ex, std::ostream& os) {
+                handler(ex, os);
+                _exceptionTypeBlurb(typeid(ex), os);
+            });
+        });
+    }
+
+    /**
+     * Writes a description of the active exception to `os`, using built-in
+     * exception probes, augmented by any configured exception probes given by calls to
+     * `addHandler`.
+     *
+     * Called by our std::terminate handler when it detects an active
+     * exception. The active exception is probably related to why the process
+     * is terminating, but not necessarily. Consult a dynamic registry of
+     * exception types to diagnose the active exception.
+     */
+    void describe(std::ostream& os);
+
+private:
+    /**
+     * A DynamicCatch that provides handlers with an std::ostream& into which
+     * to describe the exception they've caught.
+     */
+    using CatchAndDescribe = DynamicCatch<std::ostream&>;
+
+    static void _exceptionTypeBlurb(const std::type_info& ex, std::ostream& os);
+
+    std::vector<std::function<void(CatchAndDescribe&)>> _configurators;
+};
+
+/** The singleton ActiveExceptionWitness. */
+ActiveExceptionWitness& globalActiveExceptionWitness();
+
 }  // namespace mongo