SERVER-27877 Write function to deduplicate stacks from different threads in GDB

2 files changed, 87 insertions, 4 deletions

diff --git a/buildscripts/gdb/mongo.py b/buildscripts/gdb/mongo.py
index e6f4209bb5e..00e45712cbc 100644
--- a/buildscripts/gdb/mongo.py
+++ b/buildscripts/gdb/mongo.py
@@ -1,8 +1,8 @@
 """GDB Pretty-printers and utility commands for MongoDB
 """
+from __future__ import print_function
+
 import gdb.printing
-import glob
-import os
 import sys
 
 try:
@@ -20,6 +20,7 @@ def get_unique_ptr(obj):
     """Read the value of a libstdc++ std::unique_ptr"""
     return obj["_M_t"]['_M_head_impl']
 
+
 ###################################################################################################
 #
 # Pretty-Printers
@@ -355,13 +356,78 @@ class MongoDBAnalyze(gdb.Command):
             # Note that output will go to mongod's standard output, not the debugger output window
             gdb.execute("call ('mongo::(anonymous namespace)::globalLockManager').dump()",
                         from_tty=False, to_string=False)
-        except gdb.Error as gdberr:
+        except gdb.error as gdberr:
             print("Ignoring error '%s'" % str(gdberr))
 
 # Register command
 MongoDBAnalyze()
 
 
+class MongoDBUniqueStack(gdb.Command):
+    """Print unique stack traces of all threads in current process"""
+
+    def __init__(self):
+        register_mongo_command(self, "mongodb-uniqstack", gdb.COMMAND_DATA)
+
+    def invoke(self, arg, _from_tty):
+        stacks = {}
+        if not arg:
+            arg = 'bt' # default to 'bt'
+
+        current_thread = gdb.selected_thread()
+        try:
+            for thread in gdb.selected_inferior().threads():
+                if not thread.is_valid():
+                    continue
+                thread.switch()
+                self._process_thread_stack(arg, stacks, thread)
+            self._dump_unique_stacks(stacks)
+        finally:
+            if current_thread and current_thread.is_valid():
+                current_thread.switch()
+
+    def _process_thread_stack(self, arg, stacks, thread):
+        thread_info = {} # thread dict to hold per thread data
+        thread_info['frames'] = [] # the complete backtrace per thread from gdb
+        thread_info['functions'] = [] # list of function names from frames
+        thread_info['pthread'] = hex(int(gdb.parse_and_eval("(uint64_t)pthread_self()")))
+
+        frame = gdb.newest_frame()
+        while frame:
+            thread_info['functions'].append(frame.name())
+            frame = frame.older()
+
+        thread_info['functions'] = tuple(thread_info['functions'])
+        if thread_info['functions'] in stacks:
+            stacks[thread_info['functions']]['tids'].append(thread.num)
+        else:
+            thread_info['header'] = "Thread {} (Thread {} (LWP {})):".format(
+                thread.num, thread_info['pthread'], thread.ptid[1])
+            try:
+                thread_info['frames'] = gdb.execute(arg, to_string=True).rstrip()
+            except gdb.error as err:
+                raise gdb.GdbError("{} {}".format(thread_info['header'], err))
+            else:
+                thread_info['tids'] = []
+                thread_info['tids'].append(thread.num)
+                stacks[thread_info['functions']] = thread_info
+
+    def _dump_unique_stacks(self, stacks):
+        def first_tid(stack):
+            return stack['tids'][0]
+
+        for stack in sorted(stacks.values(), key=first_tid, reverse=True):
+            print(stack['header'])
+            if len(stack['tids']) > 1:
+                print("{} duplicate thread(s):".format(len(stack['tids']) - 1), end=' ')
+                print(", ".join((str(tid) for tid in stack['tids'][1:])))
+            print(stack['frames'])
+            print() # leave extra blank line after each thread stack
+
+# Register command
+MongoDBUniqueStack()
+
+
 class MongoDBHelp(gdb.Command):
     """Dump list of mongodb commands"""
 
diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py
index da000998711..118501b307b 100755
--- a/buildscripts/hang_analyzer.py
+++ b/buildscripts/hang_analyzer.py
@@ -302,6 +302,23 @@ class GDBDumper(object):
         gdb_dir = os.path.join(script_dir, "gdb")
         printers_script = os.path.join(gdb_dir, "mongo.py")
 
+        bt_command = "mongodb-uniqstack bt"
+        if sys.platform.startswith("sunos"):
+            '''
+            On Solaris, currently calling mongo-uniqstack leads to an error:
+
+            Thread 198 received signal SIGSEGV, Segmentation fault.
+            0x0000000000000000 in ?? ()
+            Python Exception <class 'gdb.error'> The program being debugged was signaled while in a
+            function called from GDB.
+            GDB remains in the frame where the signal was received.
+            To change this behavior use "set unwindonsignal on".
+            Evaluation of the expression containing the function
+            (at 0x0x0) will be abandoned.
+            When the function is done executing, GDB will silently stop.
+            '''
+            bt_command = "thread apply all bt"
+
         cmds = [
             "set pagination off",
             "attach %d" % pid,
@@ -309,7 +326,7 @@ class GDBDumper(object):
             "info threads",  # Dump a simple list of commands to get the thread name
             "set python print-stack full",
             "source " + printers_script,
-            "thread apply all bt",
+            bt_command,
             dump_command,
             "mongodb-analyze",
             "set confirm off",