diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2017-08-08 11:31:49 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2017-08-14 09:00:25 -0400 |
commit | 01f3279e039f610a99ef84238ca49498c716ee14 (patch) | |
tree | 70586860a6fb9f6e37b9b7d9b4b733ab3cad67d2 /buildscripts/hang_analyzer.py | |
parent | b8eb96c3c9c2ad2e0c4520785ba49cb9830b1a21 (diff) | |
download | mongo-01f3279e039f610a99ef84238ca49498c716ee14.tar.gz |
SERVER-28824 Trap debugger excptions in hang analyzer and display at end for all processes
(cherry picked from commit 324839c0c0c2b294a44d130105797ccdbb3b17a9)
Diffstat (limited to 'buildscripts/hang_analyzer.py')
-rwxr-xr-x | buildscripts/hang_analyzer.py | 54 |
1 files changed, 35 insertions, 19 deletions
diff --git a/buildscripts/hang_analyzer.py b/buildscripts/hang_analyzer.py index 73e1cbe6d41..09245904a7f 100755 --- a/buildscripts/hang_analyzer.py +++ b/buildscripts/hang_analyzer.py @@ -24,6 +24,7 @@ import signal import subprocess import sys import tempfile +import traceback import time from distutils import spawn from optparse import OptionParser @@ -53,7 +54,7 @@ def call(a, logger): if ret != 0: logger.error("Bad exit code %d" % (ret)) - raise Exception() + raise Exception("Bad exit code %d from %s" % (ret, " ".join(a))) # Copied from python 2.7 version of subprocess.py @@ -375,14 +376,14 @@ class GDBDumper(object): base, ext = os.path.splitext(logger.mongo_process_filename) raw_stacks_filename = base + '_raw_stacks' + ext raw_stacks_commands = [ - 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, - # This sends output to log file rather than stdout until we turn logging off. - 'set logging redirect on', - 'set logging file ' + raw_stacks_filename, - 'set logging on', - 'thread apply all bt', - 'set logging off', - ] + 'echo \\nWriting raw stacks to %s.\\n' % raw_stacks_filename, + # This sends output to log file rather than stdout until we turn logging off. + 'set logging redirect on', + 'set logging file ' + raw_stacks_filename, + 'set logging on', + 'thread apply all bt', + 'set logging off', + ] cmds = [ "set interactive-mode off", @@ -732,28 +733,38 @@ def main(): # a signal handler to wait for the signal since it supports POSIX signals. if _is_windows: root_logger.info("Calling SetEvent to signal python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_event_object(root_logger, pid) else: root_logger.info("Sending signal SIGUSR1 to python process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGUSR1) + trapped_exceptions = [] + # Dump all processes, except python & java. for (pid, process_name) in [(p, pn) for (p, pn) in processes if not re.match("^(java|python)", pn)]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - dbg.dump_info( - root_logger, - process_logger, - pid, - process_name, - options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + try: + dbg.dump_info( + root_logger, + process_logger, + pid, + process_name, + options.dump_core and check_dump_quota(max_dump_size_bytes, dbg.get_dump_ext())) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Dump java processes using jstack. for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn.startswith("java")]: process_logger = get_process_logger(options.debugger_output, pid, process_name) - jstack.dump_info(root_logger, process_logger, pid, process_name) + try: + jstack.dump_info(root_logger, process_logger, pid, process_name) + except Exception as err: + root_logger.info("Error encountered when invoking debugger %s" % err) + trapped_exceptions.append(traceback.format_exc()) # Signal go processes to ensure they print out stack traces, and die on POSIX OSes. # On Windows, this will simply kill the process since python emulates SIGABRT as @@ -761,10 +772,15 @@ def main(): # Note: The stacktrace output may be captured elsewhere (i.e. resmoke). for (pid, process_name) in [(p, pn) for (p, pn) in processes if pn in go_processes]: root_logger.info("Sending signal SIGABRT to go process %s with PID %d" % - (process_name, pid)) + (process_name, pid)) signal_process(root_logger, pid, signal.SIGABRT) root_logger.info("Done analyzing all processes for hangs") + for exception in trapped_exceptions: + root_logger.info(exception) + if trapped_exceptions: + sys.exit(1) + if __name__ == "__main__": main() |