diff options
-rw-r--r-- | buildscripts/resmokelib/hang_analyzer/dumper.py | 28 | ||||
-rw-r--r-- | buildscripts/resmokelib/hang_analyzer/process.py | 13 |
2 files changed, 38 insertions, 3 deletions
diff --git a/buildscripts/resmokelib/hang_analyzer/dumper.py b/buildscripts/resmokelib/hang_analyzer/dumper.py index 2b31514a007..92d58d7d190 100644 --- a/buildscripts/resmokelib/hang_analyzer/dumper.py +++ b/buildscripts/resmokelib/hang_analyzer/dumper.py @@ -5,12 +5,14 @@ import logging import os import sys import tempfile +from datetime import datetime from abc import ABCMeta, abstractmethod from collections import namedtuple from distutils import spawn from buildscripts.resmokelib.hang_analyzer.process import call, callo, find_program from buildscripts.resmokelib.hang_analyzer.process_list import Pinfo +from buildscripts.resmokelib import config as resmoke_config Dumpers = namedtuple('Dumpers', ['dbg', 'jstack']) @@ -329,6 +331,20 @@ class LLDBDumper(Dumper): class GDBDumper(Dumper): """GDBDumper class.""" + def __init__(self, root_logger: logging.Logger, dbg_output: str, + timeout_seconds_for_gdb_process=720): + """Initialize GDBDumper.""" + if resmoke_config.EVERGREEN_TASK_ID is None: + # Set 24 hours time out for hang analyzer being run in locally + timeout_seconds_for_gdb_process = 86400 + #Timeout for hang analyzer, default timeout is 12mins(out of total 15mins) in Evergreen + self._timeout_seconds_for_gdb_process = timeout_seconds_for_gdb_process + super().__init__(root_logger, dbg_output) + + def _reduce_timeout_for_gdb_process(self, timeout_period: int): + """Reduce timeout for remaining gdb processes.""" + self._timeout_seconds_for_gdb_process -= timeout_period + def _find_debugger(self, debugger): """Find the installed debugger.""" return find_program(debugger, ['/opt/mongodbtoolchain/v4/bin', '/usr/bin']) @@ -443,12 +459,19 @@ class GDBDumper(Dumper): debugger = "gdb" dbg = self._find_debugger(debugger) logger = _get_process_logger(self._dbg_output, pinfo.name) + _start_time = datetime.now() if dbg is None: self._root_logger.warning("Debugger %s not found, skipping dumping of %s", debugger, str(pinfo.pidv)) return + if self._timeout_seconds_for_gdb_process <= 0: + self._root_logger.warning( + "Skipping dumping of %s processes with PIDs %s because the time limit expired", + pinfo.name, str(pinfo.pidv)) + return + self._root_logger.info("Debugger %s, analyzing %s processes with PIDs %s", dbg, pinfo.name, str(pinfo.pidv)) @@ -457,8 +480,11 @@ class GDBDumper(Dumper): cmds = self._prefix() + self._process_specific(pinfo, take_dump, logger) + self._postfix() call([dbg, "--quiet", "--nx"] + list( - itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger) + itertools.chain.from_iterable([['-ex', b] for b in cmds])), logger, + self._timeout_seconds_for_gdb_process, pinfo) + time_period = (datetime.now() - _start_time).total_seconds() + self._reduce_timeout_for_gdb_process(time_period) self._root_logger.info("Done analyzing %s processes with PIDs %s", pinfo.name, str(pinfo.pidv)) diff --git a/buildscripts/resmokelib/hang_analyzer/process.py b/buildscripts/resmokelib/hang_analyzer/process.py index 9f40c20fe12..17d8713be2e 100644 --- a/buildscripts/resmokelib/hang_analyzer/process.py +++ b/buildscripts/resmokelib/hang_analyzer/process.py @@ -22,7 +22,7 @@ if _IS_WINDOWS: PROCS_TIMEOUT_SECS = 60 -def call(args, logger): +def call(args, logger, timeout_seconds=None, pinfo=None): """Call subprocess on args list.""" logger.info(str(args)) @@ -31,7 +31,16 @@ def call(args, logger): logger_pipe = core.pipe.LoggerPipe(logger, logging.INFO, process.stdout) logger_pipe.wait_until_started() - ret = process.wait() + try: + ret = process.wait(timeout=timeout_seconds) + except subprocess.TimeoutExpired: + logger.error("Killing %s processes with PIDs %s because time limit expired", pinfo.name, + str(pinfo.pidv)) + process.kill() + process.wait() + logger_pipe.wait_until_finished() + return + logger_pipe.wait_until_finished() if ret != 0: |