diff options
| -rwxr-xr-x | buildscripts/evergreen_run_tests.py | 8 | ||||
| -rwxr-xr-x | buildscripts/resmoke.py | 400 | ||||
| -rw-r--r-- | buildscripts/resmokelib/parser.py | 30 |
3 files changed, 230 insertions, 208 deletions
diff --git a/buildscripts/evergreen_run_tests.py b/buildscripts/evergreen_run_tests.py index 69520dd472e..cec88b86d0f 100755 --- a/buildscripts/evergreen_run_tests.py +++ b/buildscripts/evergreen_run_tests.py @@ -17,7 +17,7 @@ from buildscripts import resmokelib # pylint: disable=wrong-import-position _TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"]) -class Main(resmoke.Main): +class Main(resmoke.Resmoke): """Execute Main class. A class for executing potentially multiple resmoke.py test suites in a way that handles @@ -112,7 +112,7 @@ class Main(resmoke.Main): suites = [] - for suite in resmoke.Main._get_suites(self): + for suite in resmoke.Resmoke._get_suites(self): if suite.test_kind != "js_test": # Tags are only support for JavaScript tests, so we leave the test suite alone when # running any other kind of test. @@ -145,4 +145,6 @@ class Main(resmoke.Main): if __name__ == "__main__": - Main().run() + main = Main() + main.configure_from_command_line() + main.run() diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 033fab1889d..0d57e3ff6f2 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -12,222 +12,234 @@ import time if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from buildscripts import resmokelib # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from buildscripts.resmokelib import config +from buildscripts.resmokelib import errors +from buildscripts.resmokelib import logging +from buildscripts.resmokelib import parser +from buildscripts.resmokelib import reportfile +from buildscripts.resmokelib import sighandler +from buildscripts.resmokelib import suitesconfig +from buildscripts.resmokelib import testing +from buildscripts.resmokelib import utils -def _execute_suite(suite): # pylint: disable=too-many-branches,too-many-return-statements - """Execute the test suite, failing fast if requested. +class Resmoke(object): + """The main class to run tests with resmoke.""" - Return true if the execution of the suite was interrupted by the - user, and false otherwise. - """ - - logger = resmokelib.logging.loggers.EXECUTOR_LOGGER + def __init__(self): + """Initialize the Resmoke instance.""" + self.__start_time = time.time() + self._config = None + self._exec_logger = None + self._resmoke_logger = None + self._archive = None + self._interrupted = False + + def configure_from_command_line(self): + """Configure this instance using the command line arguments.""" + self._config = parser.parse_command_line() + + def _setup_logging(self): + logging.loggers.configure_loggers(self._config.logging_config) + logging.flush.start_thread() + self._exec_logger = logging.loggers.EXECUTOR_LOGGER + self._resmoke_logger = self._exec_logger.new_resmoke_logger() - if resmokelib.config.SHUFFLE: - logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.", - suite.test_kind, suite.get_display_name(), resmokelib.config.RANDOM_SEED) - random.seed(resmokelib.config.RANDOM_SEED) - random.shuffle(suite.tests) + def run(self): + """Run resmoke.""" + if self._config is None: + raise RuntimeError("Resmoke must be configured before calling run()") + self._setup_logging() - if resmokelib.config.DRY_RUN == "tests": - sb = [] - sb.append("Tests that would be run in suite %s:" % suite.get_display_name()) - if suite.tests: + try: + if self._config.list_suites: + self.list_suites() + elif self._config.find_suites: + self.find_suites() + elif self._config.dry_run == "tests": + self.dry_run() + else: + self.run_tests() + finally: + if not self._interrupted: + logging.flush.stop_thread() + + def list_suites(self): + """List the suites that are available to execute.""" + suite_names = suitesconfig.get_named_suites() + self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names)) + + def find_suites(self): + """List the suites that run the specified tests.""" + suites = self._get_suites() + suites_by_test = self._find_suites_by_test(suites) + for test in sorted(suites_by_test): + suite_names = suites_by_test[test] + self._resmoke_logger.info("%s will be run by the following suite(s): %s", test, + suite_names) + + @staticmethod + def _find_suites_by_test(suites): + """ + Look up what other resmoke suites run the tests specified in the suites parameter. + + Return a dict keyed by test name, value is array of suite names. + """ + memberships = {} + test_membership = suitesconfig.create_test_membership_map() + for suite in suites: for test in suite.tests: - sb.append(test) - else: - sb.append("(no tests)") - logger.info("\n".join(sb)) - sb = [] - sb.append("Tests that would be excluded from suite %s:" % suite.get_display_name()) - if suite.excluded: - for test in suite.excluded: - sb.append(test) - else: - sb.append("(no tests)") - logger.info("\n".join(sb)) - - # Set a successful return code on the test suite because we want to output the tests - # that would get run by any other suites the user specified. - suite.return_code = 0 - return False - - if not suite.tests: - logger.info("Skipping %ss, no tests to run", suite.test_kind) + memberships[test] = test_membership[test] + return memberships + + def dry_run(self): + """List which tests would run and which tests would be excluded in a resmoke invocation.""" + suites = self._get_suites() + for suite in suites: + self._shuffle_tests(suite) + sb = ["Tests that would be run in suite {}".format(suite.get_display_name())] + sb.extend(suite.tests or ["(no tests)"]) + sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name())) + sb.extend(suite.excluded or ["(no tests)"]) + self._exec_logger.info("\n".join(sb)) + + def run_tests(self): + """Run the suite and tests specified.""" + self._resmoke_logger.info("resmoke.py invocation: %s", " ".join(sys.argv)) + suites = None + try: + suites = self._get_suites() - # Set a successful return code on the test suite because we want to output the tests - # that would get run by any other suites the user specified. - suite.return_code = 0 - return False + self._setup_archival() + self._setup_signal_handler(suites) - archive = None - if resmokelib.config.ARCHIVE_FILE: - archive = resmokelib.utils.archival.Archival( - archival_json_file=resmokelib.config.ARCHIVE_FILE, - limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, - limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, logger=logger) + for suite in suites: + self._interrupted = self._run_suite(suite) + if self._interrupted or (suite.options.fail_fast and suite.return_code != 0): + self._log_resmoke_summary(suites) + self.exit(suite.return_code) - executor_config = suite.get_executor_config() - executor = resmokelib.testing.executor.TestSuiteExecutor( - logger, suite, archive_instance=archive, **executor_config) + self._log_resmoke_summary(suites) - try: - executor.run() - if suite.options.fail_fast and suite.return_code != 0: + # Exit with a nonzero code if any of the suites failed. + exit_code = max(suite.return_code for suite in suites) + self.exit(exit_code) + finally: + if not self._interrupted: + self._exit_archival() + if suites: + reportfile.write(suites) + + def _run_suite(self, suite): + """Run a test suite.""" + self._log_suite_config(suite) + suite.record_suite_start() + interrupted = self._execute_suite(suite) + suite.record_suite_end() + self._log_suite_summary(suite) + return interrupted + + def _log_resmoke_summary(self, suites): + """Log a summary of the resmoke run.""" + time_taken = time.time() - self.__start_time + if len(self._config.suite_files) > 1: + testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken) + + def _log_suite_summary(self, suite): + """Log a summary of the suite run.""" + self._resmoke_logger.info("=" * 80) + self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(), + self._get_suite_summary(suite)) + + def _execute_suite(self, suite): + """Execute a suite and return True if interrupted, False otherwise. """ + self._shuffle_tests(suite) + if not suite.tests: + self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind) + suite.return_code = 0 + return False + executor_config = suite.get_executor_config() + executor = testing.executor.TestSuiteExecutor( + self._exec_logger, suite, archive_instance=self._archive, **executor_config) + try: + executor.run() + except errors.UserInterrupt: + suite.return_code = 130 # Simulate SIGINT as exit code. + return True + except IOError: + suite.return_code = 74 # Exit code for IOError on POSIX systems. + return True + except: # pylint: disable=bare-except + self._exec_logger.exception("Encountered an error when running %ss of suite %s.", + suite.test_kind, suite.get_display_name()) + suite.return_code = 2 return False - except resmokelib.errors.UserInterrupt: - suite.return_code = 130 # Simulate SIGINT as exit code. - return True - except IOError: - suite.return_code = 74 # Exit code for IOError on POSIX systems. - return True - except: # pylint: disable=bare-except - logger.exception("Encountered an error when running %ss of suite %s.", suite.test_kind, - suite.get_display_name()) - suite.return_code = 2 return False - finally: - if archive: - archive.exit() - return False - - -def _log_summary(logger, suites, time_taken): - if len(suites) > 1: - resmokelib.testing.suite.Suite.log_summaries(logger, suites, time_taken) - - -def _summarize_suite(suite): - sb = [] - suite.summarize(sb) - return "\n".join(sb) - - -def _dump_suite_config(suite, logging_config): - """Return a string that represents the YAML configuration of a suite. - - TODO: include the "options" key in the result - """ - - sb = [] - sb.append("YAML configuration of suite %s" % (suite.get_display_name())) - sb.append(resmokelib.utils.dump_yaml({"test_kind": suite.get_test_kind_config()})) - sb.append("") - sb.append(resmokelib.utils.dump_yaml({"selector": suite.get_selector_config()})) - sb.append("") - sb.append(resmokelib.utils.dump_yaml({"executor": suite.get_executor_config()})) - sb.append("") - sb.append(resmokelib.utils.dump_yaml({"logging": logging_config})) - return "\n".join(sb) - -def find_suites_by_test(suites): - """Look up what other resmoke suites run the tests specified in the suites parameter. - - Return a dict keyed by test name, value is array of suite names. - """ - - memberships = {} - test_membership = resmokelib.suitesconfig.create_test_membership_map() - for suite in suites: - for test in suite.tests: - memberships[test] = test_membership[test] - return memberships - - -def _list_suites_and_exit(logger, exit_code=0): - suite_names = resmokelib.suitesconfig.get_named_suites() - logger.info("Suites available to execute:\n%s", "\n".join(suite_names)) - sys.exit(exit_code) - - -class Main(object): - """A class for executing potentially multiple resmoke.py test suites.""" - - def __init__(self): - """Initialize the Main instance by parsing the command line arguments.""" - - self.__start_time = time.time() - - values, args = resmokelib.parser.parse_command_line() - self.__values = values - self.__args = args + def _shuffle_tests(self, suite): + """Shuffle the tests if the shuffle cli option was set.""" + random.seed(config.RANDOM_SEED) + if not config.SHUFFLE: + return + self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.", + suite.test_kind, suite.get_display_name(), config.RANDOM_SEED) + random.shuffle(suite.tests) def _get_suites(self): - """Return a list of resmokelib.testing.suite.Suite instances to execute.""" - - return resmokelib.suitesconfig.get_suites( - suite_files=self.__values.suite_files.split(","), test_files=self.__args) - - def run(self): - """Execute the list of resmokelib.testing.suite.Suite instances.""" - - logging_config = resmokelib.parser.get_logging_config(self.__values) - resmokelib.logging.loggers.configure_loggers(logging_config) - resmokelib.logging.flush.start_thread() - - resmokelib.parser.update_config_vars(self.__values) - - exec_logger = resmokelib.logging.loggers.EXECUTOR_LOGGER - resmoke_logger = exec_logger.new_resmoke_logger() - - if self.__values.list_suites: - _list_suites_and_exit(resmoke_logger) - - # Log the command line arguments specified to resmoke.py to make it easier to re-run the - # resmoke.py invocation used by an Evergreen task. - resmoke_logger.info("resmoke.py invocation: %s", " ".join(sys.argv)) - - interrupted = False - try: - suites = self._get_suites() - except resmokelib.errors.SuiteNotFound as err: - resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err)) - _list_suites_and_exit(resmoke_logger, exit_code=1) - - # Register a signal handler or Windows event object so we can write the report file if the - # task times out. - resmokelib.sighandler.register(resmoke_logger, suites, self.__start_time) - - # Run the suite finder after the test suite parsing is complete. - if self.__values.find_suites: - suites_by_test = find_suites_by_test(suites) - for test in sorted(suites_by_test): - suite_names = suites_by_test[test] - resmoke_logger.info("%s will be run by the following suite(s): %s", test, - suite_names) - sys.exit(0) - + """Return the list of suites for this resmoke invocation.""" try: - for suite in suites: - resmoke_logger.info(_dump_suite_config(suite, logging_config)) + return suitesconfig.get_suites(self._config.suite_files, self._config.test_files) + except errors.SuiteNotFound as err: + self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err)) + self.list_suites() + self.exit(1) + + def _log_suite_config(self, suite): + sb = [ + "YAML configuration of suite {}".format(suite.get_display_name()), + utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "", + utils.dump_yaml({"selector": suite.get_selector_config()}), "", + utils.dump_yaml({"executor": suite.get_executor_config()}), "", + utils.dump_yaml({"logging": self._config.logging_config}) + ] + self._resmoke_logger.info("\n".join(sb)) + + @staticmethod + def _get_suite_summary(suite): + """Return a summary of the suite run.""" + sb = [] + suite.summarize(sb) + return "\n".join(sb) - suite.record_suite_start() - interrupted = _execute_suite(suite) - suite.record_suite_end() + def _setup_signal_handler(self, suites): + """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump.""" + sighandler.register(self._resmoke_logger, suites, self.__start_time) - resmoke_logger.info("=" * 80) - resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(), - _summarize_suite(suite)) + def _setup_archival(self): + """Set up the archival feature if enabled in the cli options.""" + if config.ARCHIVE_FILE: + self._archive = utils.archival.Archival( + archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB, + limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger) - if interrupted or (suite.options.fail_fast and suite.return_code != 0): - time_taken = time.time() - self.__start_time - _log_summary(resmoke_logger, suites, time_taken) - sys.exit(suite.return_code) + def _exit_archival(self): + """Finish up archival tasks before exit if enabled in the cli options.""" + if self._archive: + self._archive.exit() - time_taken = time.time() - self.__start_time - _log_summary(resmoke_logger, suites, time_taken) + def exit(self, exit_code): + """Exit with the provided exit code.""" + self._resmoke_logger.info("Exiting with code: %d", exit_code) + sys.exit(exit_code) - # Exit with a nonzero code if any of the suites failed. - exit_code = max(suite.return_code for suite in suites) - sys.exit(exit_code) - finally: - if not interrupted: - resmokelib.logging.flush.stop_thread() - resmokelib.reportfile.write(suites) +def main(): + """Main function for resmoke.""" + resmoke = Resmoke() + resmoke.configure_from_command_line() + resmoke.run() if __name__ == "__main__": - Main().run() + main() diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index d9f40da3e90..9dc464d8129 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -2,6 +2,7 @@ from __future__ import absolute_import +import collections import os import os.path @@ -12,10 +13,13 @@ from . import config as _config from . import utils from .. import resmokeconfig +ResmokeConfig = collections.namedtuple( + "ResmokeConfig", + ["list_suites", "find_suites", "dry_run", "suite_files", "test_files", "logging_config"]) -def parse_command_line(): # pylint: disable=too-many-statements - """Parse the command line arguments passed to resmoke.py.""" +def _make_parser(): # pylint: disable=too-many-statements + """Create and return the command line arguments parser.""" parser = optparse.OptionParser() parser.add_option("--suites", dest="suite_files", metavar="SUITE1,SUITE2", @@ -291,15 +295,23 @@ def parse_command_line(): # pylint: disable=too-many-statements parser.set_defaults(logger_file="console", dry_run="off", find_suites=False, list_suites=False, suite_files="with_server", prealloc_journal="off", shuffle="auto", stagger_jobs="off") + return parser + +def parse_command_line(): + """Parses the command line arguments passed to resmoke.py.""" + parser = _make_parser() options, args = parser.parse_args() - validate_options(parser, options, args) + _validate_options(parser, options, args) + _update_config_vars(options) - return options, args + return ResmokeConfig(list_suites=options.list_suites, find_suites=options.find_suites, + dry_run=options.dry_run, suite_files=options.suite_files.split(","), + test_files=args, logging_config=_get_logging_config(options.logger_file)) -def validate_options(parser, options, args): +def _validate_options(parser, options, args): """Do preliminary validation on the options and error on any invalid options.""" if options.shell_port is not None and options.shell_conn_string is not None: @@ -329,13 +341,9 @@ def validate_benchmark_options(): "results. Please use --jobs=1" % _config.JOBS) -def get_logging_config(values): - """Return logging config values.""" - return _get_logging_config(values.logger_file) - +def _update_config_vars(values): # pylint: disable=too-many-statements + """Update the variables of the config module.""" -def update_config_vars(values): # pylint: disable=too-many-statements - """Update config vars.""" config = _config.DEFAULTS.copy() # Override `config` with values from command line arguments. |