SERVER-1424 Rewrite smoke.py.

Split out the passthrough tests into separate suites. The MongoDB
deployment is started up by resmoke.py so that we can record the
success/failure of each individual test in MCI.

Added support for parallel execution of tests by dispatching to
multiple MongoDB deployments.

Added support for grouping different kinds of tests (e.g. C++ unit
tests, dbtests, and jstests) so that they can be run together. This
allows for customizability in specifying what tests to execute when
changes are made to a particular part of the code.

1 files changed, 181 insertions, 196 deletions

diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py
index 92b00bee72d..d7c3aff27e9 100755
--- a/buildscripts/resmoke.py
+++ b/buildscripts/resmoke.py
@@ -1,226 +1,211 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 
 """
-Command line test utility for MongoDB tests of all kinds.
-
-CURRENTLY IN ACTIVE DEVELOPMENT
-If you are not a developer, you probably want to use smoke.py
+Command line utility for executing MongoDB tests of all kinds.
 """
 
-import logging
-import logging.config
-import optparse
-import os
-import re
-import urllib
-
-import smoke
-import smoke_config
-
-USAGE = \
-    """resmoke.py <YAML/JSON CONFIG>
-
-All options are specified as YAML or JSON - the configuration can be loaded via a file, as a named
-configuration in the "smoke_config" module, piped as stdin, or specified on the command line as
-options via the --set, --unset, and --push operators.
-
-NOTE: YAML can only be used if the PyYaml library is available on your system.  Only JSON is
-supported on the command line.
-
-For example:
-    resmoke.py './jstests/disk/*.js'
-
-results in:
-
-    Test Configuration:
-    ---
-    tests:
-      roots:
-      - ./jstests/disk/*.js
-    suite:
-      ...
-    executor:
-      fixtures:
-        ...
-      testers:
-        ...
-    logging:
-      ...
-    
-Named sets of options are available in the "smoke_config" module, including:
-
-    --jscore
-    --sharding
-    --replicasets
-    --disk
-    
-For example:
-    resmoke.py --jscore
-    resmoke.py --sharding
-    
-""" + smoke.json_options.JSONOptionParser.DEFAULT_USAGE
-
-DEFAULT_LOGGER_CONFIG = {}
-
-
-def get_local_logger_filenames(logging_root):
-    """Helper to extract filenames from the logging config for helpful reporting to the user."""
-
-    filenames = []
-    if "handlers" not in logging_root:
-        return filenames
-
-    for handler_name, handler_info in logging_root["handlers"].iteritems():
-        if "filename" in handler_info:
-            logger_filename = handler_info["filename"]
-            filenames.append("file://%s" %
-                             urllib.pathname2url(os.path.abspath(logger_filename)))
-
-    return filenames
+from __future__ import absolute_import
+
+import json
+import os.path
+import random
+import signal
+import sys
+import time
+import traceback
+
+# Get relative imports to work when the package is not installed on the PYTHONPATH.
+if __name__ == "__main__" and __package__ is None:
+    sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+    from buildscripts import resmokelib
+
+
+def _execute_suite(suite, logging_config):
+    """
+    Executes each test group of 'suite', failing fast if requested.
+    """
+
+    logger = resmokelib.logging.loggers.EXECUTOR
+
+    for group in suite.test_groups:
+        if resmokelib.config.SHUFFLE:
+            logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.",
+                        group.test_kind, suite.get_name(), resmokelib.config.RANDOM_SEED)
+            random.seed(resmokelib.config.RANDOM_SEED)
+            random.shuffle(group.tests)
+
+        if resmokelib.config.DRY_RUN == "tests":
+            sb = []
+            sb.append("Tests that would be run for %ss in suite %s:"
+                      % (group.test_kind, suite.get_name()))
+            if len(group.tests) > 0:
+                for test in group.tests:
+                    sb.append(test)
+            else:
+                sb.append("(no tests)")
+            logger.info("\n".join(sb))
+
+            # Set a successful return code on the test group because we want to output the tests
+            # that would get run by any other suites the user specified.
+            group.return_code = 0
+            continue
+
+        if len(group.tests) == 0:
+            logger.info("Skipping %ss, no tests to run", group.test_kind)
+            continue
+
+        group_config = suite.get_executor_config().get(group.test_kind, {})
+        executor = resmokelib.testing.executor.TestGroupExecutor(logger,
+                                                                 group,
+                                                                 logging_config,
+                                                                 **group_config)
+
+        try:
+            executor.run()
+            if resmokelib.config.FAIL_FAST and group.return_code != 0:
+                suite.return_code = group.return_code
+                return
+        except resmokelib.errors.StopExecution:
+            suite.return_code = 130  # Simulate SIGINT as exit code.
+            return
+        except:
+            logger.exception("Encountered an error when running %ss of suite %s.",
+                             group.test_kind, suite.get_name())
+            suite.return_code = 2
+            return
+
+
+def _log_summary(logger, suites, time_taken):
+    if len(suites) > 1:
+        sb = []
+        sb.append("Summary of all suites: %d suites ran in %0.2f seconds"
+                  % (len(suites), time_taken))
+        for suite in suites:
+            suite_sb = []
+            suite.summarize(suite_sb)
+            sb.append("    %s: %s" % (suite.get_name(), "\n    ".join(suite_sb)))
+
+        logger.info("=" * 80)
+        logger.info("\n".join(sb))
+
+
+def _summarize_suite(suite):
+    sb = []
+    suite.summarize(sb)
+    return "\n".join(sb)
+
+
+def _dump_suite_config(suite, logging_config):
+    """
+    Returns a string that represents the YAML configuration of a suite.
+
+    TODO: include the "options" key in the result
+    """
+
+    sb = []
+    sb.append("YAML configuration of suite %s" % (suite.get_name()))
+    sb.append(resmokelib.utils.dump_yaml({"selector": suite.get_selector_config()}))
+    sb.append("")
+    sb.append(resmokelib.utils.dump_yaml({"executor": suite.get_executor_config()}))
+    sb.append("")
+    sb.append(resmokelib.utils.dump_yaml({"logging": logging_config}))
+    return "\n".join(sb)
+
+
+def _write_report_file(suites, pathname):
+    """
+    Writes the report.json file if requested.
+    """
+
+    reports = []
+    for suite in suites:
+        for group in suite.test_groups:
+            report = group.get_latest_report()
+            if report is not None:
+                reports.append(report)
+
+    combined_report_dict = resmokelib.testing.report.TestReport.combine(*reports).as_dict()
+    with open(pathname, "w") as fp:
+        json.dump(combined_report_dict, fp)
 
 
 def main():
+    start_time = time.time()
 
-    named_configs = smoke_config.get_named_configs()
-
-    parser = smoke.json_options.JSONOptionParser(usage=USAGE,
-                                                 configfile_args=named_configs)
-
-    help = \
-        """Just outputs the configured JSON options."""
-
-    parser.add_option('--dump-options', default=False, dest='dump_options', action="store_true",
-                      help=help)
-
-    help = \
-        """Outputs all the tests found with metadata."""
-
-    parser.add_option('--dump-tests', default=False, dest='dump_tests', action="store_true",
-                      help=help)
-
-    help = \
-        """Outputs the tests in the suite."""
-
-    parser.add_option('--dump-suite', default=False, dest='dump_suite', action="store_true",
-                      help=help)
-
-    values, args, json_root = parser.parse_json_args()
-
-    # Assume remaining arguments are test roots
-    if args:
-        json_root = smoke.json_options.json_update_path(json_root, "tests.roots", args)
-
-    # Assume all files in suite if not specified
-    if "suite" not in json_root or json_root["suite"] is None:
-        json_root["suite"] = {}
-
-    # Assume default_logging if no other logging specified
-    if "logging" not in json_root or json_root["logging"] is None:
-        default_logging = \
-            smoke.json_options.json_file_load(named_configs["log_default"])
-        json_root["logging"] = default_logging["logging"]
+    values, args = resmokelib.parser.parse_command_line()
 
-    if "executor" not in json_root or json_root["executor"] is None:
-        default_executor = \
-            smoke.json_options.json_file_load(named_configs["executor_default"])
-        json_root["executor"] = default_executor["executor"]
+    logging_config = resmokelib.parser.get_logging_config(values)
+    resmokelib.logging.config.apply_config(logging_config)
+    resmokelib.logging.flush.start_thread()
 
-    if not values.dump_options:
-        print "Test Configuration: \n---"
+    resmokelib.parser.update_config_vars(values)
 
-    for key in ["tests", "suite", "executor", "logging"]:
-        if key in json_root:
-            print smoke.json_options.json_dump({key: json_root[key]}),
-    print
+    exec_logger = resmokelib.logging.loggers.EXECUTOR
+    resmoke_logger = resmokelib.logging.loggers.new_logger("resmoke", parent=exec_logger)
 
-    if values.dump_options:
-        return
+    if values.list_suites:
+        suite_names = resmokelib.parser.get_named_suites()
+        resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names))
+        sys.exit(0)
 
-    def validate_config(tests=None, suite=None, executor=None, logging=None, **kwargs):
-
-        if len(kwargs) > 0:
-            raise optparse.OptionValueError(
-                "Unrecognized test options: %s" % kwargs)
-
-        if not all([tests is not None, executor is not None]):
-            raise optparse.OptionValueError(
-                "Test options must contain \"tests\" and \"executor\".")
-
-    validate_config(**json_root)
-    logging.config.dictConfig(json_root["logging"])
-
-    def re_compile_all(re_patterns):
-        if isinstance(re_patterns, basestring):
-            re_patterns = [re_patterns]
-        return [re.compile(pattern) for pattern in re_patterns]
-
-    def build_tests(roots=["./"],
-                    include_files=[],
-                    include_files_except=[],
-                    exclude_files=[],
-                    exclude_files_except=[],
-                    extract_metadata=True,
-                    **kwargs):
-
-        if len(kwargs) > 0:
-            raise optparse.OptionValueError(
-                "Unrecognized options for tests: %s" % kwargs)
+    suites = resmokelib.parser.get_suites(values, args)
+    try:
+        for suite in suites:
+            resmoke_logger.info(_dump_suite_config(suite, logging_config))
 
-        file_regex_query = smoke.suites.RegexQuery(re_compile_all(include_files),
-                                                   re_compile_all(
-                                                       include_files_except),
-                                                   re_compile_all(
-                                                       exclude_files),
-                                                   re_compile_all(exclude_files_except))
+            suite.record_start()
+            _execute_suite(suite, logging_config)
+            suite.record_end()
 
-        if isinstance(roots, basestring):
-            roots = [roots]
+            resmoke_logger.info("=" * 80)
+            resmoke_logger.info("Summary of %s suite: %s",
+                                suite.get_name(), _summarize_suite(suite))
 
-        return smoke.tests.build_tests(roots, file_regex_query, extract_metadata)
+            if resmokelib.config.FAIL_FAST and suite.return_code != 0:
+                time_taken = time.time() - start_time
+                _log_summary(resmoke_logger, suites, time_taken)
+                sys.exit(suite.return_code)
 
-    tests = build_tests(**json_root["tests"])
+        time_taken = time.time() - start_time
+        _log_summary(resmoke_logger, suites, time_taken)
 
-    if values.dump_tests:
-        print "Tests:\n%s" % tests
+        # Exit with a nonzero code if any of the suites failed.
+        exit_code = max(suite.return_code for suite in suites)
+        sys.exit(exit_code)
+    finally:
+        if resmokelib.config.REPORT_FILE is not None:
+            _write_report_file(suites, resmokelib.config.REPORT_FILE)
 
-    def build_suite(tests,
-                    include_tags=[],
-                    include_tags_except=[],
-                    exclude_tags=[],
-                    exclude_tags_except=[],
-                    **kwargs):
 
-        if len(kwargs) > 0:
-            raise optparse.OptionValueError(
-                "Unrecognized options for suite: %s" % kwargs)
+if __name__ == "__main__":
 
-        tag_regex_query = smoke.suites.RegexQuery(re_compile_all(include_tags),
-                                                  re_compile_all(
-                                                      include_tags_except),
-                                                  re_compile_all(exclude_tags),
-                                                  re_compile_all(exclude_tags_except))
+    def _dump_stacks(signum, frame):
+        """
+        Signal handler that will dump the stacks of all threads.
+        """
 
-        return smoke.suites.build_suite(tests, tag_regex_query)
+        header_msg = "Dumping stacks due to SIGUSR1 signal"
 
-    suite = build_suite(tests, **json_root["suite"])
-    suite.sort(key=lambda test: test.uri)
+        sb = []
+        sb.append("=" * len(header_msg))
+        sb.append(header_msg)
+        sb.append("=" * len(header_msg))
 
-    if values.dump_suite:
-        print "Suite:\n%s" % suite
+        frames = sys._current_frames()
+        sb.append("Total threads: %d" % (len(frames)))
+        sb.append("")
 
-    print "Running %s tests in suite (out of %s tests found)..." % (len(tests), len(suite))
+        for thread_id in frames:
+            stack = frames[thread_id]
+            sb.append("Thread %d:" % (thread_id))
+            sb.append("".join(traceback.format_stack(stack)))
 
-    local_logger_filenames = get_local_logger_filenames(json_root["logging"])
-    if local_logger_filenames:
-        print "\nOutput from tests redirected to:\n\t%s\n" % \
-            "\n\t".join(local_logger_filenames)
+        sb.append("=" * len(header_msg))
+        print "\n".join(sb)
 
     try:
-        smoke.executor.exec_suite(suite, logging.getLogger("executor"), **json_root["executor"])
-    finally:
-        if local_logger_filenames:
-            print "\nOutput from tests was redirected to:\n\t%s\n" % \
-                "\n\t".join(local_logger_filenames)
+        signal.signal(signal.SIGUSR1, _dump_stacks)
+    except AttributeError:
+        print "Cannot catch signals on Windows"
 
-if __name__ == "__main__":
     main()