diff options
23 files changed, 1282 insertions, 1121 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index efc261c00e6..c7769ff4366 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -398,7 +398,7 @@ def create_task_list(evergreen_conf: EvergreenProjectConfig, build_variant: str, def _set_resmoke_cmd(repeat_config: RepeatConfig, resmoke_args: [str]) -> [str]: """Build the resmoke command, if a resmoke.py command wasn't passed in.""" - new_args = [sys.executable, "buildscripts/resmoke.py"] + new_args = [sys.executable, "buildscripts/resmoke.py", "run"] if resmoke_args: new_args = copy.deepcopy(resmoke_args) @@ -645,7 +645,7 @@ def create_tests_by_task(build_variant: str, repos: Iterable[Repo], exclude_suites, exclude_tasks, exclude_tests = find_excludes(SELECTOR_FILE) changed_tests = filter_tests(changed_tests, exclude_tests) - buildscripts.resmokelib.parser.set_options() + buildscripts.resmokelib.parser.set_run_options() if changed_tests: return create_task_list_for_tests(changed_tests, build_variant, evg_conf, exclude_suites, exclude_tasks) diff --git a/buildscripts/evergreen_gen_multiversion_tests.py b/buildscripts/evergreen_gen_multiversion_tests.py index 4e44e86ebe0..20395d892a6 100755 --- a/buildscripts/evergreen_gen_multiversion_tests.py +++ b/buildscripts/evergreen_gen_multiversion_tests.py @@ -415,7 +415,7 @@ def generate_exclude_yaml(suite: str, task_path_suffix: str, is_generated_suite: if not is_generated_suite: # Populate the config values to get the resmoke config directory. - buildscripts.resmokelib.parser.set_options() + buildscripts.resmokelib.parser.set_run_options() suites_dir = os.path.join(_config.CONFIG_DIR, "suites") # Update the static suite config with the excluded files and write to disk. diff --git a/buildscripts/evergreen_generate_resmoke_tasks.py b/buildscripts/evergreen_generate_resmoke_tasks.py index 63ffd156080..f6217881f6c 100755 --- a/buildscripts/evergreen_generate_resmoke_tasks.py +++ b/buildscripts/evergreen_generate_resmoke_tasks.py @@ -760,7 +760,7 @@ class GenerateSubSuites(object): self.test_list = [] # Populate config values for methods like list_tests() - _parser.set_options() + _parser.set_run_options() def calculate_suites(self, start_date: datetime, end_date: datetime) -> List[Suite]: """ diff --git a/buildscripts/evergreen_run_tests.py b/buildscripts/evergreen_run_tests.py deleted file mode 100755 index 861970f63db..00000000000 --- a/buildscripts/evergreen_run_tests.py +++ /dev/null @@ -1,149 +0,0 @@ -#!/usr/bin/env python3 -"""Command line utility for executing MongoDB tests in Evergreen.""" - -import collections -import os.path -import sys - -# Get relative imports to work when the package is not installed on the PYTHONPATH. -if __name__ == "__main__" and __package__ is None: - sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from buildscripts import resmoke # pylint: disable=wrong-import-position -from buildscripts import resmokelib # pylint: disable=wrong-import-position - -_TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"]) - - -class Main(resmoke.Resmoke): - """Execute Main class. - - A class for executing potentially multiple resmoke.py test suites in a way that handles - additional options for running unreliable tests in Evergreen. - """ - - UNRELIABLE_TAG = _TagInfo( - tag_name="unreliable", - evergreen_aware=True, - suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - report_failure_status="silentfail")) - - RESOURCE_INTENSIVE_TAG = _TagInfo( - tag_name="resource_intensive", - evergreen_aware=False, - suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - num_jobs=1)) - - RETRY_ON_FAILURE_TAG = _TagInfo( - tag_name="retry_on_failure", - evergreen_aware=True, - suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, - report_failure_status="silentfail")) - - @staticmethod - def _make_evergreen_aware_tags(tag_name): - """Return a list of resmoke.py tags. - - This list is for task, variant, and distro combinations in Evergreen. - """ - - tags_format = ["{tag_name}"] - - if resmokelib.config.EVERGREEN_TASK_NAME is not None: - tags_format.append("{tag_name}|{task_name}") - - if resmokelib.config.EVERGREEN_VARIANT_NAME is not None: - tags_format.append("{tag_name}|{task_name}|{variant_name}") - - if resmokelib.config.EVERGREEN_DISTRO_ID is not None: - tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}") - - return [ - tag.format(tag_name=tag_name, task_name=resmokelib.config.EVERGREEN_TASK_NAME, - variant_name=resmokelib.config.EVERGREEN_VARIANT_NAME, - distro_id=resmokelib.config.EVERGREEN_DISTRO_ID) for tag in tags_format - ] - - @classmethod - def _make_tag_combinations(cls): - """Return a list of (tag, enabled) pairs. - - These pairs represent all possible combinations of all possible pairings - of whether the tags are enabled or disabled together. - """ - - combinations = [] - - if resmokelib.config.EVERGREEN_PATCH_BUILD: - combinations.append(("unreliable and resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("unreliable and not resource intensive", - ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) - combinations.append(("reliable and resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) - combinations.append(("reliable and not resource intensive", - ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - else: - combinations.append(("retry on failure and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("retry on failure and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - combinations.append(("run once and resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - True)))) - combinations.append(("run once and not resource intensive", - ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, - False)))) - - return combinations - - def _get_suites(self): - """Return a list of resmokelib.testing.suite.Suite instances to execute. - - For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(), - multiple copies of that test suite are run using different resmokelib.config.SuiteOptions() - depending on whether each tag in the combination is enabled or not. - """ - - suites = [] - - for suite in resmoke.Resmoke._get_suites(self): - if suite.test_kind != "js_test": - # Tags are only support for JavaScript tests, so we leave the test suite alone when - # running any other kind of test. - suites.append(suite) - continue - - for (tag_desc, tag_combo) in self._make_tag_combinations(): - suite_options_list = [] - - for (tag_info, enabled) in tag_combo: - if tag_info.evergreen_aware: - tags = self._make_evergreen_aware_tags(tag_info.tag_name) - include_tags = {"$anyOf": tags} - else: - include_tags = tag_info.tag_name - - if enabled: - suite_options = tag_info.suite_options._replace(include_tags=include_tags) - else: - suite_options = resmokelib.config.SuiteOptions.ALL_INHERITED._replace( - include_tags={"$not": include_tags}) - - suite_options_list.append(suite_options) - - suite_options = resmokelib.config.SuiteOptions.combine(*suite_options_list) - suite_options = suite_options._replace(description=tag_desc) - suites.append(suite.with_options(suite_options)) - - return suites - - -if __name__ == "__main__": - main = Main() # pylint: disable=invalid-name - main.configure_from_command_line() - main.run() diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 35262513792..9cc54d5c5f3 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -1,412 +1,23 @@ #!/usr/bin/env python3 """Command line utility for executing MongoDB tests of all kinds.""" -import os import os.path -import random -import shlex -import subprocess import sys -import tarfile import time -import pkg_resources -import requests - -try: - import grpc_tools.protoc - import grpc -except ImportError: - pass - # Get relative imports to work when the package is not installed on the PYTHONPATH. if __name__ == "__main__" and __package__ is None: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # pylint: disable=wrong-import-position -from buildscripts.resmokelib import config -from buildscripts.resmokelib import errors -from buildscripts.resmokelib import logging from buildscripts.resmokelib import parser -from buildscripts.resmokelib import reportfile -from buildscripts.resmokelib import sighandler -from buildscripts.resmokelib import suitesconfig -from buildscripts.resmokelib import testing -from buildscripts.resmokelib import utils - -from buildscripts.resmokelib.core import process -from buildscripts.resmokelib.core import jasper_process - - -class Resmoke(object): # pylint: disable=too-many-instance-attributes - """The main class to run tests with resmoke.""" - - def __init__(self): - """Initialize the Resmoke instance.""" - self.__start_time = time.time() - self._config = None - self._exec_logger = None - self._resmoke_logger = None - self._archive = None - self._jasper_server = None - self._interrupted = False - self._exit_code = 0 - - def configure_from_command_line(self): - """Configure this instance using the command line arguments.""" - self._config = parser.parse_command_line() - - def _setup_logging(self): - logging.loggers.configure_loggers(self._config.logging_config) - logging.flush.start_thread() - self._exec_logger = logging.loggers.EXECUTOR_LOGGER - self._resmoke_logger = self._exec_logger.new_resmoke_logger() - - def _exit_logging(self): - if self._interrupted: - # We want to exit as quickly as possible when interrupted by a user and therefore don't - # bother waiting for all log output to be flushed to logkeeper. - return - - if logging.buildlogger.is_log_output_incomplete(): - # If we already failed to write log output to logkeeper, then we don't bother waiting - # for any remaining log output to be flushed as it'll likely fail too. Exiting without - # joining the flush thread here also means that resmoke.py won't hang due a logger from - # a fixture or a background hook not being closed. - self._exit_on_incomplete_logging() - - flush_success = logging.flush.stop_thread() - if not flush_success: - self._resmoke_logger.error( - 'Failed to flush all logs within a reasonable amount of time, ' - 'treating logs as incomplete') - - if not flush_success or logging.buildlogger.is_log_output_incomplete(): - self._exit_on_incomplete_logging() - - def _exit_on_incomplete_logging(self): - if self._exit_code == 0: - # We don't anticipate users to look at passing Evergreen tasks very often that even if - # the log output is incomplete, we'd still rather not show anything in the Evergreen UI - # or cause a JIRA ticket to be created. - self._resmoke_logger.info( - "We failed to flush all log output to logkeeper but all tests passed, so" - " ignoring.") - else: - exit_code = errors.LoggerRuntimeConfigError.EXIT_CODE - self._resmoke_logger.info( - "Exiting with code %d rather than requested code %d because we failed to flush all" - " log output to logkeeper.", exit_code, self._exit_code) - self._exit_code = exit_code - - # Force exit the process without cleaning up or calling the finally block - # to avoid threads making system calls from blocking process termination. - # This must be the last line of code that is run. - # pylint: disable=protected-access - os._exit(self._exit_code) - - def run(self): - """Run resmoke.""" - if self._config is None: - raise RuntimeError("Resmoke must be configured before calling run()") - self._setup_logging() - - try: - if self._config.list_suites: - self.list_suites() - elif self._config.find_suites: - self.find_suites() - elif self._config.dry_run == "tests": - self.dry_run() - else: - self.run_tests() - finally: - # self._exit_logging() may never return when the log output is incomplete. - # Our workaround is to call os._exit(). - self._exit_logging() - - def list_suites(self): - """List the suites that are available to execute.""" - suite_names = suitesconfig.get_named_suites() - self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names)) - - def find_suites(self): - """List the suites that run the specified tests.""" - suites = self._get_suites() - suites_by_test = self._find_suites_by_test(suites) - for test in sorted(suites_by_test): - suite_names = suites_by_test[test] - self._resmoke_logger.info("%s will be run by the following suite(s): %s", test, - suite_names) - - @staticmethod - def _find_suites_by_test(suites): - """ - Look up what other resmoke suites run the tests specified in the suites parameter. - - Return a dict keyed by test name, value is array of suite names. - """ - memberships = {} - test_membership = suitesconfig.create_test_membership_map() - for suite in suites: - for test in suite.tests: - memberships[test] = test_membership[test] - return memberships - - def dry_run(self): - """List which tests would run and which tests would be excluded in a resmoke invocation.""" - suites = self._get_suites() - for suite in suites: - self._shuffle_tests(suite) - sb = ["Tests that would be run in suite {}".format(suite.get_display_name())] - sb.extend(suite.tests or ["(no tests)"]) - sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name())) - sb.extend(suite.excluded or ["(no tests)"]) - self._exec_logger.info("\n".join(sb)) - - def run_tests(self): - """Run the suite and tests specified.""" - self._resmoke_logger.info( - "verbatim resmoke.py invocation: %s", - " ".join([shlex.quote(arg) for arg in shlex.split(" ".join(sys.argv))])) - - if config.EVERGREEN_TASK_ID: - local_args = parser.to_local_args() - self._resmoke_logger.info("resmoke.py invocation for local usage: %s %s", - os.path.join("buildscripts", "resmoke.py"), - " ".join(local_args)) - - suites = None - try: - suites = self._get_suites() - self._setup_archival() - if config.SPAWN_USING == "jasper": - self._setup_jasper() - self._setup_signal_handler(suites) - - for suite in suites: - self._interrupted = self._run_suite(suite) - if self._interrupted or (suite.options.fail_fast and suite.return_code != 0): - self._log_resmoke_summary(suites) - self.exit(suite.return_code) - - self._log_resmoke_summary(suites) - - # Exit with a nonzero code if any of the suites failed. - exit_code = max(suite.return_code for suite in suites) - self.exit(exit_code) - finally: - if config.SPAWN_USING == "jasper": - self._exit_jasper() - self._exit_archival() - if suites: - reportfile.write(suites) - - def _run_suite(self, suite): - """Run a test suite.""" - self._log_suite_config(suite) - suite.record_suite_start() - interrupted = self._execute_suite(suite) - suite.record_suite_end() - self._log_suite_summary(suite) - return interrupted - - def _log_resmoke_summary(self, suites): - """Log a summary of the resmoke run.""" - time_taken = time.time() - self.__start_time - if len(self._config.suite_files) > 1: - testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken) - - def _log_suite_summary(self, suite): - """Log a summary of the suite run.""" - self._resmoke_logger.info("=" * 80) - self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(), - self._get_suite_summary(suite)) - - def _execute_suite(self, suite): - """Execute a suite and return True if interrupted, False otherwise.""" - self._shuffle_tests(suite) - if not suite.tests: - self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind) - suite.return_code = 0 - return False - executor_config = suite.get_executor_config() - try: - executor = testing.executor.TestSuiteExecutor( - self._exec_logger, suite, archive_instance=self._archive, **executor_config) - executor.run() - except (errors.UserInterrupt, errors.LoggerRuntimeConfigError) as err: - self._exec_logger.error("Encountered an error when running %ss of suite %s: %s", - suite.test_kind, suite.get_display_name(), err) - suite.return_code = err.EXIT_CODE - return True - except IOError: - suite.return_code = 74 # Exit code for IOError on POSIX systems. - return True - except: # pylint: disable=bare-except - self._exec_logger.exception("Encountered an error when running %ss of suite %s.", - suite.test_kind, suite.get_display_name()) - suite.return_code = 2 - return False - return False - - def _shuffle_tests(self, suite): - """Shuffle the tests if the shuffle cli option was set.""" - random.seed(config.RANDOM_SEED) - if not config.SHUFFLE: - return - self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.", - suite.test_kind, suite.get_display_name(), config.RANDOM_SEED) - random.shuffle(suite.tests) - - def _get_suites(self): - """Return the list of suites for this resmoke invocation.""" - try: - return suitesconfig.get_suites(self._config.suite_files, self._config.test_files) - except errors.SuiteNotFound as err: - self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err)) - self.list_suites() - self.exit(1) - - def _log_suite_config(self, suite): - sb = [ - "YAML configuration of suite {}".format(suite.get_display_name()), - utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "", - utils.dump_yaml({"selector": suite.get_selector_config()}), "", - utils.dump_yaml({"executor": suite.get_executor_config()}), "", - utils.dump_yaml({"logging": self._config.logging_config}) - ] - self._resmoke_logger.info("\n".join(sb)) - - @staticmethod - def _get_suite_summary(suite): - """Return a summary of the suite run.""" - sb = [] - suite.summarize(sb) - return "\n".join(sb) - - def _setup_signal_handler(self, suites): - """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump.""" - sighandler.register(self._resmoke_logger, suites, self.__start_time) - - def _setup_archival(self): - """Set up the archival feature if enabled in the cli options.""" - if config.ARCHIVE_FILE: - self._archive = utils.archival.Archival( - archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB, - limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger) - - def _exit_archival(self): - """Finish up archival tasks before exit if enabled in the cli options.""" - if self._archive and not self._interrupted: - self._archive.exit() - - # pylint: disable=too-many-instance-attributes,too-many-statements,too-many-locals - def _setup_jasper(self): - """Start up the jasper process manager.""" - root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto") - try: - well_known_protos_include = pkg_resources.resource_filename("grpc_tools", "_proto") - except ImportError: - raise ImportError("You must run: sys.executable + '-m pip install grpcio grpcio-tools " - "googleapis-common-protos' to use --spawnUsing=jasper.") - - # We use the build/ directory as the output directory because the generated files aren't - # meant to because tracked by git or linted. - proto_out = os.path.join(root_dir, "build", "jasper") - - utils.rmtree(proto_out, ignore_errors=True) - os.makedirs(proto_out) - - # We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the - # *pb2*.py modules from it. - with open(os.path.join(proto_out, "__init__.py"), "w"): - pass - - ret = grpc_tools.protoc.main([ - grpc_tools.protoc.__file__, - "--grpc_python_out", - proto_out, - "--python_out", - proto_out, - "--proto_path", - os.path.dirname(proto_file), - "--proto_path", - well_known_protos_include, - os.path.basename(proto_file), - ]) - - if ret != 0: - raise RuntimeError("Failed to generated gRPC files from the jasper.proto file") - - sys.path.append(os.path.dirname(proto_out)) - - from jasper import jasper_pb2 - from jasper import jasper_pb2_grpc - - jasper_process.Process.jasper_pb2 = jasper_pb2 - jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc - - curator_path = "build/curator" - if sys.platform == "win32": - curator_path += ".exe" - git_hash = "d846f0c875716e9377044ab2a50542724369662a" - curator_exists = os.path.isfile(curator_path) - curator_same_version = False - if curator_exists: - curator_version = subprocess.check_output([curator_path, - "--version"]).decode('utf-8').split() - curator_same_version = git_hash in curator_version - - if curator_exists and not curator_same_version: - os.remove(curator_path) - self._resmoke_logger.info( - "Found a different version of curator. Downloading version %s of curator to enable" - "process management using jasper.", git_hash) - - if not curator_exists or not curator_same_version: - if sys.platform == "darwin": - os_platform = "macos" - elif sys.platform == "win32": - os_platform = "windows-64" - elif sys.platform.startswith("linux"): - os_platform = "ubuntu1604" - else: - raise OSError("Unrecognized platform. " - "This program is meant to be run on MacOS, Windows, or Linux.") - url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/" - "curator-dist-%s-%s.tar.gz") % (os_platform, git_hash) - response = requests.get(url, stream=True) - with tarfile.open(mode="r|gz", fileobj=response.raw) as tf: - tf.extractall(path="./build/") - - jasper_port = config.BASE_PORT - 1 - jasper_conn_str = "localhost:%d" % jasper_port - jasper_process.Process.connection_str = jasper_conn_str - jasper_command = [curator_path, "jasper", "grpc", "--port", str(jasper_port)] - self._jasper_server = process.Process(self._resmoke_logger, jasper_command) - self._jasper_server.start() - - channel = grpc.insecure_channel(jasper_conn_str) - grpc.channel_ready_future(channel).result() - - def _exit_jasper(self): - if self._jasper_server: - self._jasper_server.stop() - - def exit(self, exit_code): - """Exit with the provided exit code.""" - self._exit_code = exit_code - self._resmoke_logger.info("Exiting with code: %d", exit_code) - sys.exit(exit_code) def main(): """Execute Main function for resmoke.""" - resmoke = Resmoke() - resmoke.configure_from_command_line() - resmoke.run() + __start_time = time.time() + subcommand = parser.parse_command_line(sys.argv[1:], start_time=__start_time) + subcommand.execute() if __name__ == "__main__": diff --git a/buildscripts/resmokelib/commands/__init__.py b/buildscripts/resmokelib/commands/__init__.py new file mode 100644 index 00000000000..1a3235631a7 --- /dev/null +++ b/buildscripts/resmokelib/commands/__init__.py @@ -0,0 +1,4 @@ +"""Resmokelib subcommands.""" + +from . import interface +from . import run diff --git a/buildscripts/resmokelib/commands/interface.py b/buildscripts/resmokelib/commands/interface.py new file mode 100644 index 00000000000..c436b758f9c --- /dev/null +++ b/buildscripts/resmokelib/commands/interface.py @@ -0,0 +1,9 @@ +"""Interface for creating a resmoke subcommand.""" + + +class Subcommand(object): + """A resmoke subcommand to execute.""" + + def execute(self): + """Execute the subcommand.""" + raise NotImplementedError("execue must be implemented by Subcommand subclasses") diff --git a/buildscripts/resmokelib/commands/run.py b/buildscripts/resmokelib/commands/run.py new file mode 100644 index 00000000000..9f7afb964cd --- /dev/null +++ b/buildscripts/resmokelib/commands/run.py @@ -0,0 +1,525 @@ +"""Command line utility for executing MongoDB tests of all kinds.""" + +import collections +import os +import os.path +import random +import shlex +import subprocess +import sys +import tarfile +import time + +import pkg_resources +import requests + +try: + import grpc_tools.protoc + import grpc +except ImportError: + pass + +# pylint: disable=wrong-import-position +from buildscripts.resmokelib import config +from buildscripts.resmokelib import errors +from buildscripts.resmokelib import logging +from buildscripts.resmokelib import reportfile +from buildscripts.resmokelib import sighandler +from buildscripts.resmokelib import suitesconfig +from buildscripts.resmokelib import testing +from buildscripts.resmokelib import utils + +from buildscripts.resmokelib.commands import interface + +from buildscripts.resmokelib.core import process +from buildscripts.resmokelib.core import jasper_process + + +class TestRunner(interface.Subcommand): # pylint: disable=too-many-instance-attributes + """The main class to run tests with resmoke.""" + + def __init__(self, command, start_time=time.time()): + """Initialize the Resmoke instance.""" + self.__start_time = start_time + self.__command = command + self._exec_logger = None + self._resmoke_logger = None + self._archive = None + self._jasper_server = None + self._interrupted = False + self._exit_code = 0 + + def _setup_logging(self): + logging.loggers.configure_loggers(config.LOGGING_CONFIG) + logging.flush.start_thread() + self._exec_logger = logging.loggers.EXECUTOR_LOGGER + self._resmoke_logger = self._exec_logger.new_resmoke_logger() + + def _exit_logging(self): + if self._interrupted: + # We want to exit as quickly as possible when interrupted by a user and therefore don't + # bother waiting for all log output to be flushed to logkeeper. + return + + if logging.buildlogger.is_log_output_incomplete(): + # If we already failed to write log output to logkeeper, then we don't bother waiting + # for any remaining log output to be flushed as it'll likely fail too. Exiting without + # joining the flush thread here also means that resmoke.py won't hang due a logger from + # a fixture or a background hook not being closed. + self._exit_on_incomplete_logging() + + flush_success = logging.flush.stop_thread() + if not flush_success: + self._resmoke_logger.error( + 'Failed to flush all logs within a reasonable amount of time, ' + 'treating logs as incomplete') + + if not flush_success or logging.buildlogger.is_log_output_incomplete(): + self._exit_on_incomplete_logging() + + def _exit_on_incomplete_logging(self): + if self._exit_code == 0: + # We don't anticipate users to look at passing Evergreen tasks very often that even if + # the log output is incomplete, we'd still rather not show anything in the Evergreen UI + # or cause a JIRA ticket to be created. + self._resmoke_logger.info( + "We failed to flush all log output to logkeeper but all tests passed, so" + " ignoring.") + else: + exit_code = errors.LoggerRuntimeConfigError.EXIT_CODE + self._resmoke_logger.info( + "Exiting with code %d rather than requested code %d because we failed to flush all" + " log output to logkeeper.", exit_code, self._exit_code) + self._exit_code = exit_code + + # Force exit the process without cleaning up or calling the finally block + # to avoid threads making system calls from blocking process termination. + # This must be the last line of code that is run. + # pylint: disable=protected-access + os._exit(self._exit_code) + + def execute(self): + """Execute the 'run' subcommand.""" + self._setup_logging() + + try: + if self.__command == "list-suites": + self.list_suites() + elif self.__command == "find-suites": + self.find_suites() + elif config.DRY_RUN == "tests": + self.dry_run() + else: + self.run_tests() + finally: + # self._exit_logging() may never return when the log output is incomplete. + # Our workaround is to call os._exit(). + self._exit_logging() + + def list_suites(self): + """List the suites that are available to execute.""" + suite_names = suitesconfig.get_named_suites() + self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names)) + + def find_suites(self): + """List the suites that run the specified tests.""" + suites = self._get_suites() + suites_by_test = self._find_suites_by_test(suites) + for test in sorted(suites_by_test): + suite_names = suites_by_test[test] + self._resmoke_logger.info("%s will be run by the following suite(s): %s", test, + suite_names) + + @staticmethod + def _find_suites_by_test(suites): + """ + Look up what other resmoke suites run the tests specified in the suites parameter. + + Return a dict keyed by test name, value is array of suite names. + """ + memberships = {} + test_membership = suitesconfig.create_test_membership_map() + for suite in suites: + for test in suite.tests: + memberships[test] = test_membership[test] + return memberships + + def dry_run(self): + """List which tests would run and which tests would be excluded in a resmoke invocation.""" + suites = self._get_suites() + for suite in suites: + self._shuffle_tests(suite) + sb = ["Tests that would be run in suite {}".format(suite.get_display_name())] + sb.extend(suite.tests or ["(no tests)"]) + sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name())) + sb.extend(suite.excluded or ["(no tests)"]) + self._exec_logger.info("\n".join(sb)) + + def run_tests(self): + """Run the suite and tests specified.""" + self._resmoke_logger.info( + "verbatim resmoke.py invocation: %s", + " ".join([shlex.quote(arg) for arg in shlex.split(" ".join(sys.argv))])) + + # TODO: SERVER-47611 + # if config.EVERGREEN_TASK_ID: + # local_args = parser.to_local_args() + # self._resmoke_logger.info("resmoke.py invocation for local usage: %s %s", + # os.path.join("buildscripts", "resmoke.py"), + # " ".join(local_args)) + + suites = None + try: + suites = self._get_suites() + self._setup_archival() + if config.SPAWN_USING == "jasper": + self._setup_jasper() + self._setup_signal_handler(suites) + + for suite in suites: + self._interrupted = self._run_suite(suite) + if self._interrupted or (suite.options.fail_fast and suite.return_code != 0): + self._log_resmoke_summary(suites) + self.exit(suite.return_code) + + self._log_resmoke_summary(suites) + + # Exit with a nonzero code if any of the suites failed. + exit_code = max(suite.return_code for suite in suites) + self.exit(exit_code) + finally: + if config.SPAWN_USING == "jasper": + self._exit_jasper() + self._exit_archival() + if suites: + reportfile.write(suites) + + def _run_suite(self, suite): + """Run a test suite.""" + self._log_suite_config(suite) + suite.record_suite_start() + interrupted = self._execute_suite(suite) + suite.record_suite_end() + self._log_suite_summary(suite) + return interrupted + + def _log_resmoke_summary(self, suites): + """Log a summary of the resmoke run.""" + time_taken = time.time() - self.__start_time + if len(config.SUITE_FILES) > 1: + testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken) + + def _log_suite_summary(self, suite): + """Log a summary of the suite run.""" + self._resmoke_logger.info("=" * 80) + self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(), + self._get_suite_summary(suite)) + + def _execute_suite(self, suite): + """Execute a suite and return True if interrupted, False otherwise.""" + self._shuffle_tests(suite) + if not suite.tests: + self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind) + suite.return_code = 0 + return False + executor_config = suite.get_executor_config() + try: + executor = testing.executor.TestSuiteExecutor( + self._exec_logger, suite, archive_instance=self._archive, **executor_config) + executor.run() + except (errors.UserInterrupt, errors.LoggerRuntimeConfigError) as err: + self._exec_logger.error("Encountered an error when running %ss of suite %s: %s", + suite.test_kind, suite.get_display_name(), err) + suite.return_code = err.EXIT_CODE + return True + except IOError: + suite.return_code = 74 # Exit code for IOError on POSIX systems. + return True + except: # pylint: disable=bare-except + self._exec_logger.exception("Encountered an error when running %ss of suite %s.", + suite.test_kind, suite.get_display_name()) + suite.return_code = 2 + return False + return False + + def _shuffle_tests(self, suite): + """Shuffle the tests if the shuffle cli option was set.""" + random.seed(config.RANDOM_SEED) + if not config.SHUFFLE: + return + self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.", + suite.test_kind, suite.get_display_name(), config.RANDOM_SEED) + random.shuffle(suite.tests) + + def _get_suites(self): + """Return the list of suites for this resmoke invocation.""" + try: + return suitesconfig.get_suites(config.SUITE_FILES, config.TEST_FILES) + except errors.SuiteNotFound as err: + self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err)) + self.list_suites() + self.exit(1) + + def _log_suite_config(self, suite): + sb = [ + "YAML configuration of suite {}".format(suite.get_display_name()), + utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "", + utils.dump_yaml({"selector": suite.get_selector_config()}), "", + utils.dump_yaml({"executor": suite.get_executor_config()}), "", + utils.dump_yaml({"logging": config.LOGGING_CONFIG}) + ] + self._resmoke_logger.info("\n".join(sb)) + + @staticmethod + def _get_suite_summary(suite): + """Return a summary of the suite run.""" + sb = [] + suite.summarize(sb) + return "\n".join(sb) + + def _setup_signal_handler(self, suites): + """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump.""" + sighandler.register(self._resmoke_logger, suites, self.__start_time) + + def _setup_archival(self): + """Set up the archival feature if enabled in the cli options.""" + if config.ARCHIVE_FILE: + self._archive = utils.archival.Archival( + archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB, + limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger) + + def _exit_archival(self): + """Finish up archival tasks before exit if enabled in the cli options.""" + if self._archive and not self._interrupted: + self._archive.exit() + + # pylint: disable=too-many-instance-attributes,too-many-statements,too-many-locals + def _setup_jasper(self): + """Start up the jasper process manager.""" + root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto") + try: + well_known_protos_include = pkg_resources.resource_filename("grpc_tools", "_proto") + except ImportError: + raise ImportError("You must run: sys.executable + '-m pip install grpcio grpcio-tools " + "googleapis-common-protos' to use --spawnUsing=jasper.") + + # We use the build/ directory as the output directory because the generated files aren't + # meant to because tracked by git or linted. + proto_out = os.path.join(root_dir, "build", "jasper") + + utils.rmtree(proto_out, ignore_errors=True) + os.makedirs(proto_out) + + # We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the + # *pb2*.py modules from it. + with open(os.path.join(proto_out, "__init__.py"), "w"): + pass + + ret = grpc_tools.protoc.main([ + grpc_tools.protoc.__file__, + "--grpc_python_out", + proto_out, + "--python_out", + proto_out, + "--proto_path", + os.path.dirname(proto_file), + "--proto_path", + well_known_protos_include, + os.path.basename(proto_file), + ]) + + if ret != 0: + raise RuntimeError("Failed to generated gRPC files from the jasper.proto file") + + sys.path.append(os.path.dirname(proto_out)) + + from jasper import jasper_pb2 + from jasper import jasper_pb2_grpc + + jasper_process.Process.jasper_pb2 = jasper_pb2 + jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc + + curator_path = "build/curator" + if sys.platform == "win32": + curator_path += ".exe" + git_hash = "d846f0c875716e9377044ab2a50542724369662a" + curator_exists = os.path.isfile(curator_path) + curator_same_version = False + if curator_exists: + curator_version = subprocess.check_output([curator_path, + "--version"]).decode('utf-8').split() + curator_same_version = git_hash in curator_version + + if curator_exists and not curator_same_version: + os.remove(curator_path) + self._resmoke_logger.info( + "Found a different version of curator. Downloading version %s of curator to enable" + "process management using jasper.", git_hash) + + if not curator_exists or not curator_same_version: + if sys.platform == "darwin": + os_platform = "macos" + elif sys.platform == "win32": + os_platform = "windows-64" + elif sys.platform.startswith("linux"): + os_platform = "ubuntu1604" + else: + raise OSError("Unrecognized platform. " + "This program is meant to be run on MacOS, Windows, or Linux.") + url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/" + "curator-dist-%s-%s.tar.gz") % (os_platform, git_hash) + response = requests.get(url, stream=True) + with tarfile.open(mode="r|gz", fileobj=response.raw) as tf: + tf.extractall(path="./build/") + + jasper_port = config.BASE_PORT - 1 + jasper_conn_str = "localhost:%d" % jasper_port + jasper_process.Process.connection_str = jasper_conn_str + jasper_command = [curator_path, "jasper", "grpc", "--port", str(jasper_port)] + self._jasper_server = process.Process(self._resmoke_logger, jasper_command) + self._jasper_server.start() + + channel = grpc.insecure_channel(jasper_conn_str) + grpc.channel_ready_future(channel).result() + + def _exit_jasper(self): + if self._jasper_server: + self._jasper_server.stop() + + def exit(self, exit_code): + """Exit with the provided exit code.""" + self._exit_code = exit_code + self._resmoke_logger.info("Exiting with code: %d", exit_code) + sys.exit(exit_code) + + +_TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"]) + + +class TestRunnerEvg(TestRunner): + """Execute Main class. + + A class for executing potentially multiple resmoke.py test suites in a way that handles + additional options for running unreliable tests in Evergreen. + """ + + UNRELIABLE_TAG = _TagInfo( + tag_name="unreliable", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + report_failure_status="silentfail")) + + RESOURCE_INTENSIVE_TAG = _TagInfo( + tag_name="resource_intensive", + evergreen_aware=False, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + num_jobs=1)) + + RETRY_ON_FAILURE_TAG = _TagInfo( + tag_name="retry_on_failure", + evergreen_aware=True, + suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore + fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, + report_failure_status="silentfail")) + + @staticmethod + def _make_evergreen_aware_tags(tag_name): + """Return a list of resmoke.py tags. + + This list is for task, variant, and distro combinations in Evergreen. + """ + + tags_format = ["{tag_name}"] + + if config.EVERGREEN_TASK_NAME is not None: + tags_format.append("{tag_name}|{task_name}") + + if config.EVERGREEN_VARIANT_NAME is not None: + tags_format.append("{tag_name}|{task_name}|{variant_name}") + + if config.EVERGREEN_DISTRO_ID is not None: + tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}") + + return [ + tag.format(tag_name=tag_name, task_name=config.EVERGREEN_TASK_NAME, + variant_name=config.EVERGREEN_VARIANT_NAME, + distro_id=config.EVERGREEN_DISTRO_ID) for tag in tags_format + ] + + @classmethod + def _make_tag_combinations(cls): + """Return a list of (tag, enabled) pairs. + + These pairs represent all possible combinations of all possible pairings + of whether the tags are enabled or disabled together. + """ + + combinations = [] + + if config.EVERGREEN_PATCH_BUILD: + combinations.append(("unreliable and resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("unreliable and not resource intensive", + ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False)))) + combinations.append(("reliable and resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True)))) + combinations.append(("reliable and not resource intensive", + ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + else: + combinations.append(("retry on failure and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("retry on failure and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + combinations.append(("run once and resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + True)))) + combinations.append(("run once and not resource intensive", + ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, + False)))) + + return combinations + + def _get_suites(self): + """Return a list of resmokelib.testing.suite.Suite instances to execute. + + For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(), + multiple copies of that test suite are run using different resmokelib.config.SuiteOptions() + depending on whether each tag in the combination is enabled or not. + """ + + suites = [] + + for suite in TestRunner._get_suites(self): + if suite.test_kind != "js_test": + # Tags are only support for JavaScript tests, so we leave the test suite alone when + # running any other kind of test. + suites.append(suite) + continue + + for (tag_desc, tag_combo) in self._make_tag_combinations(): + suite_options_list = [] + + for (tag_info, enabled) in tag_combo: + if tag_info.evergreen_aware: + tags = self._make_evergreen_aware_tags(tag_info.tag_name) + include_tags = {"$anyOf": tags} + else: + include_tags = tag_info.tag_name + + if enabled: + suite_options = tag_info.suite_options._replace(include_tags=include_tags) + else: + suite_options = config.SuiteOptions.ALL_INHERITED._replace( + include_tags={"$not": include_tags}) + + suite_options_list.append(suite_options) + + suite_options = config.SuiteOptions.combine(*suite_options_list) + suite_options = suite_options._replace(description=tag_desc) + suites.append(suite.with_options(suite_options)) + + return suites diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index 8296f599e54..371b87da34d 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -62,6 +62,7 @@ DEFAULTS = { "include_with_any_tags": None, "install_dir": None, "jobs": 1, + "logger_file": None, "mongo_executable": None, "mongod_executable": None, "mongod_set_parameters": None, @@ -89,7 +90,9 @@ DEFAULTS = { "majority_read_concern": None, # Default is set on the commandline. "storage_engine": None, "storage_engine_cache_size_gb": None, + "suite_files": None, "tag_file": None, + "test_files": None, "transport_layer": None, "mixed_bin_versions": None, "linear_chain": None, @@ -311,6 +314,9 @@ INTERNAL_PARAMS = [] # If set, then resmoke.py starts the specified number of Job instances to run tests. JOBS = None +# Yaml file that specified logging configuration. +LOGGER_FILE = None + # Where to find the MONGO*_EXECUTABLE binaries INSTALL_DIR = None @@ -420,9 +426,15 @@ STORAGE_ENGINE = None # storage engine cache size. STORAGE_ENGINE_CACHE_SIZE = None +# Yaml suites that specify how tests should be executed. +SUITE_FILES = None + # The tag file to use that associates tests with tags. TAG_FILE = None +# The test files to execute. +TEST_FILES = None + # If set, then mongod/mongos's started by resmoke.py will use the specified transport layer. TRANSPORT_LAYER = None @@ -476,3 +488,6 @@ EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST, CONFIG_DIR = None NAMED_SUITES = None LOGGER_DIR = None + +# Generated logging config for the current invocation. +LOGGING_CONFIG = None diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py new file mode 100644 index 00000000000..9210dd4b563 --- /dev/null +++ b/buildscripts/resmokelib/configure_resmoke.py @@ -0,0 +1,289 @@ +"""Configure the command line input for the resmoke 'run' subcommand.""" + +import os +import os.path +import sys +import shlex +import configparser + +from typing import NamedTuple + +import datetime +import pymongo.uri_parser + +from . import config as _config +from . import utils + + +def validate_and_update_config(parser, args): + """Validate inputs and update config module.""" + _validate_options(parser, args) + _update_config_vars(args) + _validate_config(parser) + _set_logging_config() + + +def _validate_options(parser, args): + """Do preliminary validation on the options and error on any invalid options.""" + + if not 'shell_port' in args or not 'shell_conn_strin' in args: + return + + if args.shell_port is not None and args.shell_conn_string is not None: + parser.error("Cannot specify both `shellPort` and `shellConnString`") + + if args.executor_file: + parser.error("--executor is superseded by --suites; specify --suites={} {} to run the" + " test(s) under those suite configuration(s)".format( + args.executor_file, " ".join(args.test_files))) + + +def _validate_config(parser): + """Do validation on the config settings.""" + + if _config.REPEAT_TESTS_MAX: + if not _config.REPEAT_TESTS_SECS: + parser.error("Must specify --repeatTestsSecs with --repeatTestsMax") + + if _config.REPEAT_TESTS_MIN > _config.REPEAT_TESTS_MAX: + parser.error("--repeatTestsSecsMin > --repeatTestsMax") + + if _config.REPEAT_TESTS_MIN and not _config.REPEAT_TESTS_SECS: + parser.error("Must specify --repeatTestsSecs with --repeatTestsMin") + + if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS: + parser.error("Cannot specify --repeatTests and --repeatTestsSecs") + + if _config.MIXED_BIN_VERSIONS is not None: + for version in _config.MIXED_BIN_VERSIONS: + if version not in set(['old', 'new']): + parser.error("Must specify binary versions as 'old' or 'new' in format" + " 'version1-version2'") + + +def _update_config_vars(values): # pylint: disable=too-many-statements,too-many-locals,too-many-branches + """Update the variables of the config module.""" + + config = _config.DEFAULTS.copy() + + # Use RSK_ prefixed environment variables to indicate resmoke-specific values. + # The list of configuration is detailed in config.py + resmoke_env_prefix = 'RSK_' + for key in os.environ.keys(): + if key.startswith(resmoke_env_prefix): + # Windows env vars are case-insensitive, we use lowercase to be consistent + # with existing resmoke options. + config[key[len(resmoke_env_prefix):].lower()] = os.environ[key] + + # Override `config` with values from command line arguments. + cmdline_vars = vars(values) + for cmdline_key in cmdline_vars: + if cmdline_key not in _config.DEFAULTS: + # Ignore options that don't map to values in config.py + continue + if cmdline_vars[cmdline_key] is not None: + config[cmdline_key] = cmdline_vars[cmdline_key] + + if os.path.isfile("resmoke.ini"): + config_parser = configparser.ConfigParser() + config_parser.read("resmoke.ini") + if "resmoke" in config_parser.sections(): + user_config = dict(config_parser["resmoke"]) + config.update(user_config) + + _config.ALWAYS_USE_LOG_FILES = config.pop("always_use_log_files") + _config.IS_ASAN_BUILD = config.pop("is_asan_build") + _config.BASE_PORT = int(config.pop("base_port")) + _config.BUILDLOGGER_URL = config.pop("buildlogger_url") + _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix")) + _config.DRY_RUN = config.pop("dry_run") + # EXCLUDE_WITH_ANY_TAGS will always contain the implicitly defined EXCLUDED_TAG. + _config.EXCLUDE_WITH_ANY_TAGS = [_config.EXCLUDED_TAG] + _config.EXCLUDE_WITH_ANY_TAGS.extend( + utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) + _config.FAIL_FAST = not config.pop("continue_on_failure") + _config.FLOW_CONTROL = config.pop("flow_control") + _config.FLOW_CONTROL_TICKETS = config.pop("flow_control_tickets") + _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags")) + _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable")) + _config.JOBS = config.pop("jobs") + _config.LINEAR_CHAIN = config.pop("linear_chain") == "on" + _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on" + _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions") + if _config.MIXED_BIN_VERSIONS is not None: + _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-") + + _config.INSTALL_DIR = config.pop("install_dir") + if _config.INSTALL_DIR is not None: + # Normalize the path so that on Windows dist-test/bin + # translates to .\dist-test\bin then absolutify it since the + # Windows PATH variable requires absolute paths. + _config.INSTALL_DIR = os.path.abspath(_expand_user(os.path.normpath(_config.INSTALL_DIR))) + + for binary in ["mongo", "mongod", "mongos", "dbtest"]: + keyname = binary + "_executable" + if config.get(keyname, None) is None: + config[keyname] = os.path.join(_config.INSTALL_DIR, binary) + + _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable")) + _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable")) + _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) + _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters") + _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) + + _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters") + _config.NO_JOURNAL = config.pop("no_journal") + _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture") + _config.NUM_REPLSET_NODES = config.pop("num_replset_nodes") + _config.NUM_SHARDS = config.pop("num_shards") + _config.PERF_REPORT_FILE = config.pop("perf_report_file") + _config.RANDOM_SEED = config.pop("seed") + _config.REPEAT_SUITES = config.pop("repeat_suites") + _config.REPEAT_TESTS = config.pop("repeat_tests") + _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") + _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") + _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") + _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") + _config.REPORT_FILE = config.pop("report_file") + _config.SERVICE_EXECUTOR = config.pop("service_executor") + _config.SHELL_READ_MODE = config.pop("shell_read_mode") + _config.SHELL_WRITE_MODE = config.pop("shell_write_mode") + _config.SPAWN_USING = config.pop("spawn_using") + _config.STAGGER_JOBS = config.pop("stagger_jobs") == "on" + _config.STORAGE_ENGINE = config.pop("storage_engine") + _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storage_engine_cache_size_gb") + _config.SUITE_FILES = config.pop("suite_files") + if _config.SUITE_FILES is not None: + _config.SUITE_FILES = _config.SUITE_FILES.split(",") + _config.TAG_FILE = config.pop("tag_file") + _config.TEST_FILES = config.pop("test_files") + _config.TRANSPORT_LAYER = config.pop("transport_layer") + + # Internal testing options. + _config.INTERNAL_PARAMS = config.pop("internal_params") + + # Evergreen options. + _config.EVERGREEN_BUILD_ID = config.pop("build_id") + _config.EVERGREEN_DISTRO_ID = config.pop("distro_id") + _config.EVERGREEN_EXECUTION = config.pop("execution_number") + _config.EVERGREEN_PATCH_BUILD = config.pop("patch_build") + _config.EVERGREEN_PROJECT_NAME = config.pop("project_name") + _config.EVERGREEN_REVISION = config.pop("git_revision") + _config.EVERGREEN_REVISION_ORDER_ID = config.pop("revision_order_id") + _config.EVERGREEN_TASK_ID = config.pop("task_id") + _config.EVERGREEN_TASK_NAME = config.pop("task_name") + _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") + _config.EVERGREEN_VERSION_ID = config.pop("version_id") + + # Archival options. Archival is enabled only when running on evergreen. + if not _config.EVERGREEN_TASK_ID: + _config.ARCHIVE_FILE = None + _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb") + _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests") + + # Wiredtiger options. + _config.WT_COLL_CONFIG = config.pop("wt_coll_config") + _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config") + _config.WT_INDEX_CONFIG = config.pop("wt_index_config") + + # Benchmark/Benchrun options. + _config.BENCHMARK_FILTER = config.pop("benchmark_filter") + _config.BENCHMARK_LIST_TESTS = config.pop("benchmark_list_tests") + benchmark_min_time = config.pop("benchmark_min_time_secs") + if benchmark_min_time is not None: + _config.BENCHMARK_MIN_TIME = datetime.timedelta(seconds=benchmark_min_time) + _config.BENCHMARK_REPETITIONS = config.pop("benchmark_repetitions") + + # Config Dir options. + _config.CONFIG_DIR = config.pop("config_dir") + + # Populate the named suites by scanning config_dir/suites + named_suites = {} + + suites_dir = os.path.join(_config.CONFIG_DIR, "suites") + root = os.path.abspath(suites_dir) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml"): + pathname = os.path.join(root, filename) + named_suites[short_name] = pathname + + _config.NAMED_SUITES = named_suites + + _config.LOGGER_DIR = os.path.join(_config.CONFIG_DIR, "loggers") + + shuffle = config.pop("shuffle") + if shuffle == "auto": + # If the user specified a value for --jobs > 1 (or -j > 1), then default to randomize + # the order in which tests are executed. This is because with multiple threads the tests + # wouldn't run in a deterministic order anyway. + _config.SHUFFLE = _config.JOBS > 1 + else: + _config.SHUFFLE = shuffle == "on" + + conn_string = config.pop("shell_conn_string") + port = config.pop("shell_port") + + if port is not None: + conn_string = "mongodb://localhost:" + port + + if conn_string is not None: + # The --shellConnString command line option must be a MongoDB connection URI, which means it + # must specify the mongodb:// or mongodb+srv:// URI scheme. pymongo.uri_parser.parse_uri() + # raises an exception if the connection string specified isn't considered a valid MongoDB + # connection URI. + pymongo.uri_parser.parse_uri(conn_string) + _config.SHELL_CONN_STRING = conn_string + + _config.LOGGER_FILE = config.pop("logger_file") + + if config: + raise ValueError(f"Unkown option(s): {list(config.keys())}s") + + +def _set_logging_config(): + """Read YAML configuration from 'pathname' how to log tests and fixtures.""" + pathname = _config.LOGGER_FILE + try: + # If the user provides a full valid path to a logging config + # we don't need to search LOGGER_DIR for the file. + if os.path.exists(pathname): + _config.LOGGING_CONFIG = utils.load_yaml_file(pathname).pop("logging") + return + + root = os.path.abspath(_config.LOGGER_DIR) + files = os.listdir(root) + for filename in files: + (short_name, ext) = os.path.splitext(filename) + if ext in (".yml", ".yaml") and short_name == pathname: + config_file = os.path.join(root, filename) + if not os.path.isfile(config_file): + raise ValueError("Expected a logger YAML config, but got '%s'" % pathname) + _config.LOGGING_CONFIG = utils.load_yaml_file(config_file).pop("logging") + return + + raise ValueError("Unknown logger '%s'" % pathname) + except FileNotFoundError: + raise IOError("Directory {} does not exist.".format(_config.LOGGER_DIR)) + + +def _expand_user(pathname): + """Provide wrapper around os.path.expanduser() to do nothing when given None.""" + if pathname is None: + return None + return os.path.expanduser(pathname) + + +def _tags_from_list(tags_list): + """Return the list of tags from a list of tag parameter values. + + Each parameter value in the list may be a list of comma separated tags, with empty strings + ignored. + """ + tags = [] + if tags_list is not None: + for tag in tags_list: + tags.extend([t for t in tag.split(",") if t != ""]) + return tags + return None diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 2fd2e2f1568..3cbcfa397f8 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -1,789 +1,570 @@ """Parser for command line arguments.""" -import collections import os -import os.path import sys import shlex -import configparser -import datetime -import optparse -import pymongo.uri_parser +import argparse from . import config as _config -from . import utils +from . import configure_resmoke +from . import commands -ResmokeConfig = collections.namedtuple( - "ResmokeConfig", - ["list_suites", "find_suites", "dry_run", "suite_files", "test_files", "logging_config"]) +_EVERGREEN_ARGUMENT_TITLE = "Evergreen options" -_EVERGREEN_OPTIONS_TITLE = "Evergreen options" - -def _make_parser(): # pylint: disable=too-many-statements +def _make_parser(): """Create and return the command line arguments parser.""" - parser = optparse.OptionParser() + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command") + + # Add sub-commands. + _add_run(subparsers) + _add_list_suites(subparsers) + _add_find_suites(subparsers) + + return parser + - parser.add_option( +def _add_run(subparsers): # pylint: disable=too-many-statements + """Create and add the parser for the Run subcommand.""" + parser = subparsers.add_parser("run", help="Runs the specified tests.") + + parser.set_defaults(dry_run="off", shuffle="auto", stagger_jobs="off", + suite_files="with_server", majority_read_concern="on") + + parser.add_argument("test_files", metavar="TEST_FILES", nargs="*", + help="Explicit test files to run") + + parser.add_argument( "--suites", dest="suite_files", metavar="SUITE1,SUITE2", help=("Comma separated list of YAML files that each specify the configuration" " of a suite. If the file is located in the resmokeconfig/suites/" " directory, then the basename without the .yml extension can be" " specified, e.g. 'core'. If a list of files is passed in as" " positional arguments, they will be run using the suites'" - " configurations")) - - parser.add_option( - "--log", dest="logger_file", metavar="LOGGER", - help=("A YAML file that specifies the logging configuration. If the file is" - " located in the resmokeconfig/suites/ directory, then the basename" - " without the .yml extension can be specified, e.g. 'console'.")) + " configurations.")) - parser.add_option("--configDir", dest="config_dir", metavar="CONFIG_DIR", - help="Directory to search for resmoke configuration files") + parser.add_argument("--configDir", dest="config_dir", metavar="CONFIG_DIR", + help="Directory to search for resmoke configuration files") - parser.add_option("--installDir", dest="install_dir", metavar="INSTALL_DIR", - help="Directory to search for MongoDB binaries") + parser.add_argument("--installDir", dest="install_dir", metavar="INSTALL_DIR", + help="Directory to search for MongoDB binaries") - parser.add_option( + parser.add_argument( "--alwaysUseLogFiles", dest="always_use_log_files", action="store_true", help=("Logs server output to a file located in the db path and prevents the" " cleaning of dbpaths after testing. Note that conflicting options" " passed in from test files may cause an error.")) - parser.add_option( - "--archiveLimitMb", type="int", dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB", + parser.add_argument( + "--archiveLimitMb", type=int, dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB", help=("Sets the limit (in MB) for archived files to S3. A value of 0" " indicates there is no limit.")) - parser.add_option( - "--archiveLimitTests", type="int", dest="archive_limit_tests", - metavar="ARCHIVE_LIMIT_TESTS", + parser.add_argument( + "--archiveLimitTests", type=int, dest="archive_limit_tests", metavar="ARCHIVE_LIMIT_TESTS", help=("Sets the maximum number of tests to archive to S3. A value" " of 0 indicates there is no limit.")) - parser.add_option( + parser.add_argument( "--basePort", dest="base_port", metavar="PORT", help=("The starting port number to use for mongod and mongos processes" " spawned by resmoke.py or the tests themselves. Each fixture and Job" " allocates a contiguous range of ports.")) - parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL", - help="The root url of the buildlogger server.") + parser.add_argument("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL", + help="The root url of the buildlogger server.") - parser.add_option("--continueOnFailure", action="store_true", dest="continue_on_failure", - help="Executes all tests in all suites, even if some of them fail.") + parser.add_argument("--continueOnFailure", action="store_true", dest="continue_on_failure", + help="Executes all tests in all suites, even if some of them fail.") - parser.add_option( + parser.add_argument( "--dbpathPrefix", dest="dbpath_prefix", metavar="PATH", help=("The directory which will contain the dbpaths of any mongod's started" " by resmoke.py or the tests themselves.")) - parser.add_option("--dbtest", dest="dbtest_executable", metavar="PATH", - help="The path to the dbtest executable for resmoke to use.") + parser.add_argument("--dbtest", dest="dbtest_executable", metavar="PATH", + help="The path to the dbtest executable for resmoke to use.") - parser.add_option( + parser.add_argument( "--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", metavar="TAG1,TAG2", help=("Comma separated list of tags. Any jstest that contains any of the" " specified tags will be excluded from any suites that are run." " The tag '{}' is implicitly part of this list.".format(_config.EXCLUDED_TAG))) - parser.add_option("-f", "--findSuites", action="store_true", dest="find_suites", - help="Lists the names of the suites that will execute the specified tests.") + parser.add_argument("--genny", dest="genny_executable", metavar="PATH", + help="The path to the genny executable for resmoke to use.") - parser.add_option("--genny", dest="genny_executable", metavar="PATH", - help="The path to the genny executable for resmoke to use.") - - parser.add_option( - "--spawnUsing", type="choice", dest="spawn_using", choices=("python", "jasper"), + parser.add_argument( + "--spawnUsing", dest="spawn_using", choices=("python", "jasper"), help=("Allows you to spawn resmoke processes using python or Jasper." "Defaults to python. Options are 'python' or 'jasper'.")) - parser.add_option( + parser.add_argument( "--includeWithAnyTags", action="append", dest="include_with_any_tags", metavar="TAG1,TAG2", help=("Comma separated list of tags. For the jstest portion of the suite(s)," " only tests which have at least one of the specified tags will be" " run.")) # Used for testing resmoke. Do not set this. - parser.add_option("--internalParam", action="append", dest="internal_params", - help=optparse.SUPPRESS_HELP) + parser.add_argument("--internalParam", action="append", dest="internal_params", + help=argparse.SUPPRESS) - parser.add_option("-n", action="store_const", const="tests", dest="dry_run", - help="Outputs the tests that would be run.") + parser.add_argument("-n", action="store_const", const="tests", dest="dry_run", + help="Outputs the tests that would be run.") # TODO: add support for --dryRun=commands - parser.add_option( - "--dryRun", type="choice", action="store", dest="dry_run", choices=("off", "tests"), - metavar="MODE", help=("Instead of running the tests, outputs the tests that would be run" - " (if MODE=tests). Defaults to MODE=%default.")) + parser.add_argument( + "--dryRun", action="store", dest="dry_run", choices=("off", "tests"), metavar="MODE", + help=("Instead of running the tests, outputs the tests that would be run" + " (if MODE=tests). Defaults to MODE=%%default.")) - parser.add_option( - "-j", "--jobs", type="int", dest="jobs", metavar="JOBS", + parser.add_argument( + "-j", "--jobs", type=int, dest="jobs", metavar="JOBS", help=("The number of Job instances to use. Each instance will receive its" " own MongoDB deployment to dispatch tests to.")) - parser.add_option("-l", "--listSuites", action="store_true", dest="list_suites", - help="Lists the names of the suites available to execute.") + parser.add_argument( + "--log", dest="logger_file", metavar="LOGGER", + help=("A YAML file that specifies the logging configuration. If the file is" + " located in the resmokeconfig/suites/ directory, then the basename" + " without the .yml extension can be specified, e.g. 'console'.")) + parser.set_defaults(logger_file="console") - parser.add_option("--mongo", dest="mongo_executable", metavar="PATH", - help="The path to the mongo shell executable for resmoke.py to use.") + parser.add_argument("--mongo", dest="mongo_executable", metavar="PATH", + help="The path to the mongo shell executable for resmoke.py to use.") - parser.add_option("--mongod", dest="mongod_executable", metavar="PATH", - help="The path to the mongod executable for resmoke.py to use.") + parser.add_argument("--mongod", dest="mongod_executable", metavar="PATH", + help="The path to the mongod executable for resmoke.py to use.") - parser.add_option( + parser.add_argument( "--mongodSetParameters", dest="mongod_set_parameters", metavar="{key1: value1, key2: value2, ..., keyN: valueN}", help=("Passes one or more --setParameter options to all mongod processes" " started by resmoke.py. The argument is specified as bracketed YAML -" " i.e. JSON with support for single quoted and unquoted keys.")) - parser.add_option("--mongos", dest="mongos_executable", metavar="PATH", - help="The path to the mongos executable for resmoke.py to use.") + parser.add_argument("--mongos", dest="mongos_executable", metavar="PATH", + help="The path to the mongos executable for resmoke.py to use.") - parser.add_option( + parser.add_argument( "--mongosSetParameters", dest="mongos_set_parameters", metavar="{key1: value1, key2: value2, ..., keyN: valueN}", help=("Passes one or more --setParameter options to all mongos processes" " started by resmoke.py. The argument is specified as bracketed YAML -" " i.e. JSON with support for single quoted and unquoted keys.")) - parser.add_option("--nojournal", action="store_true", dest="no_journal", - help="Disables journaling for all mongod's.") + parser.add_argument("--nojournal", action="store_true", dest="no_journal", + help="Disables journaling for all mongod's.") - parser.add_option("--numClientsPerFixture", type="int", dest="num_clients_per_fixture", - help="Number of clients running tests per fixture.") + parser.add_argument("--numClientsPerFixture", type=int, dest="num_clients_per_fixture", + help="Number of clients running tests per fixture.") - parser.add_option("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT", - help="Writes a JSON file with performance test results.") + parser.add_argument("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT", + help="Writes a JSON file with performance test results.") - parser.add_option( + parser.add_argument( "--shellConnString", dest="shell_conn_string", metavar="CONN_STRING", help="Overrides the default fixture and connects with a mongodb:// connection" " string to an existing MongoDB cluster instead. This is useful for" " connecting to a MongoDB deployment started outside of resmoke.py including" " one running in a debugger.") - parser.add_option( + parser.add_argument( "--shellPort", dest="shell_port", metavar="PORT", help="Convenience form of --shellConnString for connecting to an" " existing MongoDB cluster with the URL mongodb://localhost:[PORT]." " This is useful for connecting to a server running in a debugger.") - parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N", - help="Repeats the given suite(s) N times, or until one fails.") + parser.add_argument("--repeat", "--repeatSuites", type=int, dest="repeat_suites", metavar="N", + help="Repeats the given suite(s) N times, or until one fails.") - parser.add_option( - "--repeatTests", type="int", dest="repeat_tests", metavar="N", + parser.add_argument( + "--repeatTests", type=int, dest="repeat_tests", metavar="N", help="Repeats the tests inside each suite N times. This applies to tests" " defined in the suite configuration as well as tests defined on the command" " line.") - parser.add_option( - "--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N", + parser.add_argument( + "--repeatTestsMax", type=int, dest="repeat_tests_max", metavar="N", help="Repeats the tests inside each suite no more than N time when" " --repeatTestsSecs is specified. This applies to tests defined in the suite" " configuration as well as tests defined on the command line.") - parser.add_option( - "--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N", + parser.add_argument( + "--repeatTestsMin", type=int, dest="repeat_tests_min", metavar="N", help="Repeats the tests inside each suite at least N times when" " --repeatTestsSecs is specified. This applies to tests defined in the suite" " configuration as well as tests defined on the command line.") - parser.add_option( - "--repeatTestsSecs", type="float", dest="repeat_tests_secs", metavar="SECONDS", + parser.add_argument( + "--repeatTestsSecs", type=float, dest="repeat_tests_secs", metavar="SECONDS", help="Repeats the tests inside each suite this amount of time. Note that" " this option is mutually exclusive with --repeatTests. This applies to" " tests defined in the suite configuration as well as tests defined on the" " command line.") - parser.add_option( - "--reportFailureStatus", type="choice", action="store", dest="report_failure_status", + parser.add_argument( + "--reportFailureStatus", action="store", dest="report_failure_status", choices=("fail", "silentfail"), metavar="STATUS", help="Controls if the test failure status should be reported as failed" " or be silently ignored (STATUS=silentfail). Dynamic test failures will" - " never be silently ignored. Defaults to STATUS=%default.") + " never be silently ignored. Defaults to STATUS=%%default.") - parser.add_option("--reportFile", dest="report_file", metavar="REPORT", - help="Writes a JSON file with test status and timing information.") + parser.add_argument("--reportFile", dest="report_file", metavar="REPORT", + help="Writes a JSON file with test status and timing information.") - parser.add_option( - "--seed", type="int", dest="seed", metavar="SEED", + parser.add_argument( + "--seed", type=int, dest="seed", metavar="SEED", help=("Seed for the random number generator. Useful in combination with the" " --shuffle option for producing a consistent test execution order.")) - parser.add_option("--serviceExecutor", dest="service_executor", metavar="EXECUTOR", - help="The service executor used by jstests") + parser.add_argument("--serviceExecutor", dest="service_executor", metavar="EXECUTOR", + help="The service executor used by jstests") - parser.add_option("--transportLayer", dest="transport_layer", metavar="TRANSPORT", - help="The transport layer used by jstests") + parser.add_argument("--transportLayer", dest="transport_layer", metavar="TRANSPORT", + help="The transport layer used by jstests") - parser.add_option("--shellReadMode", type="choice", action="store", dest="shell_read_mode", - choices=("commands", "compatibility", "legacy"), metavar="READ_MODE", - help="The read mode used by the mongo shell.") + parser.add_argument("--shellReadMode", action="store", dest="shell_read_mode", + choices=("commands", "compatibility", "legacy"), metavar="READ_MODE", + help="The read mode used by the mongo shell.") - parser.add_option("--shellWriteMode", type="choice", action="store", dest="shell_write_mode", - choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE", - help="The write mode used by the mongo shell.") + parser.add_argument("--shellWriteMode", action="store", dest="shell_write_mode", + choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE", + help="The write mode used by the mongo shell.") - parser.add_option( + parser.add_argument( "--shuffle", action="store_const", const="on", dest="shuffle", help=("Randomizes the order in which tests are executed. This is equivalent" " to specifying --shuffleMode=on.")) - parser.add_option( - "--shuffleMode", type="choice", action="store", dest="shuffle", - choices=("on", "off", "auto"), metavar="ON|OFF|AUTO", + parser.add_argument( + "--shuffleMode", action="store", dest="shuffle", choices=("on", "off", + "auto"), metavar="ON|OFF|AUTO", help=("Controls whether to randomize the order in which tests are executed." " Defaults to auto when not supplied. auto enables randomization in" " all cases except when the number of jobs requested is 1.")) - parser.add_option( - "--staggerJobs", type="choice", action="store", dest="stagger_jobs", choices=("on", "off"), + parser.add_argument( + "--staggerJobs", action="store", dest="stagger_jobs", choices=("on", "off"), metavar="ON|OFF", help=("Enables or disables the stagger of launching resmoke jobs." - " Defaults to %default.")) + " Defaults to %%default.")) - parser.add_option( - "--majorityReadConcern", type="choice", action="store", dest="majority_read_concern", - choices=("on", - "off"), metavar="ON|OFF", help=("Enable or disable majority read concern support." - " Defaults to %default.")) + parser.add_argument( + "--majorityReadConcern", action="store", dest="majority_read_concern", choices=("on", + "off"), + metavar="ON|OFF", help=("Enable or disable majority read concern support." + " Defaults to %%default.")) - parser.add_option("--flowControl", type="choice", action="store", dest="flow_control", - choices=("on", - "off"), metavar="ON|OFF", help=("Enable or disable flow control.")) + parser.add_argument("--flowControl", action="store", dest="flow_control", choices=("on", "off"), + metavar="ON|OFF", help=("Enable or disable flow control.")) - parser.add_option("--flowControlTicketOverride", type="int", action="store", - dest="flow_control_tickets", metavar="TICKET_OVERRIDE", - help=("Number of tickets available for flow control.")) + parser.add_argument("--flowControlTicketOverride", type=int, action="store", + dest="flow_control_tickets", metavar="TICKET_OVERRIDE", + help=("Number of tickets available for flow control.")) - parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE", - help="The storage engine used by dbtests and jstests.") + parser.add_argument("--storageEngine", dest="storage_engine", metavar="ENGINE", + help="The storage engine used by dbtests and jstests.") - parser.add_option( + parser.add_argument( "--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", metavar="CONFIG", help="Sets the storage engine cache size configuration" " setting for all mongod's.") - parser.add_option( - "--numReplSetNodes", type="int", dest="num_replset_nodes", metavar="N", + parser.add_argument( + "--numReplSetNodes", type=int, dest="num_replset_nodes", metavar="N", help="The number of nodes to initialize per ReplicaSetFixture. This is also " "used to indicate the number of replica set members per shard in a " "ShardedClusterFixture.") - parser.add_option("--numShards", type="int", dest="num_shards", metavar="N", - help="The number of shards to use in a ShardedClusterFixture.") + parser.add_argument("--numShards", type=int, dest="num_shards", metavar="N", + help="The number of shards to use in a ShardedClusterFixture.") - parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS", - help="A YAML file that associates tests and tags.") + parser.add_argument("--tagFile", dest="tag_file", metavar="OPTIONS", + help="A YAML file that associates tests and tags.") - parser.add_option("--wiredTigerCollectionConfigString", dest="wt_coll_config", metavar="CONFIG", - help="Sets the WiredTiger collection configuration setting for all mongod's.") + parser.add_argument( + "--wiredTigerCollectionConfigString", dest="wt_coll_config", metavar="CONFIG", + help="Sets the WiredTiger collection configuration setting for all mongod's.") - parser.add_option("--wiredTigerEngineConfigString", dest="wt_engine_config", metavar="CONFIG", - help="Sets the WiredTiger engine configuration setting for all mongod's.") + parser.add_argument("--wiredTigerEngineConfigString", dest="wt_engine_config", metavar="CONFIG", + help="Sets the WiredTiger engine configuration setting for all mongod's.") - parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", - help="Sets the WiredTiger index configuration setting for all mongod's.") + parser.add_argument("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", + help="Sets the WiredTiger index configuration setting for all mongod's.") - parser.add_option( + parser.add_argument( "--executor", dest="executor_file", help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test" " to run a particular test under a particular suite configuration.") - parser.add_option( - "--mixedBinVersions", type="string", dest="mixed_bin_versions", + parser.add_argument( + "--mixedBinVersions", type=str, dest="mixed_bin_versions", metavar="version1-version2-..-versionN", help="Runs the test with the provided replica set" " binary version configuration. Specify 'old-new' to configure a replica set with a" " 'last-stable' version primary and 'latest' version secondary. For a sharded cluster" " with two shards and two replica set nodes each, specify 'old-new-old-new'.") - parser.add_option( - "--linearChain", type="choice", action="store", dest="linear_chain", choices=("on", "off"), + parser.add_argument( + "--linearChain", action="store", dest="linear_chain", choices=("on", "off"), metavar="ON|OFF", help="Enable or disable linear chaining for tests using " "ReplicaSetFixture.") - evergreen_options = optparse.OptionGroup( - parser, title=_EVERGREEN_OPTIONS_TITLE, + evergreen_options = parser.add_argument_group( + title=_EVERGREEN_ARGUMENT_TITLE, description=("Options used to propagate information about the Evergreen task running this" " script.")) - parser.add_option_group(evergreen_options) - evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID", - help="Sets the build ID of the task.") + evergreen_options.add_argument("--buildId", dest="build_id", metavar="BUILD_ID", + help="Sets the build ID of the task.") - evergreen_options.add_option( + evergreen_options.add_argument( "--distroId", dest="distro_id", metavar="DISTRO_ID", help=("Sets the identifier for the Evergreen distro running the" " tests.")) - evergreen_options.add_option( - "--executionNumber", type="int", dest="execution_number", metavar="EXECUTION_NUMBER", + evergreen_options.add_argument( + "--executionNumber", type=int, dest="execution_number", metavar="EXECUTION_NUMBER", help=("Sets the number for the Evergreen execution running the" " tests.")) - evergreen_options.add_option( + evergreen_options.add_argument( "--gitRevision", dest="git_revision", metavar="GIT_REVISION", help=("Sets the git revision for the Evergreen task running the" " tests.")) # We intentionally avoid adding a new command line option that starts with --suite so it doesn't # become ambiguous with the --suites option and break how engineers run resmoke.py locally. - evergreen_options.add_option( + evergreen_options.add_argument( "--originSuite", dest="origin_suite", metavar="SUITE", help=("Indicates the name of the test suite prior to the" " evergreen_generate_resmoke_tasks.py script splitting it" " up.")) - evergreen_options.add_option( + evergreen_options.add_argument( "--patchBuild", action="store_true", dest="patch_build", help=("Indicates that the Evergreen task running the tests is a" " patch build.")) - evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME", - help=("Sets the name of the Evergreen project running the tests.")) + evergreen_options.add_argument( + "--projectName", dest="project_name", metavar="PROJECT_NAME", + help=("Sets the name of the Evergreen project running the tests.")) - evergreen_options.add_option("--revisionOrderId", dest="revision_order_id", - metavar="REVISION_ORDER_ID", - help="Sets the chronological order number of this commit.") + evergreen_options.add_argument("--revisionOrderId", dest="revision_order_id", + metavar="REVISION_ORDER_ID", + help="Sets the chronological order number of this commit.") - evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME", - help="Sets the name of the Evergreen task running the tests.") + evergreen_options.add_argument("--taskName", dest="task_name", metavar="TASK_NAME", + help="Sets the name of the Evergreen task running the tests.") - evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID", - help="Sets the Id of the Evergreen task running the tests.") + evergreen_options.add_argument("--taskId", dest="task_id", metavar="TASK_ID", + help="Sets the Id of the Evergreen task running the tests.") - evergreen_options.add_option( + evergreen_options.add_argument( "--variantName", dest="variant_name", metavar="VARIANT_NAME", help=("Sets the name of the Evergreen build variant running the" " tests.")) - evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID", - help="Sets the version ID of the task.") + evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID", + help="Sets the version ID of the task.") - benchmark_options = optparse.OptionGroup( - parser, title="Benchmark/Benchrun test options", + benchmark_options = parser.add_argument_group( + title="Benchmark/Benchrun test options", description="Options for running Benchmark/Benchrun tests") - parser.add_option_group(benchmark_options) + benchmark_options.add_argument("--benchmarkFilter", type=str, dest="benchmark_filter", + metavar="BENCHMARK_FILTER", + help="Regex to filter Google benchmark tests to run.") - benchmark_options.add_option("--benchmarkFilter", type="string", dest="benchmark_filter", - metavar="BENCHMARK_FILTER", - help="Regex to filter Google benchmark tests to run.") - - benchmark_options.add_option( - "--benchmarkListTests", dest="benchmark_list_tests", action="store_true", - metavar="BENCHMARK_LIST_TESTS", + benchmark_options.add_argument( + "--benchmarkListTests", + dest="benchmark_list_tests", + action="store_true", + # metavar="BENCHMARK_LIST_TESTS", help=("Lists all Google benchmark test configurations in each" " test file.")) benchmark_min_time_help = ( "Minimum time to run each benchmark/benchrun test for. Use this option instead of " "--benchmarkRepetitions to make a test run for a longer or shorter duration.") - benchmark_options.add_option("--benchmarkMinTimeSecs", type="int", - dest="benchmark_min_time_secs", metavar="BENCHMARK_MIN_TIME", - help=benchmark_min_time_help) + benchmark_options.add_argument("--benchmarkMinTimeSecs", type=int, + dest="benchmark_min_time_secs", metavar="BENCHMARK_MIN_TIME", + help=benchmark_min_time_help) benchmark_repetitions_help = ( "Set --benchmarkRepetitions=1 if you'd like to run the benchmark/benchrun tests only once." " By default, each test is run multiple times to provide statistics on the variance" " between runs; use --benchmarkMinTimeSecs if you'd like to run a test for a longer or" " shorter duration.") - benchmark_options.add_option("--benchmarkRepetitions", type="int", dest="benchmark_repetitions", - metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help) + benchmark_options.add_argument("--benchmarkRepetitions", type=int, dest="benchmark_repetitions", + metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help) - parser.set_defaults(dry_run="off", find_suites=False, list_suites=False, logger_file="console", - shuffle="auto", stagger_jobs="off", suite_files="with_server", - majority_read_concern="on") - return parser +def _add_list_suites(subparsers): + """Create and add the parser for the list-suites subcommand.""" + parser = subparsers.add_parser("list-suites", + help="Lists the names of the suites available to execute.") + + parser.add_argument( + "--log", dest="logger_file", metavar="LOGGER", + help=("A YAML file that specifies the logging configuration. If the file is" + " located in the resmokeconfig/suites/ directory, then the basename" + " without the .yml extension can be specified, e.g. 'console'.")) + parser.set_defaults(logger_file="console") -def to_local_args(args=None): # pylint: disable=too-many-branches,too-many-locals - """ - Return a command line invocation for resmoke.py suitable for being run outside of Evergreen. +def _add_find_suites(subparsers): + """Create and add the parser for the find-suites subcommand.""" + parser = subparsers.add_parser( + "find-suites", help="Lists the names of the suites that will execute the specified tests.") - This function parses the 'args' list of command line arguments, removes any Evergreen-centric - options, and returns a new list of command line arguments. - """ + parser.add_argument("test_files", metavar="TEST_FILES", nargs="*", + help="Explicit test files to run") - if args is None: - args = sys.argv[1:] + parser.add_argument( + "--log", dest="logger_file", metavar="LOGGER", + help=("A YAML file that specifies the logging configuration. If the file is" + " located in the resmokeconfig/suites/ directory, then the basename" + " without the .yml extension can be specified, e.g. 'console'.")) + parser.set_defaults(logger_file="console") + parser.add_argument( + "--suites", dest="suite_files", metavar="SUITE1,SUITE2", required=True, + help=("Comma separated list of YAML files that each specify the configuration" + " of a suite. If the file is located in the resmokeconfig/suites/" + " directory, then the basename without the .yml extension can be" + " specified, e.g. 'core'. If a list of files is passed in as" + " positional arguments, they will be run using the suites'" + " configurations.")) + + +# def to_local_args(args=None): # pylint: disable=too-many-branches,too-many-locals +# """ +# Return a command line invocation for resmoke.py suitable for being run outside of Evergreen. +# This function parses the 'args' list of command line arguments, removes any Evergreen-centric +# options, and returns a new list of command line arguments. +# """ + +# if args is None: +# args = sys.argv[1:] + +# parser = _make_parser() + +# # We call optparse.OptionParser.parse_args() with a new instance of optparse.Values to avoid +# # having the default values filled in. This makes it so 'options' only contains command line +# # options that were explicitly specified. +# options, extra_args = parser.parse_args(args=args, values=optparse.Values()) + +# # If --originSuite was specified, then we replace the value of --suites with it. This is done to +# # avoid needing to have engineers learn about the test suites generated by the +# # evergreen_generate_resmoke_tasks.py script. +# origin_suite = getattr(options, "origin_suite", None) +# if origin_suite is not None: +# setattr(options, "suite_files", origin_suite) + +# # optparse.OptionParser doesn't offer a public and/or documented method for getting all of the +# # options. Given that the optparse module is deprecated, it is unlikely for the +# # _get_all_options() method to ever be removed or renamed. +# all_options = parser._get_all_options() # pylint: disable=protected-access + +# options_by_dest = {} +# for option in all_options: +# options_by_dest[option.dest] = option + +# suites_arg = None +# storage_engine_arg = None +# other_local_args = [] + +# options_to_ignore = { +# "--archiveLimitMb", +# "--archiveLimitTests", +# "--buildloggerUrl", +# "--log", +# "--perfReportFile", +# "--reportFailureStatus", +# "--reportFile", +# "--staggerJobs", +# "--tagFile", +# } + +# def format_option(option_name, option_value): +# """ +# Return <option_name>=<option_value>. +# This function assumes that 'option_name' is always "--" prefix and isn't "-" prefixed. +# """ +# return "%s=%s" % (option_name, option_value) + +# for option_dest in sorted(vars(options)): +# option_value = getattr(options, option_dest) +# option = options_by_dest[option_dest] +# option_name = option.get_opt_string() + +# if option_name in options_to_ignore: +# continue + +# option_group = parser.get_option_group(option_name) +# if option_group is not None and option_group.title == _EVERGREEN_OPTIONS_TITLE: +# continue + +# if option.takes_value(): +# if option.action == "append": +# args = [format_option(option_name, elem) for elem in option_value] +# other_local_args.extend(args) +# else: +# arg = format_option(option_name, option_value) + +# # We track the value for the --suites and --storageEngine command line options +# # separately in order to more easily sort them to the front. +# if option_dest == "suite_files": +# suites_arg = arg +# elif option_dest == "storage_engine": +# storage_engine_arg = arg +# else: +# other_local_args.append(arg) +# else: +# other_local_args.append(option_name) + +# return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None +# ] + other_local_args + extra_args + + +def _parse(sys_args): + """Parse the CLI args.""" + + # Split out this function for easier testing. parser = _make_parser() + parsed_args = parser.parse_args(sys_args) + + return (parser, parsed_args) - # We call optparse.OptionParser.parse_args() with a new instance of optparse.Values to avoid - # having the default values filled in. This makes it so 'options' only contains command line - # options that were explicitly specified. - options, extra_args = parser.parse_args(args=args, values=optparse.Values()) - - # If --originSuite was specified, then we replace the value of --suites with it. This is done to - # avoid needing to have engineers learn about the test suites generated by the - # evergreen_generate_resmoke_tasks.py script. - origin_suite = getattr(options, "origin_suite", None) - if origin_suite is not None: - setattr(options, "suite_files", origin_suite) - - # optparse.OptionParser doesn't offer a public and/or documented method for getting all of the - # options. Given that the optparse module is deprecated, it is unlikely for the - # _get_all_options() method to ever be removed or renamed. - all_options = parser._get_all_options() # pylint: disable=protected-access - - options_by_dest = {} - for option in all_options: - options_by_dest[option.dest] = option - - suites_arg = None - storage_engine_arg = None - other_local_args = [] - - options_to_ignore = { - "--archiveLimitMb", - "--archiveLimitTests", - "--buildloggerUrl", - "--log", - "--perfReportFile", - "--reportFailureStatus", - "--reportFile", - "--staggerJobs", - "--tagFile", - } - - def format_option(option_name, option_value): - """ - Return <option_name>=<option_value>. - - This function assumes that 'option_name' is always "--" prefix and isn't "-" prefixed. - """ - return "%s=%s" % (option_name, option_value) - - for option_dest in sorted(vars(options)): - option_value = getattr(options, option_dest) - option = options_by_dest[option_dest] - option_name = option.get_opt_string() - - if option_name in options_to_ignore: - continue - - option_group = parser.get_option_group(option_name) - if option_group is not None and option_group.title == _EVERGREEN_OPTIONS_TITLE: - continue - - if option.takes_value(): - if option.action == "append": - args = [format_option(option_name, elem) for elem in option_value] - other_local_args.extend(args) + +def parse_command_line(sys_args, **kwargs): + """Parse the command line arguments passed to resmoke.py and return the subcommand object to execute.""" + parser, parsed_args = _parse(sys_args) + + def create_subcommand(parser, parsed_args, **kwargs): + """Create a subcommand object based on args passed into resmoke.py.""" + + subcommand = parsed_args.command + subcommand_obj = None + if subcommand in ('find-suites', 'list-suites', 'run'): + configure_resmoke.validate_and_update_config(parser, parsed_args) + if _config.EVERGREEN_TASK_ID is not None: + subcommand_obj = commands.run.TestRunnerEvg(subcommand, **kwargs) else: - arg = format_option(option_name, option_value) + subcommand_obj = commands.run.TestRunner(subcommand, **kwargs) - # We track the value for the --suites and --storageEngine command line options - # separately in order to more easily sort them to the front. - if option_dest == "suite_files": - suites_arg = arg - elif option_dest == "storage_engine": - storage_engine_arg = arg - else: - other_local_args.append(arg) - else: - other_local_args.append(option_name) + if subcommand_obj is None: + raise RuntimeError( + f"Resmoke configuration has invalid subcommand: {subcommand}. Try '--help'") - return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None - ] + other_local_args + extra_args + return subcommand_obj + return create_subcommand(parser, parsed_args, **kwargs) -def parse_command_line(): - """Parse the command line arguments passed to resmoke.py.""" - parser = _make_parser() - options, args = parser.parse_args() - _validate_options(parser, options, args) - _update_config_vars(options) - _validate_config(parser) - - return ResmokeConfig(list_suites=options.list_suites, find_suites=options.find_suites, - dry_run=options.dry_run, suite_files=options.suite_files.split(","), - test_files=args, logging_config=_get_logging_config(options.logger_file)) - - -def _validate_options(parser, options, args): - """Do preliminary validation on the options and error on any invalid options.""" - - if options.shell_port is not None and options.shell_conn_string is not None: - parser.error("Cannot specify both `shellPort` and `shellConnString`") - - if options.executor_file: - parser.error("--executor is superseded by --suites; specify --suites={} {} to run the" - " test(s) under those suite configuration(s)".format( - options.executor_file, " ".join(args))) - - -def _validate_config(parser): - """Do validation on the config settings.""" - - if _config.REPEAT_TESTS_MAX: - if not _config.REPEAT_TESTS_SECS: - parser.error("Must specify --repeatTestsSecs with --repeatTestsMax") - - if _config.REPEAT_TESTS_MIN > _config.REPEAT_TESTS_MAX: - parser.error("--repeatTestsSecsMin > --repeatTestsMax") - - if _config.REPEAT_TESTS_MIN and not _config.REPEAT_TESTS_SECS: - parser.error("Must specify --repeatTestsSecs with --repeatTestsMin") - - if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS: - parser.error("Cannot specify --repeatTests and --repeatTestsSecs") - - if _config.MIXED_BIN_VERSIONS is not None: - for version in _config.MIXED_BIN_VERSIONS: - if version not in set(['old', 'new']): - parser.error("Must specify binary versions as 'old' or 'new' in format" - " 'version1-version2'") - - -def validate_benchmark_options(): - """Error out early if any options are incompatible with benchmark test suites. - - :return: None - """ - - if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1 or _config.REPEAT_TESTS_SECS: - raise optparse.OptionValueError( - "--repeatSuites/--repeatTests cannot be used with benchmark tests. " - "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark " - "configuration.") - - if _config.JOBS > 1: - raise optparse.OptionValueError( - "--jobs=%d cannot be used for benchmark tests. Parallel jobs affect CPU cache access " - "patterns and cause additional context switching, which lead to inaccurate benchmark " - "results. Please use --jobs=1" % _config.JOBS) - - -def _update_config_vars(values): # pylint: disable=too-many-statements,too-many-locals,too-many-branches - """Update the variables of the config module.""" - - config = _config.DEFAULTS.copy() - - # Use RSK_ prefixed environment variables to indicate resmoke-specific values. - # The list of configuration is detailed in config.py - resmoke_env_prefix = 'RSK_' - for key in os.environ.keys(): - if key.startswith(resmoke_env_prefix): - # Windows env vars are case-insensitive, we use lowercase to be consistent - # with existing resmoke options. - config[key[len(resmoke_env_prefix):].lower()] = os.environ[key] - - # Override `config` with values from command line arguments. - cmdline_vars = vars(values) - for cmdline_key in cmdline_vars: - if cmdline_key not in _config.DEFAULTS: - # Ignore options that don't map to values in config.py - continue - if cmdline_vars[cmdline_key] is not None: - config[cmdline_key] = cmdline_vars[cmdline_key] - - if os.path.isfile("resmoke.ini"): - config_parser = configparser.ConfigParser() - config_parser.read("resmoke.ini") - if "resmoke" in config_parser.sections(): - user_config = dict(config_parser["resmoke"]) - config.update(user_config) - - _config.ALWAYS_USE_LOG_FILES = config.pop("always_use_log_files") - _config.IS_ASAN_BUILD = config.pop("is_asan_build") - _config.BASE_PORT = int(config.pop("base_port")) - _config.BUILDLOGGER_URL = config.pop("buildlogger_url") - _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix")) - _config.DRY_RUN = config.pop("dry_run") - # EXCLUDE_WITH_ANY_TAGS will always contain the implicitly defined EXCLUDED_TAG. - _config.EXCLUDE_WITH_ANY_TAGS = [_config.EXCLUDED_TAG] - _config.EXCLUDE_WITH_ANY_TAGS.extend( - utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), [])) - _config.FAIL_FAST = not config.pop("continue_on_failure") - _config.FLOW_CONTROL = config.pop("flow_control") - _config.FLOW_CONTROL_TICKETS = config.pop("flow_control_tickets") - _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags")) - _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable")) - _config.JOBS = config.pop("jobs") - _config.LINEAR_CHAIN = config.pop("linear_chain") == "on" - _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on" - _config.MIXED_BIN_VERSIONS = config.pop("mixed_bin_versions") - if _config.MIXED_BIN_VERSIONS is not None: - _config.MIXED_BIN_VERSIONS = _config.MIXED_BIN_VERSIONS.split("-") - - _config.INSTALL_DIR = config.pop("install_dir") - if _config.INSTALL_DIR is not None: - # Normalize the path so that on Windows dist-test/bin - # translates to .\dist-test\bin then absolutify it since the - # Windows PATH variable requires absolute paths. - _config.INSTALL_DIR = os.path.abspath(_expand_user(os.path.normpath(_config.INSTALL_DIR))) - - for binary in ["mongo", "mongod", "mongos", "dbtest"]: - keyname = binary + "_executable" - if config.get(keyname, None) is None: - config[keyname] = os.path.join(_config.INSTALL_DIR, binary) - - _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable")) - _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable")) - _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable")) - _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters") - _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable")) - - _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters") - _config.NO_JOURNAL = config.pop("no_journal") - _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture") - _config.NUM_REPLSET_NODES = config.pop("num_replset_nodes") - _config.NUM_SHARDS = config.pop("num_shards") - _config.PERF_REPORT_FILE = config.pop("perf_report_file") - _config.RANDOM_SEED = config.pop("seed") - _config.REPEAT_SUITES = config.pop("repeat_suites") - _config.REPEAT_TESTS = config.pop("repeat_tests") - _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max") - _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min") - _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs") - _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") - _config.REPORT_FILE = config.pop("report_file") - _config.SERVICE_EXECUTOR = config.pop("service_executor") - _config.SHELL_READ_MODE = config.pop("shell_read_mode") - _config.SHELL_WRITE_MODE = config.pop("shell_write_mode") - _config.SPAWN_USING = config.pop("spawn_using") - _config.STAGGER_JOBS = config.pop("stagger_jobs") == "on" - _config.STORAGE_ENGINE = config.pop("storage_engine") - _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storage_engine_cache_size_gb") - _config.TAG_FILE = config.pop("tag_file") - _config.TRANSPORT_LAYER = config.pop("transport_layer") - - # Internal testing options. - _config.INTERNAL_PARAMS = config.pop("internal_params") - - # Evergreen options. - _config.EVERGREEN_BUILD_ID = config.pop("build_id") - _config.EVERGREEN_DISTRO_ID = config.pop("distro_id") - _config.EVERGREEN_EXECUTION = config.pop("execution_number") - _config.EVERGREEN_PATCH_BUILD = config.pop("patch_build") - _config.EVERGREEN_PROJECT_NAME = config.pop("project_name") - _config.EVERGREEN_REVISION = config.pop("git_revision") - _config.EVERGREEN_REVISION_ORDER_ID = config.pop("revision_order_id") - _config.EVERGREEN_TASK_ID = config.pop("task_id") - _config.EVERGREEN_TASK_NAME = config.pop("task_name") - _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name") - _config.EVERGREEN_VERSION_ID = config.pop("version_id") - - # Archival options. Archival is enabled only when running on evergreen. - if not _config.EVERGREEN_TASK_ID: - _config.ARCHIVE_FILE = None - _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb") - _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests") - - # Wiredtiger options. - _config.WT_COLL_CONFIG = config.pop("wt_coll_config") - _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config") - _config.WT_INDEX_CONFIG = config.pop("wt_index_config") - - # Benchmark/Benchrun options. - _config.BENCHMARK_FILTER = config.pop("benchmark_filter") - _config.BENCHMARK_LIST_TESTS = config.pop("benchmark_list_tests") - benchmark_min_time = config.pop("benchmark_min_time_secs") - if benchmark_min_time is not None: - _config.BENCHMARK_MIN_TIME = datetime.timedelta(seconds=benchmark_min_time) - _config.BENCHMARK_REPETITIONS = config.pop("benchmark_repetitions") - - # Config Dir options. - _config.CONFIG_DIR = config.pop("config_dir") - - # Populate the named suites by scanning config_dir/suites - named_suites = {} - - suites_dir = os.path.join(_config.CONFIG_DIR, "suites") - root = os.path.abspath(suites_dir) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml"): - pathname = os.path.join(root, filename) - named_suites[short_name] = pathname - - _config.NAMED_SUITES = named_suites - - _config.LOGGER_DIR = os.path.join(_config.CONFIG_DIR, "loggers") - - shuffle = config.pop("shuffle") - if shuffle == "auto": - # If the user specified a value for --jobs > 1 (or -j > 1), then default to randomize - # the order in which tests are executed. This is because with multiple threads the tests - # wouldn't run in a deterministic order anyway. - _config.SHUFFLE = _config.JOBS > 1 - else: - _config.SHUFFLE = shuffle == "on" - - conn_string = config.pop("shell_conn_string") - port = config.pop("shell_port") - - if port is not None: - conn_string = "mongodb://localhost:" + port - - if conn_string is not None: - # The --shellConnString command line option must be a MongoDB connection URI, which means it - # must specify the mongodb:// or mongodb+srv:// URI scheme. pymongo.uri_parser.parse_uri() - # raises an exception if the connection string specified isn't considered a valid MongoDB - # connection URI. - pymongo.uri_parser.parse_uri(conn_string) - _config.SHELL_CONN_STRING = conn_string - - if config: - raise optparse.OptionValueError("Unknown option(s): %s" % (list(config.keys()))) - - -def _get_logging_config(pathname): - """Read YAML configuration from 'pathname' how to log tests and fixtures.""" - try: - # If the user provides a full valid path to a logging config - # we don't need to search LOGGER_DIR for the file. - if os.path.exists(pathname): - return utils.load_yaml_file(pathname).pop("logging") - - root = os.path.abspath(_config.LOGGER_DIR) - files = os.listdir(root) - for filename in files: - (short_name, ext) = os.path.splitext(filename) - if ext in (".yml", ".yaml") and short_name == pathname: - config_file = os.path.join(root, filename) - if not os.path.isfile(config_file): - raise optparse.OptionValueError( - "Expected a logger YAML config, but got '%s'" % pathname) - return utils.load_yaml_file(config_file).pop("logging") - - raise optparse.OptionValueError("Unknown logger '%s'" % pathname) - except FileNotFoundError: - raise IOError("Directory {} does not exist.".format(_config.LOGGER_DIR)) - - -def _expand_user(pathname): - """Provide wrapper around os.path.expanduser() to do nothing when given None.""" - if pathname is None: - return None - return os.path.expanduser(pathname) - - -def _tags_from_list(tags_list): - """Return the list of tags from a list of tag parameter values. - - Each parameter value in the list may be a list of comma separated tags, with empty strings - ignored. - """ - tags = [] - if tags_list is not None: - for tag in tags_list: - tags.extend([t for t in tag.split(",") if t != ""]) - return tags - return None - - -def set_options(argstr=''): - """Populate the config module variables with the default options.""" - parser = _make_parser() - options, _args = parser.parse_args(args=shlex.split(argstr)) - _update_config_vars(options) +def set_run_options(argstr=''): + """Populate the config module variables for the 'run' subcommand with the default options.""" + parser, parsed_args = _parse(['run'] + shlex.split(argstr)) + configure_resmoke.validate_and_update_config(parser, parsed_args) diff --git a/buildscripts/resmokelib/testing/testcases/benchmark_test.py b/buildscripts/resmokelib/testing/testcases/benchmark_test.py index e7760799e42..9902bd503ef 100644 --- a/buildscripts/resmokelib/testing/testcases/benchmark_test.py +++ b/buildscripts/resmokelib/testing/testcases/benchmark_test.py @@ -2,7 +2,6 @@ from buildscripts.resmokelib import config as _config from buildscripts.resmokelib import core -from buildscripts.resmokelib import parser from buildscripts.resmokelib import utils from buildscripts.resmokelib.testing.testcases import interface @@ -16,12 +15,30 @@ class BenchmarkTestCase(interface.ProcessTestCase): """Initialize the BenchmarkTestCase with the executable to run.""" interface.ProcessTestCase.__init__(self, logger, "Benchmark test", program_executable) - parser.validate_benchmark_options() + self.validate_benchmark_options() self.bm_executable = program_executable self.suite_bm_options = program_options self.bm_options = {} + def validate_benchmark_options(self): # pylint: disable=no-self-use + """Error out early if any options are incompatible with benchmark test suites. + + :return: None + """ + + if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1 or _config.REPEAT_TESTS_SECS: + raise ValueError( + "--repeatSuites/--repeatTests cannot be used with benchmark tests. " + "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark " + "configuration.") + + if _config.JOBS > 1: + raise ValueError( + "--jobs=%d cannot be used for benchmark tests. Parallel jobs affect CPU cache access " + "patterns and cause additional context switching, which lead to inaccurate benchmark " + "results. Please use --jobs=1" % _config.JOBS) + def configure(self, fixture, *args, **kwargs): """Configure BenchmarkTestCase.""" interface.ProcessTestCase.configure(self, fixture, *args, **kwargs) diff --git a/buildscripts/selected_tests.py b/buildscripts/selected_tests.py index 2d3872e863c..a9aa7f0f88a 100644 --- a/buildscripts/selected_tests.py +++ b/buildscripts/selected_tests.py @@ -484,7 +484,7 @@ def main( selected_tests_service = SelectedTestsService.from_file(selected_tests_config) repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)] - buildscripts.resmokelib.parser.set_options() + buildscripts.resmokelib.parser.set_run_options() task_expansions = read_config.read_config_file(expansion_file) origin_build_variants = task_expansions["selected_tests_buildvariants"].split(" ") diff --git a/buildscripts/tests/resmokelib/end2end/test_resmoke.py b/buildscripts/tests/resmokelib/end2end/test_resmoke.py index 70cb20cef57..60701b7eb8b 100644 --- a/buildscripts/tests/resmokelib/end2end/test_resmoke.py +++ b/buildscripts/tests/resmokelib/end2end/test_resmoke.py @@ -27,7 +27,7 @@ class _ResmokeSelftest(unittest.TestCase): cls.logger.addHandler(handler) cls.test_dir = os.path.normpath("/data/db/selftest") - cls.resmoke_const_args = ["--dbpathPrefix={}".format(cls.test_dir)] + cls.resmoke_const_args = ["run", "--dbpathPrefix={}".format(cls.test_dir)] def setUp(self): self.logger.info("Cleaning temp directory %s", self.test_dir) @@ -114,7 +114,7 @@ class TestTimeout(_ResmokeSelftest): # TODO: replace above with below after SERVER-46691. # signal_resmoke_process = core.programs.make_process( # self.logger, - # [sys.executable, "buildscripts/signal_resmoke.py", "--pid", str(resmoke_process.pid)]) + # [sys.executable, "buildscripts/signal_resmoke.py", "run", "--pid", str(resmoke_process.pid)]) # signal_resmoke_process.start() # return_code = signal_resmoke_process.wait() diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py index 85b7d20ccf4..0a95b38409a 100644 --- a/buildscripts/tests/resmokelib/test_parser.py +++ b/buildscripts/tests/resmokelib/test_parser.py @@ -7,6 +7,7 @@ from buildscripts.resmokelib import parser as _parser # pylint: disable=missing-docstring +@unittest.skip("TODO: SERVER-47611") class TestLocalCommandLine(unittest.TestCase): """Unit tests for the to_local_args() function.""" @@ -244,3 +245,57 @@ class TestLocalCommandLine(unittest.TestCase): ]) self.assertEqual(cmdline, ["--suites=my_suite", "--storageEngine=my_storage_engine"]) + + +class TestParseArgs(unittest.TestCase): + """Unit tests for the parse() function.""" + + def test_files_at_end(self): + _, args = _parser._parse([ # pylint: disable=protected-access + "run", + "--suites=my_suite1,my_suite2", + "test_file1.js", + "test_file2.js", + "test_file3.js", + ]) + + self.assertEqual(args.test_files, [ + "test_file1.js", + "test_file2.js", + "test_file3.js", + ]) + # suites get split up when config.py gets populated + self.assertEqual(args.suite_files, "my_suite1,my_suite2") + + def test_files_in_the_middle(self): + _, args = _parser._parse([ # pylint: disable=protected-access + "run", + "--storageEngine=my_storage_engine", + "test_file1.js", + "test_file2.js", + "test_file3.js", + "--suites=my_suite1", + ]) + + self.assertEqual(args.test_files, [ + "test_file1.js", + "test_file2.js", + "test_file3.js", + ]) + self.assertEqual(args.suite_files, "my_suite1") + + +class TestParseCommandLine(unittest.TestCase): + """Unit tests for the parse_command_line() function.""" + + def test_find_suites(self): + subcommand_obj = _parser.parse_command_line(['find-suites', '--suites=my_suite']) + self.assertTrue(hasattr(subcommand_obj, 'execute')) + + def test_list_suites(self): + subcommand_obj = _parser.parse_command_line(['list-suites']) + self.assertTrue(hasattr(subcommand_obj, 'execute')) + + def test_run(self): + subcommand_obj = _parser.parse_command_line(['run', '--suite=my_suite', 'my_test.js']) + self.assertTrue(hasattr(subcommand_obj, 'execute')) diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py index e080ace8a4e..cf1a42b973f 100644 --- a/buildscripts/tests/resmokelib/test_selector.py +++ b/buildscripts/tests/resmokelib/test_selector.py @@ -1,5 +1,6 @@ """Unit tests for the buildscripts.resmokelib.selector module.""" +import sys import fnmatch import os.path import unittest @@ -514,7 +515,7 @@ class TestFilterTests(unittest.TestCase): self.assertEqual(["dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"], selected) def test_filter_temporarily_disabled_tests(self): - parser.parse_command_line() + parser.parse_command_line(sys.argv[1:]) test_file_explorer = MockTestFileExplorer() test_file_explorer.tags = { "dir/subdir1/test11.js": ["tag1", "tag2", "__TEMPORARILY_DISABLED__"], diff --git a/buildscripts/tests/resmokelib/test_suitesconfig.py b/buildscripts/tests/resmokelib/test_suitesconfig.py index f90fd02addb..e7ce3f2d776 100644 --- a/buildscripts/tests/resmokelib/test_suitesconfig.py +++ b/buildscripts/tests/resmokelib/test_suitesconfig.py @@ -6,7 +6,7 @@ import mock from buildscripts.resmokelib import suitesconfig from buildscripts.resmokelib import parser -parser.set_options() +parser.set_run_options() # pylint: disable=missing-docstring diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py index bb524397909..ca02b51b559 100644 --- a/buildscripts/tests/test_burn_in_tests.py +++ b/buildscripts/tests/test_burn_in_tests.py @@ -22,7 +22,7 @@ from buildscripts.ciconfig.evergreen import parse_evergreen_file import buildscripts.util.teststats as teststats_utils import buildscripts.resmokelib.parser as _parser import buildscripts.resmokelib.config as _config -_parser.set_options() +_parser.set_run_options() # pylint: disable=missing-docstring,protected-access,too-many-lines,no-self-use @@ -77,7 +77,7 @@ def get_evergreen_config(config_file_path): class TestAcceptance(unittest.TestCase): def tearDown(self): - _parser.set_options() + _parser.set_run_options() @patch(ns("write_file")) def test_no_tests_run_if_none_changed(self, write_json_mock): @@ -432,7 +432,7 @@ class TestSetResmokeCmd(unittest.TestCase): resmoke_cmds = under_test._set_resmoke_cmd(repeat_config, []) self.assertListEqual(resmoke_cmds, - [sys.executable, "buildscripts/resmoke.py", '--repeatSuites=2']) + [sys.executable, "buildscripts/resmoke.py", "run", '--repeatSuites=2']) def test__set_resmoke_cmd_no_opts(self): repeat_config = under_test.RepeatConfig() @@ -541,7 +541,7 @@ class RunTests(unittest.TestCase): @patch(ns('subprocess.check_call')) def test_run_tests_no_tests(self, check_call_mock): tests_by_task = {} - resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"] + resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"] under_test.run_tests(tests_by_task, resmoke_cmd) @@ -551,7 +551,7 @@ class RunTests(unittest.TestCase): def test_run_tests_some_test(self, check_call_mock): n_tasks = 3 tests_by_task = create_tests_by_task_mock(n_tasks, 5) - resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"] + resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"] under_test.run_tests(tests_by_task, resmoke_cmd) @@ -563,7 +563,7 @@ class RunTests(unittest.TestCase): error_code = 42 n_tasks = 3 tests_by_task = create_tests_by_task_mock(n_tasks, 5) - resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"] + resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"] check_call_mock.side_effect = subprocess.CalledProcessError(error_code, "err1") exit_mock.side_effect = ValueError('exiting') diff --git a/buildscripts/tests/test_burn_in_tests_multiversion.py b/buildscripts/tests/test_burn_in_tests_multiversion.py index 5c694b1fcb7..159545ddf7d 100644 --- a/buildscripts/tests/test_burn_in_tests_multiversion.py +++ b/buildscripts/tests/test_burn_in_tests_multiversion.py @@ -16,7 +16,7 @@ from buildscripts.burn_in_tests import _gather_task_info, create_generate_tasks_ from buildscripts.ciconfig.evergreen import parse_evergreen_file import buildscripts.resmokelib.parser as _parser import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion -_parser.set_options() +_parser.set_run_options() MONGO_4_2_HASH = "d94888c0d0a8065ca57d354ece33b3c2a1a5a6d6" diff --git a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py index a33e27fac5b..cc9f5e201ca 100644 --- a/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py +++ b/buildscripts/tests/test_evergreen_generate_resmoke_tasks.py @@ -995,7 +995,8 @@ class GenerateSubSuitesTest(unittest.TestCase): self.assertIn(tests_runtimes[2], filtered_list) self.assertIn(tests_runtimes[1], filtered_list) - def test_filter_blacklist_files(self): + @patch(ns('_parser.set_run_options')) + def test_filter_blacklist_files(self, set_run_options_mock): tests_runtimes = [ TestRuntime(test_name="dir1/file1.js", runtime=20.32), TestRuntime(test_name="dir2/file2.js", runtime=24.32), @@ -1019,7 +1020,8 @@ class GenerateSubSuitesTest(unittest.TestCase): self.assertIn(tests_runtimes[2], filtered_list) self.assertIn(tests_runtimes[0], filtered_list) - def test_filter_blacklist_files_for_windows(self): + @patch(ns('_parser.set_run_options')) + def test_filter_blacklist_files_for_windows(self, set_run_options_mock): tests_runtimes = [ TestRuntime(test_name="dir1/file1.js", runtime=20.32), TestRuntime(test_name="dir2/file2.js", runtime=24.32), diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 9992e8ddf6a..d4d369d9863 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1478,7 +1478,7 @@ functions: ${san_symbolizer} \ ${snmp_config_path} \ ${resmoke_wrapper} \ - $python buildscripts/evergreen_run_tests.py \ + $python buildscripts/resmoke.py run \ ${resmoke_args} \ $extra_args \ ${test_flags} \ diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js index 2e84feb101a..a35bdbb9567 100644 --- a/jstests/noPassthrough/libs/backup_restore.js +++ b/jstests/noPassthrough/libs/backup_restore.js @@ -122,7 +122,7 @@ var BackupRestoreTest = function(options) { function _fsmClient(host) { // Launch FSM client const suite = 'concurrency_replication_for_backup_restore'; - const resmokeCmd = 'python buildscripts/resmoke.py --shuffle --continueOnFailure' + + const resmokeCmd = 'python buildscripts/resmoke.py run --shuffle --continueOnFailure' + ' --repeat=99999 --mongo=' + MongoRunner.mongoShellPath + ' --shellConnString=mongodb://' + host + ' --suites=' + suite; diff --git a/pytests/powertest.py b/pytests/powertest.py index 18254c9a688..a976ae02df9 100755 --- a/pytests/powertest.py +++ b/pytests/powertest.py @@ -1708,6 +1708,7 @@ def resmoke_client( # pylint: disable=too-many-arguments log_output = ">> {} 2>&1".format(log_file) if log_file else "" cmds = ("cd {}; " "python buildscripts/resmoke.py" + "run" " --mongo {}" " --suites {}" " --shellConnString mongodb://{}" |