summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorvrachev <vlad.rachev@mongodb.com>2020-04-15 13:22:09 -0400
committerEvergreen Agent <no-reply@evergreen.mongodb.com>2020-05-01 13:47:49 +0000
commit6bb97ee97405a710dd616fc572e0fa33cb29fe48 (patch)
tree11ce7c59601afa0269969ef1e1d20cd87d7010eb
parent7fd3c03c548d0febfa1e871e16d638513c417c79 (diff)
downloadmongo-6bb97ee97405a710dd616fc572e0fa33cb29fe48.tar.gz
SERVER-46769 Migrate from optparse to argparse
-rw-r--r--buildscripts/burn_in_tests.py2
-rwxr-xr-xbuildscripts/evergreen_run_tests.py149
-rwxr-xr-xbuildscripts/resmoke.py382
-rw-r--r--buildscripts/resmokelib/commands/__init__.py4
-rw-r--r--buildscripts/resmokelib/commands/interface.py9
-rw-r--r--buildscripts/resmokelib/commands/run.py511
-rw-r--r--buildscripts/resmokelib/config.py15
-rw-r--r--buildscripts/resmokelib/configure_resmoke.py234
-rw-r--r--buildscripts/resmokelib/parser.py775
-rw-r--r--buildscripts/resmokelib/testing/testcases/benchmark_test.py21
-rw-r--r--buildscripts/tests/resmokelib/test_parser.py55
-rw-r--r--buildscripts/tests/resmokelib/test_selector.py3
-rw-r--r--buildscripts/tests/test_burn_in_tests.py8
-rw-r--r--etc/drivers_nightly.yml2
-rw-r--r--etc/evergreen.yml2
-rw-r--r--jstests/noPassthrough/libs/backup_restore.js2
-rwxr-xr-xpytests/powertest.py1
-rw-r--r--src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml4
18 files changed, 1180 insertions, 999 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 08555d2f179..c67c4ca8b89 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -406,7 +406,7 @@ def _write_json_file(json_data, pathname):
def _set_resmoke_cmd(repeat_config: RepeatConfig, resmoke_args: [str]) -> [str]:
"""Build the resmoke command, if a resmoke.py command wasn't passed in."""
- new_args = [sys.executable, "buildscripts/resmoke.py"]
+ new_args = [sys.executable, "buildscripts/resmoke.py", "run"]
if resmoke_args:
new_args = copy.deepcopy(resmoke_args)
diff --git a/buildscripts/evergreen_run_tests.py b/buildscripts/evergreen_run_tests.py
deleted file mode 100755
index 861970f63db..00000000000
--- a/buildscripts/evergreen_run_tests.py
+++ /dev/null
@@ -1,149 +0,0 @@
-#!/usr/bin/env python3
-"""Command line utility for executing MongoDB tests in Evergreen."""
-
-import collections
-import os.path
-import sys
-
-# Get relative imports to work when the package is not installed on the PYTHONPATH.
-if __name__ == "__main__" and __package__ is None:
- sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from buildscripts import resmoke # pylint: disable=wrong-import-position
-from buildscripts import resmokelib # pylint: disable=wrong-import-position
-
-_TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"])
-
-
-class Main(resmoke.Resmoke):
- """Execute Main class.
-
- A class for executing potentially multiple resmoke.py test suites in a way that handles
- additional options for running unreliable tests in Evergreen.
- """
-
- UNRELIABLE_TAG = _TagInfo(
- tag_name="unreliable",
- evergreen_aware=True,
- suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
- report_failure_status="silentfail"))
-
- RESOURCE_INTENSIVE_TAG = _TagInfo(
- tag_name="resource_intensive",
- evergreen_aware=False,
- suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
- num_jobs=1))
-
- RETRY_ON_FAILURE_TAG = _TagInfo(
- tag_name="retry_on_failure",
- evergreen_aware=True,
- suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
- fail_fast=False, num_repeat_suites=2, num_repeat_tests=1,
- report_failure_status="silentfail"))
-
- @staticmethod
- def _make_evergreen_aware_tags(tag_name):
- """Return a list of resmoke.py tags.
-
- This list is for task, variant, and distro combinations in Evergreen.
- """
-
- tags_format = ["{tag_name}"]
-
- if resmokelib.config.EVERGREEN_TASK_NAME is not None:
- tags_format.append("{tag_name}|{task_name}")
-
- if resmokelib.config.EVERGREEN_VARIANT_NAME is not None:
- tags_format.append("{tag_name}|{task_name}|{variant_name}")
-
- if resmokelib.config.EVERGREEN_DISTRO_ID is not None:
- tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}")
-
- return [
- tag.format(tag_name=tag_name, task_name=resmokelib.config.EVERGREEN_TASK_NAME,
- variant_name=resmokelib.config.EVERGREEN_VARIANT_NAME,
- distro_id=resmokelib.config.EVERGREEN_DISTRO_ID) for tag in tags_format
- ]
-
- @classmethod
- def _make_tag_combinations(cls):
- """Return a list of (tag, enabled) pairs.
-
- These pairs represent all possible combinations of all possible pairings
- of whether the tags are enabled or disabled together.
- """
-
- combinations = []
-
- if resmokelib.config.EVERGREEN_PATCH_BUILD:
- combinations.append(("unreliable and resource intensive",
- ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True))))
- combinations.append(("unreliable and not resource intensive",
- ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False))))
- combinations.append(("reliable and resource intensive",
- ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True))))
- combinations.append(("reliable and not resource intensive",
- ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
- else:
- combinations.append(("retry on failure and resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
- True))))
- combinations.append(("retry on failure and not resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
- combinations.append(("run once and resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- True))))
- combinations.append(("run once and not resource intensive",
- ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
- False))))
-
- return combinations
-
- def _get_suites(self):
- """Return a list of resmokelib.testing.suite.Suite instances to execute.
-
- For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(),
- multiple copies of that test suite are run using different resmokelib.config.SuiteOptions()
- depending on whether each tag in the combination is enabled or not.
- """
-
- suites = []
-
- for suite in resmoke.Resmoke._get_suites(self):
- if suite.test_kind != "js_test":
- # Tags are only support for JavaScript tests, so we leave the test suite alone when
- # running any other kind of test.
- suites.append(suite)
- continue
-
- for (tag_desc, tag_combo) in self._make_tag_combinations():
- suite_options_list = []
-
- for (tag_info, enabled) in tag_combo:
- if tag_info.evergreen_aware:
- tags = self._make_evergreen_aware_tags(tag_info.tag_name)
- include_tags = {"$anyOf": tags}
- else:
- include_tags = tag_info.tag_name
-
- if enabled:
- suite_options = tag_info.suite_options._replace(include_tags=include_tags)
- else:
- suite_options = resmokelib.config.SuiteOptions.ALL_INHERITED._replace(
- include_tags={"$not": include_tags})
-
- suite_options_list.append(suite_options)
-
- suite_options = resmokelib.config.SuiteOptions.combine(*suite_options_list)
- suite_options = suite_options._replace(description=tag_desc)
- suites.append(suite.with_options(suite_options))
-
- return suites
-
-
-if __name__ == "__main__":
- main = Main() # pylint: disable=invalid-name
- main.configure_from_command_line()
- main.run()
diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py
index 57288be7f23..9cc54d5c5f3 100755
--- a/buildscripts/resmoke.py
+++ b/buildscripts/resmoke.py
@@ -2,398 +2,22 @@
"""Command line utility for executing MongoDB tests of all kinds."""
import os.path
-import platform
-import random
-import subprocess
import sys
-import tarfile
import time
-import pkg_resources
-import requests
-
-try:
- import grpc_tools.protoc
- import grpc
-except ImportError:
- pass
-
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# pylint: disable=wrong-import-position
-from buildscripts.resmokelib import config
-from buildscripts.resmokelib import errors
-from buildscripts.resmokelib import logging
from buildscripts.resmokelib import parser
-from buildscripts.resmokelib import reportfile
-from buildscripts.resmokelib import sighandler
-from buildscripts.resmokelib import suitesconfig
-from buildscripts.resmokelib import testing
-from buildscripts.resmokelib import utils
-
-from buildscripts.resmokelib.core import process
-from buildscripts.resmokelib.core import jasper_process
-
-
-class Resmoke(object): # pylint: disable=too-many-instance-attributes
- """The main class to run tests with resmoke."""
-
- def __init__(self):
- """Initialize the Resmoke instance."""
- self.__start_time = time.time()
- self._config = None
- self._exec_logger = None
- self._resmoke_logger = None
- self._archive = None
- self._jasper_server = None
- self._interrupted = False
- self._exit_code = 0
-
- def configure_from_command_line(self):
- """Configure this instance using the command line arguments."""
- self._config = parser.parse_command_line()
-
- def _setup_logging(self):
- logging.loggers.configure_loggers(self._config.logging_config)
- logging.flush.start_thread()
- self._exec_logger = logging.loggers.EXECUTOR_LOGGER
- self._resmoke_logger = self._exec_logger.new_resmoke_logger()
-
- def _exit_logging(self):
- if self._interrupted:
- # We want to exit as quickly as possible when interrupted by a user and therefore don't
- # bother waiting for all log output to be flushed to logkeeper.
- return
-
- if logging.buildlogger.is_log_output_incomplete():
- # If we already failed to write log output to logkeeper, then we don't bother waiting
- # for any remaining log output to be flushed as it'll likely fail too. Exiting without
- # joining the flush thread here also means that resmoke.py won't hang due a logger from
- # a fixture or a background hook not being closed.
- self._exit_on_incomplete_logging()
- return
-
- logging.flush.stop_thread()
-
- if logging.buildlogger.is_log_output_incomplete():
- self._exit_on_incomplete_logging()
-
- def _exit_on_incomplete_logging(self):
- if self._exit_code == 0:
- # We don't anticipate users to look at passing Evergreen tasks very often that even if
- # the log output is incomplete, we'd still rather not show anything in the Evergreen UI
- # or cause a JIRA ticket to be created.
- self._resmoke_logger.info(
- "We failed to flush all log output to logkeeper but all tests passed, so"
- " ignoring.")
- return
-
- exit_code = errors.LoggerRuntimeConfigError.EXIT_CODE
- self._resmoke_logger.info(
- "Exiting with code %d rather than requested code %d because we failed to flush all"
- " log output to logkeeper.", exit_code, self._exit_code)
- self.exit(exit_code)
-
- def run(self):
- """Run resmoke."""
- if self._config is None:
- raise RuntimeError("Resmoke must be configured before calling run()")
- self._setup_logging()
-
- try:
- if self._config.list_suites:
- self.list_suites()
- elif self._config.find_suites:
- self.find_suites()
- elif self._config.dry_run == "tests":
- self.dry_run()
- else:
- self.run_tests()
- finally:
- self._exit_logging()
-
- def list_suites(self):
- """List the suites that are available to execute."""
- suite_names = suitesconfig.get_named_suites()
- self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names))
-
- def find_suites(self):
- """List the suites that run the specified tests."""
- suites = self._get_suites()
- suites_by_test = self._find_suites_by_test(suites)
- for test in sorted(suites_by_test):
- suite_names = suites_by_test[test]
- self._resmoke_logger.info("%s will be run by the following suite(s): %s", test,
- suite_names)
-
- @staticmethod
- def _find_suites_by_test(suites):
- """
- Look up what other resmoke suites run the tests specified in the suites parameter.
-
- Return a dict keyed by test name, value is array of suite names.
- """
- memberships = {}
- test_membership = suitesconfig.create_test_membership_map()
- for suite in suites:
- for test in suite.tests:
- memberships[test] = test_membership[test]
- return memberships
-
- def dry_run(self):
- """List which tests would run and which tests would be excluded in a resmoke invocation."""
- suites = self._get_suites()
- for suite in suites:
- self._shuffle_tests(suite)
- sb = ["Tests that would be run in suite {}".format(suite.get_display_name())]
- sb.extend(suite.tests or ["(no tests)"])
- sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name()))
- sb.extend(suite.excluded or ["(no tests)"])
- self._exec_logger.info("\n".join(sb))
-
- def run_tests(self):
- """Run the suite and tests specified."""
- self._resmoke_logger.info("verbatim resmoke.py invocation: %s", " ".join(sys.argv))
-
- if config.EVERGREEN_TASK_ID:
- local_args = parser.to_local_args()
- self._resmoke_logger.info("resmoke.py invocation for local usage: %s %s",
- os.path.join("buildscripts", "resmoke.py"),
- " ".join(local_args))
-
- suites = None
- try:
- suites = self._get_suites()
- self._setup_archival()
- if config.SPAWN_USING == "jasper":
- self._setup_jasper()
- self._setup_signal_handler(suites)
-
- for suite in suites:
- self._interrupted = self._run_suite(suite)
- if self._interrupted or (suite.options.fail_fast and suite.return_code != 0):
- self._log_resmoke_summary(suites)
- self.exit(suite.return_code)
-
- self._log_resmoke_summary(suites)
-
- # Exit with a nonzero code if any of the suites failed.
- exit_code = max(suite.return_code for suite in suites)
- self.exit(exit_code)
- finally:
- if config.SPAWN_USING == "jasper":
- self._exit_jasper()
- self._exit_archival()
- if suites:
- reportfile.write(suites)
-
- def _run_suite(self, suite):
- """Run a test suite."""
- self._log_suite_config(suite)
- suite.record_suite_start()
- interrupted = self._execute_suite(suite)
- suite.record_suite_end()
- self._log_suite_summary(suite)
- return interrupted
-
- def _log_resmoke_summary(self, suites):
- """Log a summary of the resmoke run."""
- time_taken = time.time() - self.__start_time
- if len(self._config.suite_files) > 1:
- testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken)
-
- def _log_suite_summary(self, suite):
- """Log a summary of the suite run."""
- self._resmoke_logger.info("=" * 80)
- self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(),
- self._get_suite_summary(suite))
-
- def _execute_suite(self, suite):
- """Execute a suite and return True if interrupted, False otherwise."""
- self._shuffle_tests(suite)
- if not suite.tests:
- self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind)
- suite.return_code = 0
- return False
- executor_config = suite.get_executor_config()
- try:
- executor = testing.executor.TestSuiteExecutor(
- self._exec_logger, suite, archive_instance=self._archive, **executor_config)
- executor.run()
- except (errors.UserInterrupt, errors.LoggerRuntimeConfigError) as err:
- self._exec_logger.error("Encountered an error when running %ss of suite %s: %s",
- suite.test_kind, suite.get_display_name(), err)
- suite.return_code = err.EXIT_CODE
- return True
- except IOError:
- suite.return_code = 74 # Exit code for IOError on POSIX systems.
- return True
- except: # pylint: disable=bare-except
- self._exec_logger.exception("Encountered an error when running %ss of suite %s.",
- suite.test_kind, suite.get_display_name())
- suite.return_code = 2
- return False
- return False
-
- def _shuffle_tests(self, suite):
- """Shuffle the tests if the shuffle cli option was set."""
- random.seed(config.RANDOM_SEED)
- if not config.SHUFFLE:
- return
- self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.",
- suite.test_kind, suite.get_display_name(), config.RANDOM_SEED)
- random.shuffle(suite.tests)
-
- def _get_suites(self):
- """Return the list of suites for this resmoke invocation."""
- try:
- return suitesconfig.get_suites(self._config.suite_files, self._config.test_files)
- except errors.SuiteNotFound as err:
- self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err))
- self.list_suites()
- self.exit(1)
-
- def _log_suite_config(self, suite):
- sb = [
- "YAML configuration of suite {}".format(suite.get_display_name()),
- utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "",
- utils.dump_yaml({"selector": suite.get_selector_config()}), "",
- utils.dump_yaml({"executor": suite.get_executor_config()}), "",
- utils.dump_yaml({"logging": self._config.logging_config})
- ]
- self._resmoke_logger.info("\n".join(sb))
-
- @staticmethod
- def _get_suite_summary(suite):
- """Return a summary of the suite run."""
- sb = []
- suite.summarize(sb)
- return "\n".join(sb)
-
- def _setup_signal_handler(self, suites):
- """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump."""
- sighandler.register(self._resmoke_logger, suites, self.__start_time)
-
- def _setup_archival(self):
- """Set up the archival feature if enabled in the cli options."""
- if config.ARCHIVE_FILE:
- self._archive = utils.archival.Archival(
- archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB,
- limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger)
-
- def _exit_archival(self):
- """Finish up archival tasks before exit if enabled in the cli options."""
- if self._archive and not self._interrupted:
- self._archive.exit()
-
- # pylint: disable=too-many-instance-attributes,too-many-statements,too-many-locals
- def _setup_jasper(self):
- """Start up the jasper process manager."""
- root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto")
- try:
- well_known_protos_include = pkg_resources.resource_filename("grpc_tools", "_proto")
- except ImportError:
- raise ImportError("You must run: sys.executable + '-m pip install grpcio grpcio-tools "
- "googleapis-common-protos' to use --spawnUsing=jasper.")
-
- # We use the build/ directory as the output directory because the generated files aren't
- # meant to because tracked by git or linted.
- proto_out = os.path.join(root_dir, "build", "jasper")
-
- utils.rmtree(proto_out, ignore_errors=True)
- os.makedirs(proto_out)
-
- # We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the
- # *pb2*.py modules from it.
- with open(os.path.join(proto_out, "__init__.py"), "w"):
- pass
-
- ret = grpc_tools.protoc.main([
- grpc_tools.protoc.__file__,
- "--grpc_python_out",
- proto_out,
- "--python_out",
- proto_out,
- "--proto_path",
- os.path.dirname(proto_file),
- "--proto_path",
- well_known_protos_include,
- os.path.basename(proto_file),
- ])
-
- if ret != 0:
- raise RuntimeError("Failed to generated gRPC files from the jasper.proto file")
-
- sys.path.append(os.path.dirname(proto_out))
-
- from jasper import jasper_pb2
- from jasper import jasper_pb2_grpc
-
- jasper_process.Process.jasper_pb2 = jasper_pb2
- jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc
-
- curator_path = "build/curator"
- if sys.platform == "win32":
- curator_path += ".exe"
- git_hash = "d846f0c875716e9377044ab2a50542724369662a"
- curator_exists = os.path.isfile(curator_path)
- curator_same_version = False
- if curator_exists:
- curator_version = subprocess.check_output([curator_path,
- "--version"]).decode('utf-8').split()
- curator_same_version = git_hash in curator_version
-
- if curator_exists and not curator_same_version:
- os.remove(curator_path)
- self._resmoke_logger.info(
- "Found a different version of curator. Downloading version %s of curator to enable"
- "process management using jasper.", git_hash)
-
- if not curator_exists or not curator_same_version:
- if sys.platform == "darwin":
- os_platform = "macos"
- elif sys.platform == "win32":
- os_platform = "windows-64"
- elif sys.platform.startswith("linux"):
- os_platform = "ubuntu1604"
- else:
- raise OSError("Unrecognized platform. "
- "This program is meant to be run on MacOS, Windows, or Linux.")
- url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/"
- "curator-dist-%s-%s.tar.gz") % (os_platform, git_hash)
- response = requests.get(url, stream=True)
- with tarfile.open(mode="r|gz", fileobj=response.raw) as tf:
- tf.extractall(path="./build/")
-
- jasper_port = config.BASE_PORT - 1
- jasper_conn_str = "localhost:%d" % jasper_port
- jasper_process.Process.connection_str = jasper_conn_str
- jasper_command = [curator_path, "jasper", "grpc", "--port", str(jasper_port)]
- self._jasper_server = process.Process(self._resmoke_logger, jasper_command)
- self._jasper_server.start()
-
- channel = grpc.insecure_channel(jasper_conn_str)
- grpc.channel_ready_future(channel).result()
-
- def _exit_jasper(self):
- if self._jasper_server:
- self._jasper_server.stop()
-
- def exit(self, exit_code):
- """Exit with the provided exit code."""
- self._exit_code = exit_code
- self._resmoke_logger.info("Exiting with code: %d", exit_code)
- sys.exit(exit_code)
def main():
"""Execute Main function for resmoke."""
- resmoke = Resmoke()
- resmoke.configure_from_command_line()
- resmoke.run()
+ __start_time = time.time()
+ subcommand = parser.parse_command_line(sys.argv[1:], start_time=__start_time)
+ subcommand.execute()
if __name__ == "__main__":
diff --git a/buildscripts/resmokelib/commands/__init__.py b/buildscripts/resmokelib/commands/__init__.py
new file mode 100644
index 00000000000..1a3235631a7
--- /dev/null
+++ b/buildscripts/resmokelib/commands/__init__.py
@@ -0,0 +1,4 @@
+"""Resmokelib subcommands."""
+
+from . import interface
+from . import run
diff --git a/buildscripts/resmokelib/commands/interface.py b/buildscripts/resmokelib/commands/interface.py
new file mode 100644
index 00000000000..c436b758f9c
--- /dev/null
+++ b/buildscripts/resmokelib/commands/interface.py
@@ -0,0 +1,9 @@
+"""Interface for creating a resmoke subcommand."""
+
+
+class Subcommand(object):
+ """A resmoke subcommand to execute."""
+
+ def execute(self):
+ """Execute the subcommand."""
+ raise NotImplementedError("execue must be implemented by Subcommand subclasses")
diff --git a/buildscripts/resmokelib/commands/run.py b/buildscripts/resmokelib/commands/run.py
new file mode 100644
index 00000000000..d60bbd4e256
--- /dev/null
+++ b/buildscripts/resmokelib/commands/run.py
@@ -0,0 +1,511 @@
+"""Command line utility for executing MongoDB tests of all kinds."""
+
+import collections
+import os
+import os.path
+import random
+import subprocess
+import sys
+import tarfile
+import time
+
+import pkg_resources
+import requests
+
+try:
+ import grpc_tools.protoc
+ import grpc
+except ImportError:
+ pass
+
+# pylint: disable=wrong-import-position
+from buildscripts.resmokelib import config
+from buildscripts.resmokelib import errors
+from buildscripts.resmokelib import logging
+from buildscripts.resmokelib import reportfile
+from buildscripts.resmokelib import sighandler
+from buildscripts.resmokelib import suitesconfig
+from buildscripts.resmokelib import testing
+from buildscripts.resmokelib import utils
+
+from buildscripts.resmokelib.commands import interface
+
+from buildscripts.resmokelib.core import process
+from buildscripts.resmokelib.core import jasper_process
+
+
+class TestRunner(interface.Subcommand): # pylint: disable=too-many-instance-attributes
+ """The main class to run tests with resmoke."""
+
+ def __init__(self, command, start_time=time.time()):
+ """Initialize the Resmoke instance."""
+ self.__start_time = start_time
+ self.__command = command
+ self._exec_logger = None
+ self._resmoke_logger = None
+ self._archive = None
+ self._jasper_server = None
+ self._interrupted = False
+ self._exit_code = 0
+
+ def _setup_logging(self):
+ logging.loggers.configure_loggers(config.LOGGING_CONFIG)
+ logging.flush.start_thread()
+ self._exec_logger = logging.loggers.EXECUTOR_LOGGER
+ self._resmoke_logger = self._exec_logger.new_resmoke_logger()
+
+ def _exit_logging(self):
+ if self._interrupted:
+ # We want to exit as quickly as possible when interrupted by a user and therefore don't
+ # bother waiting for all log output to be flushed to logkeeper.
+ return
+
+ if logging.buildlogger.is_log_output_incomplete():
+ # If we already failed to write log output to logkeeper, then we don't bother waiting
+ # for any remaining log output to be flushed as it'll likely fail too. Exiting without
+ # joining the flush thread here also means that resmoke.py won't hang due a logger from
+ # a fixture or a background hook not being closed.
+ self._exit_on_incomplete_logging()
+ return
+
+ logging.flush.stop_thread()
+
+ if logging.buildlogger.is_log_output_incomplete():
+ self._exit_on_incomplete_logging()
+
+ def _exit_on_incomplete_logging(self):
+ if self._exit_code == 0:
+ # We don't anticipate users to look at passing Evergreen tasks very often that even if
+ # the log output is incomplete, we'd still rather not show anything in the Evergreen UI
+ # or cause a JIRA ticket to be created.
+ self._resmoke_logger.info(
+ "We failed to flush all log output to logkeeper but all tests passed, so"
+ " ignoring.")
+ return
+
+ exit_code = errors.LoggerRuntimeConfigError.EXIT_CODE
+ self._resmoke_logger.info(
+ "Exiting with code %d rather than requested code %d because we failed to flush all"
+ " log output to logkeeper.", exit_code, self._exit_code)
+ self.exit(exit_code)
+
+ def execute(self):
+ """Execute the 'run' subcommand."""
+ self._setup_logging()
+
+ try:
+ if self.__command == "list-suites":
+ self.list_suites()
+ elif self.__command == "find-suites":
+ self.find_suites()
+ elif config.DRY_RUN == "tests":
+ self.dry_run()
+ else:
+ self.run_tests()
+ finally:
+ self._exit_logging()
+
+ def list_suites(self):
+ """List the suites that are available to execute."""
+ suite_names = suitesconfig.get_named_suites()
+ self._resmoke_logger.info("Suites available to execute:\n%s", "\n".join(suite_names))
+
+ def find_suites(self):
+ """List the suites that run the specified tests."""
+ suites = self._get_suites()
+ suites_by_test = self._find_suites_by_test(suites)
+ for test in sorted(suites_by_test):
+ suite_names = suites_by_test[test]
+ self._resmoke_logger.info("%s will be run by the following suite(s): %s", test,
+ suite_names)
+
+ @staticmethod
+ def _find_suites_by_test(suites):
+ """
+ Look up what other resmoke suites run the tests specified in the suites parameter.
+
+ Return a dict keyed by test name, value is array of suite names.
+ """
+ memberships = {}
+ test_membership = suitesconfig.create_test_membership_map()
+ for suite in suites:
+ for test in suite.tests:
+ memberships[test] = test_membership[test]
+ return memberships
+
+ def dry_run(self):
+ """List which tests would run and which tests would be excluded in a resmoke invocation."""
+ suites = self._get_suites()
+ for suite in suites:
+ self._shuffle_tests(suite)
+ sb = ["Tests that would be run in suite {}".format(suite.get_display_name())]
+ sb.extend(suite.tests or ["(no tests)"])
+ sb.append("Tests that would be excluded from suite {}".format(suite.get_display_name()))
+ sb.extend(suite.excluded or ["(no tests)"])
+ self._exec_logger.info("\n".join(sb))
+
+ def run_tests(self):
+ """Run the suite and tests specified."""
+ self._resmoke_logger.info("verbatim resmoke.py invocation: %s", " ".join(sys.argv))
+
+ # if config.EVERGREEN_TASK_ID:
+ # local_args = parser.to_local_args()
+ # self._resmoke_logger.info("resmoke.py invocation for local usage: %s %s",
+ # os.path.join("buildscripts", "resmoke.py"),
+ # " ".join(local_args))
+
+ suites = None
+ try:
+ suites = self._get_suites()
+ self._setup_archival()
+ if config.SPAWN_USING == "jasper":
+ self._setup_jasper()
+ self._setup_signal_handler(suites)
+
+ for suite in suites:
+ self._interrupted = self._run_suite(suite)
+ if self._interrupted or (suite.options.fail_fast and suite.return_code != 0):
+ self._log_resmoke_summary(suites)
+ self.exit(suite.return_code)
+
+ self._log_resmoke_summary(suites)
+
+ # Exit with a nonzero code if any of the suites failed.
+ exit_code = max(suite.return_code for suite in suites)
+ self.exit(exit_code)
+ finally:
+ if config.SPAWN_USING == "jasper":
+ self._exit_jasper()
+ self._exit_archival()
+ if suites:
+ reportfile.write(suites)
+
+ def _run_suite(self, suite):
+ """Run a test suite."""
+ self._log_suite_config(suite)
+ suite.record_suite_start()
+ interrupted = self._execute_suite(suite)
+ suite.record_suite_end()
+ self._log_suite_summary(suite)
+ return interrupted
+
+ def _log_resmoke_summary(self, suites):
+ """Log a summary of the resmoke run."""
+ time_taken = time.time() - self.__start_time
+ if len(config.SUITE_FILES) > 1:
+ testing.suite.Suite.log_summaries(self._resmoke_logger, suites, time_taken)
+
+ def _log_suite_summary(self, suite):
+ """Log a summary of the suite run."""
+ self._resmoke_logger.info("=" * 80)
+ self._resmoke_logger.info("Summary of %s suite: %s", suite.get_display_name(),
+ self._get_suite_summary(suite))
+
+ def _execute_suite(self, suite):
+ """Execute a suite and return True if interrupted, False otherwise."""
+ self._shuffle_tests(suite)
+ if not suite.tests:
+ self._exec_logger.info("Skipping %s, no tests to run", suite.test_kind)
+ suite.return_code = 0
+ return False
+ executor_config = suite.get_executor_config()
+ try:
+ executor = testing.executor.TestSuiteExecutor(
+ self._exec_logger, suite, archive_instance=self._archive, **executor_config)
+ executor.run()
+ except (errors.UserInterrupt, errors.LoggerRuntimeConfigError) as err:
+ self._exec_logger.error("Encountered an error when running %ss of suite %s: %s",
+ suite.test_kind, suite.get_display_name(), err)
+ suite.return_code = err.EXIT_CODE
+ return True
+ except IOError:
+ suite.return_code = 74 # Exit code for IOError on POSIX systems.
+ return True
+ except: # pylint: disable=bare-except
+ self._exec_logger.exception("Encountered an error when running %ss of suite %s.",
+ suite.test_kind, suite.get_display_name())
+ suite.return_code = 2
+ return False
+ return False
+
+ def _shuffle_tests(self, suite):
+ """Shuffle the tests if the shuffle cli option was set."""
+ random.seed(config.RANDOM_SEED)
+ if not config.SHUFFLE:
+ return
+ self._exec_logger.info("Shuffling order of tests for %ss in suite %s. The seed is %d.",
+ suite.test_kind, suite.get_display_name(), config.RANDOM_SEED)
+ random.shuffle(suite.tests)
+
+ def _get_suites(self):
+ """Return the list of suites for this resmoke invocation."""
+ try:
+ return suitesconfig.get_suites(config.SUITE_FILES, config.TEST_FILES)
+ except errors.SuiteNotFound as err:
+ self._resmoke_logger.error("Failed to parse YAML suite definition: %s", str(err))
+ self.list_suites()
+ self.exit(1)
+
+ def _log_suite_config(self, suite):
+ sb = [
+ "YAML configuration of suite {}".format(suite.get_display_name()),
+ utils.dump_yaml({"test_kind": suite.get_test_kind_config()}), "",
+ utils.dump_yaml({"selector": suite.get_selector_config()}), "",
+ utils.dump_yaml({"executor": suite.get_executor_config()}), "",
+ utils.dump_yaml({"logging": config.LOGGING_CONFIG})
+ ]
+ self._resmoke_logger.info("\n".join(sb))
+
+ @staticmethod
+ def _get_suite_summary(suite):
+ """Return a summary of the suite run."""
+ sb = []
+ suite.summarize(sb)
+ return "\n".join(sb)
+
+ def _setup_signal_handler(self, suites):
+ """Set up a SIGUSR1 signal handler that logs a test result summary and a thread dump."""
+ sighandler.register(self._resmoke_logger, suites, self.__start_time)
+
+ def _setup_archival(self):
+ """Set up the archival feature if enabled in the cli options."""
+ if config.ARCHIVE_FILE:
+ self._archive = utils.archival.Archival(
+ archival_json_file=config.ARCHIVE_FILE, limit_size_mb=config.ARCHIVE_LIMIT_MB,
+ limit_files=config.ARCHIVE_LIMIT_TESTS, logger=self._exec_logger)
+
+ def _exit_archival(self):
+ """Finish up archival tasks before exit if enabled in the cli options."""
+ if self._archive and not self._interrupted:
+ self._archive.exit()
+
+ # pylint: disable=too-many-instance-attributes,too-many-statements,too-many-locals
+ def _setup_jasper(self):
+ """Start up the jasper process manager."""
+ root_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ proto_file = os.path.join(root_dir, "buildscripts", "resmokelib", "core", "jasper.proto")
+ try:
+ well_known_protos_include = pkg_resources.resource_filename("grpc_tools", "_proto")
+ except ImportError:
+ raise ImportError("You must run: sys.executable + '-m pip install grpcio grpcio-tools "
+ "googleapis-common-protos' to use --spawnUsing=jasper.")
+
+ # We use the build/ directory as the output directory because the generated files aren't
+ # meant to because tracked by git or linted.
+ proto_out = os.path.join(root_dir, "build", "jasper")
+
+ utils.rmtree(proto_out, ignore_errors=True)
+ os.makedirs(proto_out)
+
+ # We make 'proto_out' into a Python package so we can add it to 'sys.path' and import the
+ # *pb2*.py modules from it.
+ with open(os.path.join(proto_out, "__init__.py"), "w"):
+ pass
+
+ ret = grpc_tools.protoc.main([
+ grpc_tools.protoc.__file__,
+ "--grpc_python_out",
+ proto_out,
+ "--python_out",
+ proto_out,
+ "--proto_path",
+ os.path.dirname(proto_file),
+ "--proto_path",
+ well_known_protos_include,
+ os.path.basename(proto_file),
+ ])
+
+ if ret != 0:
+ raise RuntimeError("Failed to generated gRPC files from the jasper.proto file")
+
+ sys.path.append(os.path.dirname(proto_out))
+
+ from jasper import jasper_pb2
+ from jasper import jasper_pb2_grpc
+
+ jasper_process.Process.jasper_pb2 = jasper_pb2
+ jasper_process.Process.jasper_pb2_grpc = jasper_pb2_grpc
+
+ curator_path = "build/curator"
+ if sys.platform == "win32":
+ curator_path += ".exe"
+ git_hash = "d846f0c875716e9377044ab2a50542724369662a"
+ curator_exists = os.path.isfile(curator_path)
+ curator_same_version = False
+ if curator_exists:
+ curator_version = subprocess.check_output([curator_path,
+ "--version"]).decode('utf-8').split()
+ curator_same_version = git_hash in curator_version
+
+ if curator_exists and not curator_same_version:
+ os.remove(curator_path)
+ self._resmoke_logger.info(
+ "Found a different version of curator. Downloading version %s of curator to enable"
+ "process management using jasper.", git_hash)
+
+ if not curator_exists or not curator_same_version:
+ if sys.platform == "darwin":
+ os_platform = "macos"
+ elif sys.platform == "win32":
+ os_platform = "windows-64"
+ elif sys.platform.startswith("linux"):
+ os_platform = "ubuntu1604"
+ else:
+ raise OSError("Unrecognized platform. "
+ "This program is meant to be run on MacOS, Windows, or Linux.")
+ url = ("https://s3.amazonaws.com/boxes.10gen.com/build/curator/"
+ "curator-dist-%s-%s.tar.gz") % (os_platform, git_hash)
+ response = requests.get(url, stream=True)
+ with tarfile.open(mode="r|gz", fileobj=response.raw) as tf:
+ tf.extractall(path="./build/")
+
+ jasper_port = config.BASE_PORT - 1
+ jasper_conn_str = "localhost:%d" % jasper_port
+ jasper_process.Process.connection_str = jasper_conn_str
+ jasper_command = [curator_path, "jasper", "grpc", "--port", str(jasper_port)]
+ self._jasper_server = process.Process(self._resmoke_logger, jasper_command)
+ self._jasper_server.start()
+
+ channel = grpc.insecure_channel(jasper_conn_str)
+ grpc.channel_ready_future(channel).result()
+
+ def _exit_jasper(self):
+ if self._jasper_server:
+ self._jasper_server.stop()
+
+ def exit(self, exit_code):
+ """Exit with the provided exit code."""
+ self._exit_code = exit_code
+ self._resmoke_logger.info("Exiting with code: %d", exit_code)
+ sys.exit(exit_code)
+
+
+_TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"])
+
+
+class TestRunnerEvg(TestRunner):
+ """Execute Main class.
+
+ A class for executing potentially multiple resmoke.py test suites in a way that handles
+ additional options for running unreliable tests in Evergreen.
+ """
+
+ UNRELIABLE_TAG = _TagInfo(
+ tag_name="unreliable",
+ evergreen_aware=True,
+ suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
+ report_failure_status="silentfail"))
+
+ RESOURCE_INTENSIVE_TAG = _TagInfo(
+ tag_name="resource_intensive",
+ evergreen_aware=False,
+ suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
+ num_jobs=1))
+
+ RETRY_ON_FAILURE_TAG = _TagInfo(
+ tag_name="retry_on_failure",
+ evergreen_aware=True,
+ suite_options=config.SuiteOptions.ALL_INHERITED._replace( # type: ignore
+ fail_fast=False, num_repeat_suites=2, num_repeat_tests=1,
+ report_failure_status="silentfail"))
+
+ @staticmethod
+ def _make_evergreen_aware_tags(tag_name):
+ """Return a list of resmoke.py tags.
+
+ This list is for task, variant, and distro combinations in Evergreen.
+ """
+
+ tags_format = ["{tag_name}"]
+
+ if config.EVERGREEN_TASK_NAME is not None:
+ tags_format.append("{tag_name}|{task_name}")
+
+ if config.EVERGREEN_VARIANT_NAME is not None:
+ tags_format.append("{tag_name}|{task_name}|{variant_name}")
+
+ if config.EVERGREEN_DISTRO_ID is not None:
+ tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}")
+
+ return [
+ tag.format(tag_name=tag_name, task_name=config.EVERGREEN_TASK_NAME,
+ variant_name=config.EVERGREEN_VARIANT_NAME,
+ distro_id=config.EVERGREEN_DISTRO_ID) for tag in tags_format
+ ]
+
+ @classmethod
+ def _make_tag_combinations(cls):
+ """Return a list of (tag, enabled) pairs.
+
+ These pairs represent all possible combinations of all possible pairings
+ of whether the tags are enabled or disabled together.
+ """
+
+ combinations = []
+
+ if config.EVERGREEN_PATCH_BUILD:
+ combinations.append(("unreliable and resource intensive",
+ ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True))))
+ combinations.append(("unreliable and not resource intensive",
+ ((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False))))
+ combinations.append(("reliable and resource intensive",
+ ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True))))
+ combinations.append(("reliable and not resource intensive",
+ ((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
+ False))))
+ else:
+ combinations.append(("retry on failure and resource intensive",
+ ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
+ True))))
+ combinations.append(("retry on failure and not resource intensive",
+ ((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
+ False))))
+ combinations.append(("run once and resource intensive",
+ ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
+ True))))
+ combinations.append(("run once and not resource intensive",
+ ((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
+ False))))
+
+ return combinations
+
+ def _get_suites(self):
+ """Return a list of resmokelib.testing.suite.Suite instances to execute.
+
+ For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(),
+ multiple copies of that test suite are run using different resmokelib.config.SuiteOptions()
+ depending on whether each tag in the combination is enabled or not.
+ """
+
+ suites = []
+
+ for suite in TestRunner._get_suites(self):
+ if suite.test_kind != "js_test":
+ # Tags are only support for JavaScript tests, so we leave the test suite alone when
+ # running any other kind of test.
+ suites.append(suite)
+ continue
+
+ for (tag_desc, tag_combo) in self._make_tag_combinations():
+ suite_options_list = []
+
+ for (tag_info, enabled) in tag_combo:
+ if tag_info.evergreen_aware:
+ tags = self._make_evergreen_aware_tags(tag_info.tag_name)
+ include_tags = {"$anyOf": tags}
+ else:
+ include_tags = tag_info.tag_name
+
+ if enabled:
+ suite_options = tag_info.suite_options._replace(include_tags=include_tags)
+ else:
+ suite_options = config.SuiteOptions.ALL_INHERITED._replace(
+ include_tags={"$not": include_tags})
+
+ suite_options_list.append(suite_options)
+
+ suite_options = config.SuiteOptions.combine(*suite_options_list)
+ suite_options = suite_options._replace(description=tag_desc)
+ suites.append(suite.with_options(suite_options))
+
+ return suites
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index d93313e3a2d..352453e774d 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -55,6 +55,7 @@ DEFAULTS = {
"genny_executable": None,
"include_with_any_tags": None,
"jobs": 1,
+ "logger_file": None,
"mongo_executable": None,
"mongod_executable": None,
"mongod_set_parameters": None,
@@ -82,7 +83,9 @@ DEFAULTS = {
"majority_read_concern": None, # Default is set on the commandline.
"storage_engine": None,
"storage_engine_cache_size_gb": None,
+ "suite_files": None,
"tag_file": None,
+ "test_files": None,
"transport_layer": None,
# Evergreen options.
@@ -287,6 +290,9 @@ INCLUDE_WITH_ANY_TAGS = None
# If set, then resmoke.py starts the specified number of Job instances to run tests.
JOBS = None
+# Yaml file that specified logging configuration.
+LOGGER_FILE = None
+
# The path to the mongo executable used by resmoke.py.
MONGO_EXECUTABLE = None
@@ -381,9 +387,15 @@ STORAGE_ENGINE = None
# storage engine cache size.
STORAGE_ENGINE_CACHE_SIZE = None
+# Yaml suites that specify how tests should be executed.
+SUITE_FILES = None
+
# The tag file to use that associates tests with tags.
TAG_FILE = None
+# The test files to execute.
+TEST_FILES = None
+
# If set, then mongod/mongos's started by resmoke.py will use the specified transport layer.
TRANSPORT_LAYER = None
@@ -427,3 +439,6 @@ DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt"
# therefore might not be available when creating a test membership map.
EXTERNAL_SUITE_SELECTORS = (DEFAULT_BENCHMARK_TEST_LIST, DEFAULT_UNIT_TEST_LIST,
DEFAULT_INTEGRATION_TEST_LIST, DEFAULT_DBTEST_EXECUTABLE)
+
+# Generated logging config for the current invocation.
+LOGGING_CONFIG = None
diff --git a/buildscripts/resmokelib/configure_resmoke.py b/buildscripts/resmokelib/configure_resmoke.py
new file mode 100644
index 00000000000..102daf86d69
--- /dev/null
+++ b/buildscripts/resmokelib/configure_resmoke.py
@@ -0,0 +1,234 @@
+"""Configure the command line input for the resmoke 'run' subcommand."""
+
+import argparse
+import os
+import os.path
+import sys
+import shlex
+import configparser
+
+from typing import NamedTuple
+
+import datetime
+import pymongo.uri_parser
+
+from buildscripts import resmokeconfig
+from . import config as _config
+from . import utils
+
+
+def validate_and_update_config(parser, args):
+ """Validate inputs and update config module."""
+ _validate_options(parser, args)
+ _update_config_vars(args)
+ _validate_config(parser)
+ _set_logging_config()
+
+
+def _validate_options(parser, args):
+ """Do preliminary validation on the options and error on any invalid options."""
+
+ if not 'shell_port' in args or not 'shell_conn_strin' in args:
+ return
+
+ if args.shell_port is not None and args.shell_conn_string is not None:
+ parser.error("Cannot specify both `shellPort` and `shellConnString`")
+
+ if args.executor_file:
+ parser.error("--executor is superseded by --suites; specify --suites={} {} to run the"
+ " test(s) under those suite configuration(s)".format(
+ args.executor_file, " ".join(args.test_files)))
+
+
+def _validate_config(parser):
+ """Do validation on the config settings."""
+
+ if _config.REPEAT_TESTS_MAX:
+ if not _config.REPEAT_TESTS_SECS:
+ parser.error("Must specify --repeatTestsSecs with --repeatTestsMax")
+
+ if _config.REPEAT_TESTS_MIN > _config.REPEAT_TESTS_MAX:
+ parser.error("--repeatTestsSecsMin > --repeatTestsMax")
+
+ if _config.REPEAT_TESTS_MIN and not _config.REPEAT_TESTS_SECS:
+ parser.error("Must specify --repeatTestsSecs with --repeatTestsMin")
+
+ if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS:
+ parser.error("Cannot specify --repeatTests and --repeatTestsSecs")
+
+
+def _update_config_vars(values): # pylint: disable=too-many-statements,too-many-locals,too-many-branches
+ """Update the variables of the config module."""
+
+ config = _config.DEFAULTS.copy()
+
+ # Use RSK_ prefixed environment variables to indicate resmoke-specific values.
+ # The list of configuration is detailed in config.py
+ resmoke_env_prefix = 'RSK_'
+ for key in os.environ.keys():
+ if key.startswith(resmoke_env_prefix):
+ # Windows env vars are case-insensitive, we use lowercase to be consistent
+ # with existing resmoke options.
+ config[key[len(resmoke_env_prefix):].lower()] = os.environ[key]
+
+ # Override `config` with values from command line arguments.
+ cmdline_vars = vars(values)
+ for cmdline_key in cmdline_vars:
+ if cmdline_key not in _config.DEFAULTS:
+ # Ignore options that don't map to values in config.py
+ continue
+ if cmdline_vars[cmdline_key] is not None:
+ config[cmdline_key] = cmdline_vars[cmdline_key]
+
+ if os.path.isfile("resmoke.ini"):
+ config_parser = configparser.ConfigParser()
+ config_parser.read("resmoke.ini")
+ if "resmoke" in config_parser.sections():
+ user_config = dict(config_parser["resmoke"])
+ config.update(user_config)
+
+ _config.ARCHIVE_FILE = config.pop("archive_file")
+ _config.BASE_PORT = int(config.pop("base_port"))
+ _config.BUILDLOGGER_URL = config.pop("buildlogger_url")
+ _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix"))
+ _config.DRY_RUN = config.pop("dry_run")
+ # EXCLUDE_WITH_ANY_TAGS will always contain the implicitly defined EXCLUDED_TAG.
+ _config.EXCLUDE_WITH_ANY_TAGS = [_config.EXCLUDED_TAG]
+ _config.EXCLUDE_WITH_ANY_TAGS.extend(
+ utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
+ _config.FAIL_FAST = not config.pop("continue_on_failure")
+ _config.FLOW_CONTROL = config.pop("flow_control")
+ _config.FLOW_CONTROL_TICKETS = config.pop("flow_control_tickets")
+ _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags"))
+ _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable"))
+ _config.JOBS = config.pop("jobs")
+ _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on"
+
+ _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable"))
+ _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
+ _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
+ _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
+ _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
+
+ _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters")
+ _config.NO_JOURNAL = config.pop("no_journal")
+ _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")
+ _config.PERF_REPORT_FILE = config.pop("perf_report_file")
+ _config.RANDOM_SEED = config.pop("seed")
+ _config.REPEAT_SUITES = config.pop("repeat_suites")
+ _config.REPEAT_TESTS = config.pop("repeat_tests")
+ _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max")
+ _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min")
+ _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs")
+ _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
+ _config.REPORT_FILE = config.pop("report_file")
+ _config.SERVICE_EXECUTOR = config.pop("service_executor")
+ _config.SHELL_READ_MODE = config.pop("shell_read_mode")
+ _config.SHELL_WRITE_MODE = config.pop("shell_write_mode")
+ _config.SPAWN_USING = config.pop("spawn_using")
+ _config.STAGGER_JOBS = config.pop("stagger_jobs") == "on"
+ _config.STORAGE_ENGINE = config.pop("storage_engine")
+ _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storage_engine_cache_size_gb")
+ _config.SUITE_FILES = config.pop("suite_files")
+ if _config.SUITE_FILES is not None:
+ _config.SUITE_FILES = _config.SUITE_FILES.split(",")
+ _config.TAG_FILE = config.pop("tag_file")
+ _config.TEST_FILES = config.pop("test_files")
+ _config.TRANSPORT_LAYER = config.pop("transport_layer")
+
+ # Evergreen options.
+ _config.EVERGREEN_BUILD_ID = config.pop("build_id")
+ _config.EVERGREEN_DISTRO_ID = config.pop("distro_id")
+ _config.EVERGREEN_EXECUTION = config.pop("execution_number")
+ _config.EVERGREEN_PATCH_BUILD = config.pop("patch_build")
+ _config.EVERGREEN_PROJECT_NAME = config.pop("project_name")
+ _config.EVERGREEN_REVISION = config.pop("git_revision")
+ _config.EVERGREEN_REVISION_ORDER_ID = config.pop("revision_order_id")
+ _config.EVERGREEN_TASK_ID = config.pop("task_id")
+ _config.EVERGREEN_TASK_NAME = config.pop("task_name")
+ _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name")
+ _config.EVERGREEN_VERSION_ID = config.pop("version_id")
+
+ # Archival options. Archival is enabled only when running on evergreen.
+ if not _config.EVERGREEN_TASK_ID:
+ _config.ARCHIVE_FILE = None
+ _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb")
+ _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests")
+
+ # Wiredtiger options.
+ _config.WT_COLL_CONFIG = config.pop("wt_coll_config")
+ _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config")
+ _config.WT_INDEX_CONFIG = config.pop("wt_index_config")
+
+ # Benchmark/Benchrun options.
+ _config.BENCHMARK_FILTER = config.pop("benchmark_filter")
+ _config.BENCHMARK_LIST_TESTS = config.pop("benchmark_list_tests")
+ benchmark_min_time = config.pop("benchmark_min_time_secs")
+ if benchmark_min_time is not None:
+ _config.BENCHMARK_MIN_TIME = datetime.timedelta(seconds=benchmark_min_time)
+ _config.BENCHMARK_REPETITIONS = config.pop("benchmark_repetitions")
+
+ shuffle = config.pop("shuffle")
+ if shuffle == "auto":
+ # If the user specified a value for --jobs > 1 (or -j > 1), then default to randomize
+ # the order in which tests are executed. This is because with multiple threads the tests
+ # wouldn't run in a deterministic order anyway.
+ _config.SHUFFLE = _config.JOBS > 1
+ else:
+ _config.SHUFFLE = shuffle == "on"
+
+ conn_string = config.pop("shell_conn_string")
+ port = config.pop("shell_port")
+
+ if port is not None:
+ conn_string = "mongodb://localhost:" + port
+
+ if conn_string is not None:
+ # The --shellConnString command line option must be a MongoDB connection URI, which means it
+ # must specify the mongodb:// or mongodb+srv:// URI scheme. pymongo.uri_parser.parse_uri()
+ # raises an exception if the connection string specified isn't considered a valid MongoDB
+ # connection URI.
+ pymongo.uri_parser.parse_uri(conn_string)
+ _config.SHELL_CONN_STRING = conn_string
+
+ _config.LOGGER_FILE = config.pop("logger_file")
+
+ if config:
+ raise ValueError(f"Unkown option(s): {list(config.keys())}s")
+
+
+def _set_logging_config():
+ """Read YAML configuration from 'pathname' how to log tests and fixtures."""
+ pathname = _config.LOGGER_FILE
+
+ # Named loggers are specified as the basename of the file, without the .yml extension.
+ if not utils.is_yaml_file(pathname) and not os.path.dirname(pathname):
+ if pathname not in resmokeconfig.NAMED_LOGGERS:
+ raise argparse.ValueError("Unknown logger '%s'" % pathname)
+ pathname = resmokeconfig.NAMED_LOGGERS[pathname] # Expand 'pathname' to full path.
+
+ if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname):
+ raise argparse.ValueError("Expected a logger YAML config, but got '%s'" % pathname)
+
+ _config.LOGGING_CONFIG = utils.load_yaml_file(pathname).pop("logging")
+
+
+def _expand_user(pathname):
+ """Provide wrapper around os.path.expanduser() to do nothing when given None."""
+ if pathname is None:
+ return None
+ return os.path.expanduser(pathname)
+
+
+def _tags_from_list(tags_list):
+ """Return the list of tags from a list of tag parameter values.
+
+ Each parameter value in the list may be a list of comma separated tags, with empty strings
+ ignored.
+ """
+ tags = []
+ if tags_list is not None:
+ for tag in tags_list:
+ tags.extend([t for t in tag.split(",") if t != ""])
+ return tags
+ return None
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 0d4bc4d4a69..44345050704 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -1,45 +1,57 @@
"""Parser for command line arguments."""
-import collections
import os
-import os.path
import sys
-import datetime
-import optparse
-import pymongo.uri_parser
+import argparse
from . import config as _config
-from . import utils
-from .. import resmokeconfig
+from . import configure_resmoke
+from . import commands
-ResmokeConfig = collections.namedtuple(
- "ResmokeConfig",
- ["list_suites", "find_suites", "dry_run", "suite_files", "test_files", "logging_config"])
+_EVERGREEN_ARGUMENT_TITLE = "Evergreen options"
-_EVERGREEN_OPTIONS_TITLE = "Evergreen options"
-
-def _make_parser(): # pylint: disable=too-many-statements
+def _make_parser():
"""Create and return the command line arguments parser."""
- parser = optparse.OptionParser()
+ parser = argparse.ArgumentParser()
+ subparsers = parser.add_subparsers(dest="command")
+
+ # Add sub-commands.
+ _add_run(subparsers)
+ _add_list_suites(subparsers)
+ _add_find_suites(subparsers)
+
+ return parser
+
+
+def _add_run(subparsers): # pylint: disable=too-many-statements
+ """Create and add the parser for the Run subcommand."""
+ parser = subparsers.add_parser("run", help="Runs the specified tests.")
+
+ parser.set_defaults(dry_run="off", shuffle="auto", stagger_jobs="off",
+ suite_files="with_server", majority_read_concern="on")
+
+ parser.add_argument("test_files", metavar="TEST_FILES", nargs="*",
+ help="Explicit test files to run")
- parser.add_option(
+ parser.add_argument(
"--suites", dest="suite_files", metavar="SUITE1,SUITE2",
help=("Comma separated list of YAML files that each specify the configuration"
" of a suite. If the file is located in the resmokeconfig/suites/"
" directory, then the basename without the .yml extension can be"
" specified, e.g. 'core'. If a list of files is passed in as"
" positional arguments, they will be run using the suites'"
- " configurations"))
+ " configurations."))
- parser.add_option(
+ parser.add_argument(
"--log", dest="logger_file", metavar="LOGGER",
help=("A YAML file that specifies the logging configuration. If the file is"
" located in the resmokeconfig/suites/ directory, then the basename"
" without the .yml extension can be specified, e.g. 'console'."))
+ parser.set_defaults(logger_file="console")
- parser.add_option(
+ parser.add_argument(
"--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
help=("Sets the archive file name for the Evergreen task running the tests."
" The archive file is JSON format containing a list of tests that were"
@@ -47,629 +59,476 @@ def _make_parser(): # pylint: disable=too-many-statements
" will be archived in S3. Tests can be designated for archival in the"
" task suite configuration file."))
- parser.add_option(
- "--archiveLimitMb", type="int", dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB",
+ parser.add_argument(
+ "--archiveLimitMb", type=int, dest="archive_limit_mb", metavar="ARCHIVE_LIMIT_MB",
help=("Sets the limit (in MB) for archived files to S3. A value of 0"
" indicates there is no limit."))
- parser.add_option(
- "--archiveLimitTests", type="int", dest="archive_limit_tests",
- metavar="ARCHIVE_LIMIT_TESTS",
+ parser.add_argument(
+ "--archiveLimitTests", type=int, dest="archive_limit_tests", metavar="ARCHIVE_LIMIT_TESTS",
help=("Sets the maximum number of tests to archive to S3. A value"
" of 0 indicates there is no limit."))
- parser.add_option(
+ parser.add_argument(
"--basePort", dest="base_port", metavar="PORT",
help=("The starting port number to use for mongod and mongos processes"
" spawned by resmoke.py or the tests themselves. Each fixture and Job"
" allocates a contiguous range of ports."))
- parser.add_option("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL",
- help="The root url of the buildlogger server.")
+ parser.add_argument("--buildloggerUrl", action="store", dest="buildlogger_url", metavar="URL",
+ help="The root url of the buildlogger server.")
- parser.add_option("--continueOnFailure", action="store_true", dest="continue_on_failure",
- help="Executes all tests in all suites, even if some of them fail.")
+ parser.add_argument("--continueOnFailure", action="store_true", dest="continue_on_failure",
+ help="Executes all tests in all suites, even if some of them fail.")
- parser.add_option(
+ parser.add_argument(
"--dbpathPrefix", dest="dbpath_prefix", metavar="PATH",
help=("The directory which will contain the dbpaths of any mongod's started"
" by resmoke.py or the tests themselves."))
- parser.add_option("--dbtest", dest="dbtest_executable", metavar="PATH",
- help="The path to the dbtest executable for resmoke to use.")
+ parser.add_argument("--dbtest", dest="dbtest_executable", metavar="PATH",
+ help="The path to the dbtest executable for resmoke to use.")
- parser.add_option(
+ parser.add_argument(
"--excludeWithAnyTags", action="append", dest="exclude_with_any_tags", metavar="TAG1,TAG2",
help=("Comma separated list of tags. Any jstest that contains any of the"
" specified tags will be excluded from any suites that are run."
" The tag '{}' is implicitly part of this list.".format(_config.EXCLUDED_TAG)))
- parser.add_option("-f", "--findSuites", action="store_true", dest="find_suites",
- help="Lists the names of the suites that will execute the specified tests.")
+ parser.add_argument("--genny", dest="genny_executable", metavar="PATH",
+ help="The path to the genny executable for resmoke to use.")
- parser.add_option("--genny", dest="genny_executable", metavar="PATH",
- help="The path to the genny executable for resmoke to use.")
-
- parser.add_option(
- "--spawnUsing", type="choice", dest="spawn_using", choices=("python", "jasper"),
+ parser.add_argument(
+ "--spawnUsing", dest="spawn_using", choices=("python", "jasper"),
help=("Allows you to spawn resmoke processes using python or Jasper."
"Defaults to python. Options are 'python' or 'jasper'."))
- parser.add_option(
+ parser.add_argument(
"--includeWithAnyTags", action="append", dest="include_with_any_tags", metavar="TAG1,TAG2",
help=("Comma separated list of tags. For the jstest portion of the suite(s),"
" only tests which have at least one of the specified tags will be"
" run."))
- parser.add_option("-n", action="store_const", const="tests", dest="dry_run",
- help="Outputs the tests that would be run.")
+ parser.add_argument("-n", action="store_const", const="tests", dest="dry_run",
+ help="Outputs the tests that would be run.")
# TODO: add support for --dryRun=commands
- parser.add_option(
- "--dryRun", type="choice", action="store", dest="dry_run", choices=("off", "tests"),
- metavar="MODE", help=("Instead of running the tests, outputs the tests that would be run"
- " (if MODE=tests). Defaults to MODE=%default."))
+ parser.add_argument(
+ "--dryRun", action="store", dest="dry_run", choices=("off", "tests"), metavar="MODE",
+ help=("Instead of running the tests, outputs the tests that would be run"
+ " (if MODE=tests). Defaults to MODE=%%default."))
- parser.add_option(
- "-j", "--jobs", type="int", dest="jobs", metavar="JOBS",
+ parser.add_argument(
+ "-j", "--jobs", type=int, dest="jobs", metavar="JOBS",
help=("The number of Job instances to use. Each instance will receive its"
" own MongoDB deployment to dispatch tests to."))
- parser.add_option("-l", "--listSuites", action="store_true", dest="list_suites",
- help="Lists the names of the suites available to execute.")
-
- parser.add_option("--mongo", dest="mongo_executable", metavar="PATH",
- help="The path to the mongo shell executable for resmoke.py to use.")
+ parser.add_argument("--mongo", dest="mongo_executable", metavar="PATH",
+ help="The path to the mongo shell executable for resmoke.py to use.")
- parser.add_option("--mongod", dest="mongod_executable", metavar="PATH",
- help="The path to the mongod executable for resmoke.py to use.")
+ parser.add_argument("--mongod", dest="mongod_executable", metavar="PATH",
+ help="The path to the mongod executable for resmoke.py to use.")
- parser.add_option(
+ parser.add_argument(
"--mongodSetParameters", dest="mongod_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongod processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
- parser.add_option("--mongos", dest="mongos_executable", metavar="PATH",
- help="The path to the mongos executable for resmoke.py to use.")
+ parser.add_argument("--mongos", dest="mongos_executable", metavar="PATH",
+ help="The path to the mongos executable for resmoke.py to use.")
- parser.add_option(
+ parser.add_argument(
"--mongosSetParameters", dest="mongos_set_parameters",
metavar="{key1: value1, key2: value2, ..., keyN: valueN}",
help=("Passes one or more --setParameter options to all mongos processes"
" started by resmoke.py. The argument is specified as bracketed YAML -"
" i.e. JSON with support for single quoted and unquoted keys."))
- parser.add_option("--nojournal", action="store_true", dest="no_journal",
- help="Disables journaling for all mongod's.")
+ parser.add_argument("--nojournal", action="store_true", dest="no_journal",
+ help="Disables journaling for all mongod's.")
- parser.add_option("--numClientsPerFixture", type="int", dest="num_clients_per_fixture",
- help="Number of clients running tests per fixture.")
+ parser.add_argument("--numClientsPerFixture", type=int, dest="num_clients_per_fixture",
+ help="Number of clients running tests per fixture.")
- parser.add_option("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT",
- help="Writes a JSON file with performance test results.")
+ parser.add_argument("--perfReportFile", dest="perf_report_file", metavar="PERF_REPORT",
+ help="Writes a JSON file with performance test results.")
- parser.add_option(
+ parser.add_argument(
"--shellConnString", dest="shell_conn_string", metavar="CONN_STRING",
help="Overrides the default fixture and connects with a mongodb:// connection"
" string to an existing MongoDB cluster instead. This is useful for"
" connecting to a MongoDB deployment started outside of resmoke.py including"
" one running in a debugger.")
- parser.add_option(
+ parser.add_argument(
"--shellPort", dest="shell_port", metavar="PORT",
help="Convenience form of --shellConnString for connecting to an"
" existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
" This is useful for connecting to a server running in a debugger.")
- parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N",
- help="Repeats the given suite(s) N times, or until one fails.")
+ parser.add_argument("--repeat", "--repeatSuites", type=int, dest="repeat_suites", metavar="N",
+ help="Repeats the given suite(s) N times, or until one fails.")
- parser.add_option(
- "--repeatTests", type="int", dest="repeat_tests", metavar="N",
+ parser.add_argument(
+ "--repeatTests", type=int, dest="repeat_tests", metavar="N",
help="Repeats the tests inside each suite N times. This applies to tests"
" defined in the suite configuration as well as tests defined on the command"
" line.")
- parser.add_option(
- "--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
+ parser.add_argument(
+ "--repeatTestsMax", type=int, dest="repeat_tests_max", metavar="N",
help="Repeats the tests inside each suite no more than N time when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
- parser.add_option(
- "--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
+ parser.add_argument(
+ "--repeatTestsMin", type=int, dest="repeat_tests_min", metavar="N",
help="Repeats the tests inside each suite at least N times when"
" --repeatTestsSecs is specified. This applies to tests defined in the suite"
" configuration as well as tests defined on the command line.")
- parser.add_option(
- "--repeatTestsSecs", type="float", dest="repeat_tests_secs", metavar="SECONDS",
+ parser.add_argument(
+ "--repeatTestsSecs", type=float, dest="repeat_tests_secs", metavar="SECONDS",
help="Repeats the tests inside each suite this amount of time. Note that"
" this option is mutually exclusive with --repeatTests. This applies to"
" tests defined in the suite configuration as well as tests defined on the"
" command line.")
- parser.add_option(
- "--reportFailureStatus", type="choice", action="store", dest="report_failure_status",
+ parser.add_argument(
+ "--reportFailureStatus", action="store", dest="report_failure_status",
choices=("fail", "silentfail"), metavar="STATUS",
help="Controls if the test failure status should be reported as failed"
" or be silently ignored (STATUS=silentfail). Dynamic test failures will"
- " never be silently ignored. Defaults to STATUS=%default.")
+ " never be silently ignored. Defaults to STATUS=%%default.")
- parser.add_option("--reportFile", dest="report_file", metavar="REPORT",
- help="Writes a JSON file with test status and timing information.")
+ parser.add_argument("--reportFile", dest="report_file", metavar="REPORT",
+ help="Writes a JSON file with test status and timing information.")
- parser.add_option(
- "--seed", type="int", dest="seed", metavar="SEED",
+ parser.add_argument(
+ "--seed", type=int, dest="seed", metavar="SEED",
help=("Seed for the random number generator. Useful in combination with the"
" --shuffle option for producing a consistent test execution order."))
- parser.add_option("--serviceExecutor", dest="service_executor", metavar="EXECUTOR",
- help="The service executor used by jstests")
+ parser.add_argument("--serviceExecutor", dest="service_executor", metavar="EXECUTOR",
+ help="The service executor used by jstests")
- parser.add_option("--transportLayer", dest="transport_layer", metavar="TRANSPORT",
- help="The transport layer used by jstests")
+ parser.add_argument("--transportLayer", dest="transport_layer", metavar="TRANSPORT",
+ help="The transport layer used by jstests")
- parser.add_option("--shellReadMode", type="choice", action="store", dest="shell_read_mode",
- choices=("commands", "compatibility", "legacy"), metavar="READ_MODE",
- help="The read mode used by the mongo shell.")
+ parser.add_argument("--shellReadMode", action="store", dest="shell_read_mode",
+ choices=("commands", "compatibility", "legacy"), metavar="READ_MODE",
+ help="The read mode used by the mongo shell.")
- parser.add_option("--shellWriteMode", type="choice", action="store", dest="shell_write_mode",
- choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE",
- help="The write mode used by the mongo shell.")
+ parser.add_argument("--shellWriteMode", action="store", dest="shell_write_mode",
+ choices=("commands", "compatibility", "legacy"), metavar="WRITE_MODE",
+ help="The write mode used by the mongo shell.")
- parser.add_option(
+ parser.add_argument(
"--shuffle", action="store_const", const="on", dest="shuffle",
help=("Randomizes the order in which tests are executed. This is equivalent"
" to specifying --shuffleMode=on."))
- parser.add_option(
- "--shuffleMode", type="choice", action="store", dest="shuffle",
- choices=("on", "off", "auto"), metavar="ON|OFF|AUTO",
+ parser.add_argument(
+ "--shuffleMode", action="store", dest="shuffle", choices=("on", "off",
+ "auto"), metavar="ON|OFF|AUTO",
help=("Controls whether to randomize the order in which tests are executed."
" Defaults to auto when not supplied. auto enables randomization in"
" all cases except when the number of jobs requested is 1."))
- parser.add_option(
- "--staggerJobs", type="choice", action="store", dest="stagger_jobs", choices=("on", "off"),
+ parser.add_argument(
+ "--staggerJobs", action="store", dest="stagger_jobs", choices=("on", "off"),
metavar="ON|OFF", help=("Enables or disables the stagger of launching resmoke jobs."
- " Defaults to %default."))
+ " Defaults to %%default."))
- parser.add_option(
- "--majorityReadConcern", type="choice", action="store", dest="majority_read_concern",
- choices=("on",
- "off"), metavar="ON|OFF", help=("Enable or disable majority read concern support."
- " Defaults to %default."))
+ parser.add_argument(
+ "--majorityReadConcern", action="store", dest="majority_read_concern", choices=("on",
+ "off"),
+ metavar="ON|OFF", help=("Enable or disable majority read concern support."
+ " Defaults to %%default."))
- parser.add_option("--flowControl", type="choice", action="store", dest="flow_control",
- choices=("on",
- "off"), metavar="ON|OFF", help=("Enable or disable flow control."))
+ parser.add_argument("--flowControl", action="store", dest="flow_control", choices=("on", "off"),
+ metavar="ON|OFF", help=("Enable or disable flow control."))
- parser.add_option("--flowControlTicketOverride", type="int", action="store",
- dest="flow_control_tickets", metavar="TICKET_OVERRIDE",
- help=("Number of tickets available for flow control."))
+ parser.add_argument("--flowControlTicketOverride", type=int, action="store",
+ dest="flow_control_tickets", metavar="TICKET_OVERRIDE",
+ help=("Number of tickets available for flow control."))
- parser.add_option("--storageEngine", dest="storage_engine", metavar="ENGINE",
- help="The storage engine used by dbtests and jstests.")
+ parser.add_argument("--storageEngine", dest="storage_engine", metavar="ENGINE",
+ help="The storage engine used by dbtests and jstests.")
- parser.add_option(
+ parser.add_argument(
"--storageEngineCacheSizeGB", dest="storage_engine_cache_size_gb", metavar="CONFIG",
help="Sets the storage engine cache size configuration"
" setting for all mongod's.")
- parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS",
- help="A YAML file that associates tests and tags.")
+ parser.add_argument("--tagFile", dest="tag_file", metavar="OPTIONS",
+ help="A YAML file that associates tests and tags.")
- parser.add_option("--wiredTigerCollectionConfigString", dest="wt_coll_config", metavar="CONFIG",
- help="Sets the WiredTiger collection configuration setting for all mongod's.")
+ parser.add_argument(
+ "--wiredTigerCollectionConfigString", dest="wt_coll_config", metavar="CONFIG",
+ help="Sets the WiredTiger collection configuration setting for all mongod's.")
- parser.add_option("--wiredTigerEngineConfigString", dest="wt_engine_config", metavar="CONFIG",
- help="Sets the WiredTiger engine configuration setting for all mongod's.")
+ parser.add_argument("--wiredTigerEngineConfigString", dest="wt_engine_config", metavar="CONFIG",
+ help="Sets the WiredTiger engine configuration setting for all mongod's.")
- parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG",
- help="Sets the WiredTiger index configuration setting for all mongod's.")
+ parser.add_argument("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG",
+ help="Sets the WiredTiger index configuration setting for all mongod's.")
- parser.add_option(
+ parser.add_argument(
"--executor", dest="executor_file",
help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
" to run a particular test under a particular suite configuration.")
- evergreen_options = optparse.OptionGroup(
- parser, title=_EVERGREEN_OPTIONS_TITLE,
+ evergreen_options = parser.add_argument_group(
+ title=_EVERGREEN_ARGUMENT_TITLE,
description=("Options used to propagate information about the Evergreen task running this"
" script."))
- parser.add_option_group(evergreen_options)
- evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID",
- help="Sets the build ID of the task.")
+ evergreen_options.add_argument("--buildId", dest="build_id", metavar="BUILD_ID",
+ help="Sets the build ID of the task.")
- evergreen_options.add_option(
+ evergreen_options.add_argument(
"--distroId", dest="distro_id", metavar="DISTRO_ID",
help=("Sets the identifier for the Evergreen distro running the"
" tests."))
- evergreen_options.add_option(
- "--executionNumber", type="int", dest="execution_number", metavar="EXECUTION_NUMBER",
+ evergreen_options.add_argument(
+ "--executionNumber", type=int, dest="execution_number", metavar="EXECUTION_NUMBER",
help=("Sets the number for the Evergreen execution running the"
" tests."))
- evergreen_options.add_option(
+ evergreen_options.add_argument(
"--gitRevision", dest="git_revision", metavar="GIT_REVISION",
help=("Sets the git revision for the Evergreen task running the"
" tests."))
# We intentionally avoid adding a new command line option that starts with --suite so it doesn't
# become ambiguous with the --suites option and break how engineers run resmoke.py locally.
- evergreen_options.add_option(
+ evergreen_options.add_argument(
"--originSuite", dest="origin_suite", metavar="SUITE",
help=("Indicates the name of the test suite prior to the"
" evergreen_generate_resmoke_tasks.py script splitting it"
" up."))
- evergreen_options.add_option(
+ evergreen_options.add_argument(
"--patchBuild", action="store_true", dest="patch_build",
help=("Indicates that the Evergreen task running the tests is a"
" patch build."))
- evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME",
- help=("Sets the name of the Evergreen project running the tests."))
+ evergreen_options.add_argument(
+ "--projectName", dest="project_name", metavar="PROJECT_NAME",
+ help=("Sets the name of the Evergreen project running the tests."))
- evergreen_options.add_option("--revisionOrderId", dest="revision_order_id",
- metavar="REVISION_ORDER_ID",
- help="Sets the chronological order number of this commit.")
+ evergreen_options.add_argument("--revisionOrderId", dest="revision_order_id",
+ metavar="REVISION_ORDER_ID",
+ help="Sets the chronological order number of this commit.")
- evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME",
- help="Sets the name of the Evergreen task running the tests.")
+ evergreen_options.add_argument("--taskName", dest="task_name", metavar="TASK_NAME",
+ help="Sets the name of the Evergreen task running the tests.")
- evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID",
- help="Sets the Id of the Evergreen task running the tests.")
+ evergreen_options.add_argument("--taskId", dest="task_id", metavar="TASK_ID",
+ help="Sets the Id of the Evergreen task running the tests.")
- evergreen_options.add_option(
+ evergreen_options.add_argument(
"--variantName", dest="variant_name", metavar="VARIANT_NAME",
help=("Sets the name of the Evergreen build variant running the"
" tests."))
- evergreen_options.add_option("--versionId", dest="version_id", metavar="VERSION_ID",
- help="Sets the version ID of the task.")
+ evergreen_options.add_argument("--versionId", dest="version_id", metavar="VERSION_ID",
+ help="Sets the version ID of the task.")
- benchmark_options = optparse.OptionGroup(
- parser, title="Benchmark/Benchrun test options",
+ benchmark_options = parser.add_argument_group(
+ title="Benchmark/Benchrun test options",
description="Options for running Benchmark/Benchrun tests")
- parser.add_option_group(benchmark_options)
+ benchmark_options.add_argument("--benchmarkFilter", type=str, dest="benchmark_filter",
+ metavar="BENCHMARK_FILTER",
+ help="Regex to filter Google benchmark tests to run.")
- benchmark_options.add_option("--benchmarkFilter", type="string", dest="benchmark_filter",
- metavar="BENCHMARK_FILTER",
- help="Regex to filter Google benchmark tests to run.")
-
- benchmark_options.add_option(
- "--benchmarkListTests", dest="benchmark_list_tests", action="store_true",
- metavar="BENCHMARK_LIST_TESTS",
+ benchmark_options.add_argument(
+ "--benchmarkListTests",
+ dest="benchmark_list_tests",
+ action="store_true",
+ # metavar="BENCHMARK_LIST_TESTS",
help=("Lists all Google benchmark test configurations in each"
" test file."))
benchmark_min_time_help = (
"Minimum time to run each benchmark/benchrun test for. Use this option instead of "
"--benchmarkRepetitions to make a test run for a longer or shorter duration.")
- benchmark_options.add_option("--benchmarkMinTimeSecs", type="int",
- dest="benchmark_min_time_secs", metavar="BENCHMARK_MIN_TIME",
- help=benchmark_min_time_help)
+ benchmark_options.add_argument("--benchmarkMinTimeSecs", type=int,
+ dest="benchmark_min_time_secs", metavar="BENCHMARK_MIN_TIME",
+ help=benchmark_min_time_help)
benchmark_repetitions_help = (
"Set --benchmarkRepetitions=1 if you'd like to run the benchmark/benchrun tests only once."
" By default, each test is run multiple times to provide statistics on the variance"
" between runs; use --benchmarkMinTimeSecs if you'd like to run a test for a longer or"
" shorter duration.")
- benchmark_options.add_option("--benchmarkRepetitions", type="int", dest="benchmark_repetitions",
- metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help)
+ benchmark_options.add_argument("--benchmarkRepetitions", type=int, dest="benchmark_repetitions",
+ metavar="BENCHMARK_REPETITIONS", help=benchmark_repetitions_help)
- parser.set_defaults(dry_run="off", find_suites=False, list_suites=False, logger_file="console",
- shuffle="auto", stagger_jobs="off", suite_files="with_server",
- majority_read_concern="on")
- return parser
+def _add_list_suites(subparsers):
+ """Create and add the parser for the list-suites subcommand."""
+ parser = subparsers.add_parser("list-suites",
+ help="Lists the names of the suites available to execute.")
+
+ parser.add_argument(
+ "--log", dest="logger_file", metavar="LOGGER",
+ help=("A YAML file that specifies the logging configuration. If the file is"
+ " located in the resmokeconfig/suites/ directory, then the basename"
+ " without the .yml extension can be specified, e.g. 'console'."))
+ parser.set_defaults(logger_file="console")
-def to_local_args(args=None): # pylint: disable=too-many-branches,too-many-locals
- """
- Return a command line invocation for resmoke.py suitable for being run outside of Evergreen.
+def _add_find_suites(subparsers):
+ """Create and add the parser for the find-suites subcommand."""
+ parser = subparsers.add_parser(
+ "find-suites", help="Lists the names of the suites that will execute the specified tests.")
- This function parses the 'args' list of command line arguments, removes any Evergreen-centric
- options, and returns a new list of command line arguments.
- """
+ parser.add_argument("test_files", metavar="TEST_FILES", nargs="*",
+ help="Explicit test files to run")
- if args is None:
- args = sys.argv[1:]
+ parser.add_argument(
+ "--log", dest="logger_file", metavar="LOGGER",
+ help=("A YAML file that specifies the logging configuration. If the file is"
+ " located in the resmokeconfig/suites/ directory, then the basename"
+ " without the .yml extension can be specified, e.g. 'console'."))
+ parser.set_defaults(logger_file="console")
+ parser.add_argument(
+ "--suites", dest="suite_files", metavar="SUITE1,SUITE2", required=True,
+ help=("Comma separated list of YAML files that each specify the configuration"
+ " of a suite. If the file is located in the resmokeconfig/suites/"
+ " directory, then the basename without the .yml extension can be"
+ " specified, e.g. 'core'. If a list of files is passed in as"
+ " positional arguments, they will be run using the suites'"
+ " configurations."))
+
+
+# def to_local_args(args=None): # pylint: disable=too-many-branches,too-many-locals
+# """
+# Return a command line invocation for resmoke.py suitable for being run outside of Evergreen.
+# This function parses the 'args' list of command line arguments, removes any Evergreen-centric
+# options, and returns a new list of command line arguments.
+# """
+
+# if args is None:
+# args = sys.argv[1:]
+
+# parser = _make_parser()
+
+# # We call optparse.OptionParser.parse_args() with a new instance of optparse.Values to avoid
+# # having the default values filled in. This makes it so 'options' only contains command line
+# # options that were explicitly specified.
+# options, extra_args = parser.parse_args(args=args, values=optparse.Values())
+
+# # If --originSuite was specified, then we replace the value of --suites with it. This is done to
+# # avoid needing to have engineers learn about the test suites generated by the
+# # evergreen_generate_resmoke_tasks.py script.
+# origin_suite = getattr(options, "origin_suite", None)
+# if origin_suite is not None:
+# setattr(options, "suite_files", origin_suite)
+
+# # optparse.OptionParser doesn't offer a public and/or documented method for getting all of the
+# # options. Given that the optparse module is deprecated, it is unlikely for the
+# # _get_all_options() method to ever be removed or renamed.
+# all_options = parser._get_all_options() # pylint: disable=protected-access
+
+# options_by_dest = {}
+# for option in all_options:
+# options_by_dest[option.dest] = option
+
+# suites_arg = None
+# storage_engine_arg = None
+# other_local_args = []
+
+# options_to_ignore = {
+# "--archiveLimitMb",
+# "--archiveLimitTests",
+# "--buildloggerUrl",
+# "--log",
+# "--perfReportFile",
+# "--reportFailureStatus",
+# "--reportFile",
+# "--staggerJobs",
+# "--tagFile",
+# }
+
+# def format_option(option_name, option_value):
+# """
+# Return <option_name>=<option_value>.
+# This function assumes that 'option_name' is always "--" prefix and isn't "-" prefixed.
+# """
+# return "%s=%s" % (option_name, option_value)
+
+# for option_dest in sorted(vars(options)):
+# option_value = getattr(options, option_dest)
+# option = options_by_dest[option_dest]
+# option_name = option.get_opt_string()
+
+# if option_name in options_to_ignore:
+# continue
+
+# option_group = parser.get_option_group(option_name)
+# if option_group is not None and option_group.title == _EVERGREEN_OPTIONS_TITLE:
+# continue
+
+# if option.takes_value():
+# if option.action == "append":
+# args = [format_option(option_name, elem) for elem in option_value]
+# other_local_args.extend(args)
+# else:
+# arg = format_option(option_name, option_value)
+
+# # We track the value for the --suites and --storageEngine command line options
+# # separately in order to more easily sort them to the front.
+# if option_dest == "suite_files":
+# suites_arg = arg
+# elif option_dest == "storage_engine":
+# storage_engine_arg = arg
+# else:
+# other_local_args.append(arg)
+# else:
+# other_local_args.append(option_name)
+
+# return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None
+# ] + other_local_args + extra_args
+
+
+def _parse(sys_args):
+ """Parse the CLI args."""
+
+ # Split out this function for easier testing.
parser = _make_parser()
+ parsed_args = parser.parse_args(sys_args)
- # We call optparse.OptionParser.parse_args() with a new instance of optparse.Values to avoid
- # having the default values filled in. This makes it so 'options' only contains command line
- # options that were explicitly specified.
- options, extra_args = parser.parse_args(args=args, values=optparse.Values())
-
- # If --originSuite was specified, then we replace the value of --suites with it. This is done to
- # avoid needing to have engineers learn about the test suites generated by the
- # evergreen_generate_resmoke_tasks.py script.
- origin_suite = getattr(options, "origin_suite", None)
- if origin_suite is not None:
- setattr(options, "suite_files", origin_suite)
-
- # optparse.OptionParser doesn't offer a public and/or documented method for getting all of the
- # options. Given that the optparse module is deprecated, it is unlikely for the
- # _get_all_options() method to ever be removed or renamed.
- all_options = parser._get_all_options() # pylint: disable=protected-access
-
- options_by_dest = {}
- for option in all_options:
- options_by_dest[option.dest] = option
-
- suites_arg = None
- storage_engine_arg = None
- other_local_args = []
-
- options_to_ignore = set([
- "--archiveFile",
- "--archiveLimitMb",
- "--archiveLimitTests",
- "--buildloggerUrl",
- "--log",
- "--perfReportFile",
- "--reportFailureStatus",
- "--reportFile",
- "--staggerJobs",
- "--tagFile",
- ])
-
- def format_option(option_name, option_value):
- """
- Return <option_name>=<option_value>.
-
- This function assumes that 'option_name' is always "--" prefix and isn't "-" prefixed.
- """
- return "%s=%s" % (option_name, option_value)
-
- for option_dest in sorted(vars(options)):
- option_value = getattr(options, option_dest)
- option = options_by_dest[option_dest]
- option_name = option.get_opt_string()
-
- if option_name in options_to_ignore:
- continue
-
- option_group = parser.get_option_group(option_name)
- if option_group is not None and option_group.title == _EVERGREEN_OPTIONS_TITLE:
- continue
-
- if option.takes_value():
- if option.action == "append":
- args = [format_option(option_name, elem) for elem in option_value]
- other_local_args.extend(args)
- else:
- arg = format_option(option_name, option_value)
+ return (parser, parsed_args)
- # We track the value for the --suites and --storageEngine command line options
- # separately in order to more easily sort them to the front.
- if option_dest == "suite_files":
- suites_arg = arg
- elif option_dest == "storage_engine":
- storage_engine_arg = arg
- else:
- other_local_args.append(arg)
- else:
- other_local_args.append(option_name)
- return [arg for arg in (suites_arg, storage_engine_arg) if arg is not None
- ] + other_local_args + extra_args
+def parse_command_line(sys_args, **kwargs):
+ """Parse the command line arguments passed to resmoke.py and return the subcommand object to execute."""
+ parser, parsed_args = _parse(sys_args)
+ def create_subcommand(parser, parsed_args, **kwargs):
+ """Create a subcommand object based on args passed into resmoke.py."""
-def parse_command_line():
- """Parse the command line arguments passed to resmoke.py."""
- parser = _make_parser()
- options, args = parser.parse_args()
-
- _validate_options(parser, options, args)
- _update_config_vars(options)
- _validate_config(parser)
-
- return ResmokeConfig(list_suites=options.list_suites, find_suites=options.find_suites,
- dry_run=options.dry_run, suite_files=options.suite_files.split(","),
- test_files=args, logging_config=_get_logging_config(options.logger_file))
-
-
-def _validate_options(parser, options, args):
- """Do preliminary validation on the options and error on any invalid options."""
-
- if options.shell_port is not None and options.shell_conn_string is not None:
- parser.error("Cannot specify both `shellPort` and `shellConnString`")
-
- if options.executor_file:
- parser.error("--executor is superseded by --suites; specify --suites={} {} to run the"
- " test(s) under those suite configuration(s)".format(
- options.executor_file, " ".join(args)))
-
-
-def _validate_config(parser):
- """Do validation on the config settings."""
-
- if _config.REPEAT_TESTS_MAX:
- if not _config.REPEAT_TESTS_SECS:
- parser.error("Must specify --repeatTestsSecs with --repeatTestsMax")
-
- if _config.REPEAT_TESTS_MIN > _config.REPEAT_TESTS_MAX:
- parser.error("--repeatTestsSecsMin > --repeatTestsMax")
-
- if _config.REPEAT_TESTS_MIN and not _config.REPEAT_TESTS_SECS:
- parser.error("Must specify --repeatTestsSecs with --repeatTestsMin")
-
- if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS:
- parser.error("Cannot specify --repeatTests and --repeatTestsSecs")
-
-
-def validate_benchmark_options():
- """Error out early if any options are incompatible with benchmark test suites.
-
- :return: None
- """
-
- if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1 or _config.REPEAT_TESTS_SECS:
- raise optparse.OptionValueError(
- "--repeatSuites/--repeatTests cannot be used with benchmark tests. "
- "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark "
- "configuration.")
-
- if _config.JOBS > 1:
- raise optparse.OptionValueError(
- "--jobs=%d cannot be used for benchmark tests. Parallel jobs affect CPU cache access "
- "patterns and cause additional context switching, which lead to inaccurate benchmark "
- "results. Please use --jobs=1" % _config.JOBS)
-
-
-def _update_config_vars(values): # pylint: disable=too-many-statements
- """Update the variables of the config module."""
-
- config = _config.DEFAULTS.copy()
-
- # Override `config` with values from command line arguments.
- cmdline_vars = vars(values)
- for cmdline_key in cmdline_vars:
- if cmdline_key not in _config.DEFAULTS:
- # Ignore options that don't map to values in config.py
- continue
- if cmdline_vars[cmdline_key] is not None:
- config[cmdline_key] = cmdline_vars[cmdline_key]
-
- _config.ARCHIVE_FILE = config.pop("archive_file")
- _config.ARCHIVE_LIMIT_MB = config.pop("archive_limit_mb")
- _config.ARCHIVE_LIMIT_TESTS = config.pop("archive_limit_tests")
- _config.BASE_PORT = int(config.pop("base_port"))
- _config.BUILDLOGGER_URL = config.pop("buildlogger_url")
- _config.DBPATH_PREFIX = _expand_user(config.pop("dbpath_prefix"))
- _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest_executable"))
- _config.DRY_RUN = config.pop("dry_run")
- # EXCLUDE_WITH_ANY_TAGS will always contain the implicitly defined EXCLUDED_TAG.
- _config.EXCLUDE_WITH_ANY_TAGS = [_config.EXCLUDED_TAG]
- _config.EXCLUDE_WITH_ANY_TAGS.extend(
- utils.default_if_none(_tags_from_list(config.pop("exclude_with_any_tags")), []))
- _config.FAIL_FAST = not config.pop("continue_on_failure")
- _config.FLOW_CONTROL = config.pop("flow_control")
- _config.FLOW_CONTROL_TICKETS = config.pop("flow_control_tickets")
- _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("include_with_any_tags"))
- _config.GENNY_EXECUTABLE = _expand_user(config.pop("genny_executable"))
- _config.JOBS = config.pop("jobs")
- _config.MAJORITY_READ_CONCERN = config.pop("majority_read_concern") == "on"
- _config.MONGO_EXECUTABLE = _expand_user(config.pop("mongo_executable"))
- _config.MONGOD_EXECUTABLE = _expand_user(config.pop("mongod_executable"))
- _config.MONGOD_SET_PARAMETERS = config.pop("mongod_set_parameters")
- _config.MONGOS_EXECUTABLE = _expand_user(config.pop("mongos_executable"))
- _config.MONGOS_SET_PARAMETERS = config.pop("mongos_set_parameters")
- _config.NO_JOURNAL = config.pop("no_journal")
- _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")
- _config.PERF_REPORT_FILE = config.pop("perf_report_file")
- _config.RANDOM_SEED = config.pop("seed")
- _config.REPEAT_SUITES = config.pop("repeat_suites")
- _config.REPEAT_TESTS = config.pop("repeat_tests")
- _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max")
- _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min")
- _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs")
- _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
- _config.REPORT_FILE = config.pop("report_file")
- _config.SERVICE_EXECUTOR = config.pop("service_executor")
- _config.SHELL_READ_MODE = config.pop("shell_read_mode")
- _config.SHELL_WRITE_MODE = config.pop("shell_write_mode")
- _config.SPAWN_USING = config.pop("spawn_using")
- _config.STAGGER_JOBS = config.pop("stagger_jobs") == "on"
- _config.STORAGE_ENGINE = config.pop("storage_engine")
- _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storage_engine_cache_size_gb")
- _config.TAG_FILE = config.pop("tag_file")
- _config.TRANSPORT_LAYER = config.pop("transport_layer")
-
- # Evergreen options.
- _config.EVERGREEN_BUILD_ID = config.pop("build_id")
- _config.EVERGREEN_DISTRO_ID = config.pop("distro_id")
- _config.EVERGREEN_EXECUTION = config.pop("execution_number")
- _config.EVERGREEN_PATCH_BUILD = config.pop("patch_build")
- _config.EVERGREEN_PROJECT_NAME = config.pop("project_name")
- _config.EVERGREEN_REVISION = config.pop("git_revision")
- _config.EVERGREEN_REVISION_ORDER_ID = config.pop("revision_order_id")
- _config.EVERGREEN_TASK_ID = config.pop("task_id")
- _config.EVERGREEN_TASK_NAME = config.pop("task_name")
- _config.EVERGREEN_VARIANT_NAME = config.pop("variant_name")
- _config.EVERGREEN_VERSION_ID = config.pop("version_id")
-
- # Wiredtiger options.
- _config.WT_COLL_CONFIG = config.pop("wt_coll_config")
- _config.WT_ENGINE_CONFIG = config.pop("wt_engine_config")
- _config.WT_INDEX_CONFIG = config.pop("wt_index_config")
-
- # Benchmark/Benchrun options.
- _config.BENCHMARK_FILTER = config.pop("benchmark_filter")
- _config.BENCHMARK_LIST_TESTS = config.pop("benchmark_list_tests")
- benchmark_min_time = config.pop("benchmark_min_time_secs")
- if benchmark_min_time is not None:
- _config.BENCHMARK_MIN_TIME = datetime.timedelta(seconds=benchmark_min_time)
- _config.BENCHMARK_REPETITIONS = config.pop("benchmark_repetitions")
-
- shuffle = config.pop("shuffle")
- if shuffle == "auto":
- # If the user specified a value for --jobs > 1 (or -j > 1), then default to randomize
- # the order in which tests are executed. This is because with multiple threads the tests
- # wouldn't run in a deterministic order anyway.
- _config.SHUFFLE = _config.JOBS > 1
- else:
- _config.SHUFFLE = shuffle == "on"
-
- conn_string = config.pop("shell_conn_string")
- port = config.pop("shell_port")
-
- if port is not None:
- conn_string = "mongodb://localhost:" + port
-
- if conn_string is not None:
- # The --shellConnString command line option must be a MongoDB connection URI, which means it
- # must specify the mongodb:// or mongodb+srv:// URI scheme. pymongo.uri_parser.parse_uri()
- # raises an exception if the connection string specified isn't considered a valid MongoDB
- # connection URI.
- pymongo.uri_parser.parse_uri(conn_string)
- _config.SHELL_CONN_STRING = conn_string
-
- if config:
- raise optparse.OptionValueError("Unknown option(s): %s" % (list(config.keys())))
-
-
-def _get_logging_config(pathname):
- """Read YAML configuration from 'pathname' how to log tests and fixtures."""
-
- # Named loggers are specified as the basename of the file, without the .yml extension.
- if not utils.is_yaml_file(pathname) and not os.path.dirname(pathname):
- if pathname not in resmokeconfig.NAMED_LOGGERS:
- raise optparse.OptionValueError("Unknown logger '%s'" % pathname)
- pathname = resmokeconfig.NAMED_LOGGERS[pathname] # Expand 'pathname' to full path.
-
- if not utils.is_yaml_file(pathname) or not os.path.isfile(pathname):
- raise optparse.OptionValueError("Expected a logger YAML config, but got '%s'" % pathname)
-
- return utils.load_yaml_file(pathname).pop("logging")
-
-
-def _expand_user(pathname):
- """Provide wrapper around os.path.expanduser() to do nothing when given None."""
- if pathname is None:
- return None
- return os.path.expanduser(pathname)
-
-
-def _tags_from_list(tags_list):
- """Return the list of tags from a list of tag parameter values.
-
- Each parameter value in the list may be a list of comma separated tags, with empty strings
- ignored.
- """
- tags = []
- if tags_list is not None:
- for tag in tags_list:
- tags.extend([t for t in tag.split(",") if t != ""])
- return tags
- return None
+ subcommand = parsed_args.command
+ subcommand_obj = None
+ if subcommand in ('find-suites', 'list-suites', 'run'):
+ configure_resmoke.validate_and_update_config(parser, parsed_args)
+ if _config.EVERGREEN_TASK_ID is not None:
+ subcommand_obj = commands.run.TestRunnerEvg(subcommand, **kwargs)
+ else:
+ subcommand_obj = commands.run.TestRunner(subcommand, **kwargs)
+
+ if subcommand_obj is None:
+ raise RuntimeError(
+ f"Resmoke configuration has invalid subcommand: {subcommand}. Try '--help'")
+
+ return subcommand_obj
+
+ return create_subcommand(parser, parsed_args, **kwargs)
diff --git a/buildscripts/resmokelib/testing/testcases/benchmark_test.py b/buildscripts/resmokelib/testing/testcases/benchmark_test.py
index e7760799e42..9902bd503ef 100644
--- a/buildscripts/resmokelib/testing/testcases/benchmark_test.py
+++ b/buildscripts/resmokelib/testing/testcases/benchmark_test.py
@@ -2,7 +2,6 @@
from buildscripts.resmokelib import config as _config
from buildscripts.resmokelib import core
-from buildscripts.resmokelib import parser
from buildscripts.resmokelib import utils
from buildscripts.resmokelib.testing.testcases import interface
@@ -16,12 +15,30 @@ class BenchmarkTestCase(interface.ProcessTestCase):
"""Initialize the BenchmarkTestCase with the executable to run."""
interface.ProcessTestCase.__init__(self, logger, "Benchmark test", program_executable)
- parser.validate_benchmark_options()
+ self.validate_benchmark_options()
self.bm_executable = program_executable
self.suite_bm_options = program_options
self.bm_options = {}
+ def validate_benchmark_options(self): # pylint: disable=no-self-use
+ """Error out early if any options are incompatible with benchmark test suites.
+
+ :return: None
+ """
+
+ if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1 or _config.REPEAT_TESTS_SECS:
+ raise ValueError(
+ "--repeatSuites/--repeatTests cannot be used with benchmark tests. "
+ "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark "
+ "configuration.")
+
+ if _config.JOBS > 1:
+ raise ValueError(
+ "--jobs=%d cannot be used for benchmark tests. Parallel jobs affect CPU cache access "
+ "patterns and cause additional context switching, which lead to inaccurate benchmark "
+ "results. Please use --jobs=1" % _config.JOBS)
+
def configure(self, fixture, *args, **kwargs):
"""Configure BenchmarkTestCase."""
interface.ProcessTestCase.configure(self, fixture, *args, **kwargs)
diff --git a/buildscripts/tests/resmokelib/test_parser.py b/buildscripts/tests/resmokelib/test_parser.py
index 712e1aef3f2..2e3dc9dddfe 100644
--- a/buildscripts/tests/resmokelib/test_parser.py
+++ b/buildscripts/tests/resmokelib/test_parser.py
@@ -7,6 +7,7 @@ from buildscripts.resmokelib import parser as _parser
# pylint: disable=missing-docstring
+@unittest.skip("TODO: SERVER-47611")
class TestLocalCommandLine(unittest.TestCase):
"""Unit tests for the to_local_args() function."""
@@ -245,3 +246,57 @@ class TestLocalCommandLine(unittest.TestCase):
])
self.assertEqual(cmdline, ["--suites=my_suite", "--storageEngine=my_storage_engine"])
+
+
+class TestParseArgs(unittest.TestCase):
+ """Unit tests for the parse() function."""
+
+ def test_files_at_end(self):
+ _, args = _parser._parse([ # pylint: disable=protected-access
+ "run",
+ "--suites=my_suite1,my_suite2",
+ "test_file1.js",
+ "test_file2.js",
+ "test_file3.js",
+ ])
+
+ self.assertEqual(args.test_files, [
+ "test_file1.js",
+ "test_file2.js",
+ "test_file3.js",
+ ])
+ # suites get split up when config.py gets populated
+ self.assertEqual(args.suite_files, "my_suite1,my_suite2")
+
+ def test_files_in_the_middle(self):
+ _, args = _parser._parse([ # pylint: disable=protected-access
+ "run",
+ "--storageEngine=my_storage_engine",
+ "test_file1.js",
+ "test_file2.js",
+ "test_file3.js",
+ "--suites=my_suite1",
+ ])
+
+ self.assertEqual(args.test_files, [
+ "test_file1.js",
+ "test_file2.js",
+ "test_file3.js",
+ ])
+ self.assertEqual(args.suite_files, "my_suite1")
+
+
+class TestParseCommandLine(unittest.TestCase):
+ """Unit tests for the parse_command_line() function."""
+
+ def test_find_suites(self):
+ subcommand_obj = _parser.parse_command_line(['find-suites', '--suites=my_suite'])
+ self.assertTrue(hasattr(subcommand_obj, 'execute'))
+
+ def test_list_suites(self):
+ subcommand_obj = _parser.parse_command_line(['list-suites'])
+ self.assertTrue(hasattr(subcommand_obj, 'execute'))
+
+ def test_run(self):
+ subcommand_obj = _parser.parse_command_line(['run', '--suite=my_suite', 'my_test.js'])
+ self.assertTrue(hasattr(subcommand_obj, 'execute'))
diff --git a/buildscripts/tests/resmokelib/test_selector.py b/buildscripts/tests/resmokelib/test_selector.py
index a37147726ab..05e36da61e7 100644
--- a/buildscripts/tests/resmokelib/test_selector.py
+++ b/buildscripts/tests/resmokelib/test_selector.py
@@ -1,5 +1,6 @@
"""Unit tests for the buildscripts.resmokelib.selector module."""
+import sys
import fnmatch
import os.path
import unittest
@@ -515,7 +516,7 @@ class TestFilterTests(unittest.TestCase):
self.assertEqual(["dir/subdir1/test12.js", "dir/subdir3/a/test3a1.js"], selected)
def test_filter_temporarily_disabled_tests(self):
- parser.parse_command_line()
+ parser.parse_command_line(sys.argv[1:])
test_file_explorer = MockTestFileExplorer()
test_file_explorer.tags = {
"dir/subdir1/test11.js": ["tag1", "tag2", "__TEMPORARILY_DISABLED__"],
diff --git a/buildscripts/tests/test_burn_in_tests.py b/buildscripts/tests/test_burn_in_tests.py
index cf99b2a846c..380ab520b1a 100644
--- a/buildscripts/tests/test_burn_in_tests.py
+++ b/buildscripts/tests/test_burn_in_tests.py
@@ -440,7 +440,7 @@ class TestSetResmokeCmd(unittest.TestCase):
resmoke_cmds = under_test._set_resmoke_cmd(repeat_config, [])
self.assertListEqual(resmoke_cmds,
- [sys.executable, "buildscripts/resmoke.py", '--repeatSuites=2'])
+ [sys.executable, "buildscripts/resmoke.py", "run", '--repeatSuites=2'])
def test__set_resmoke_cmd_no_opts(self):
repeat_config = under_test.RepeatConfig()
@@ -627,7 +627,7 @@ class RunTests(unittest.TestCase):
@patch(ns('subprocess.check_call'))
def test_run_tests_no_tests(self, check_call_mock):
tests_by_task = {}
- resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"]
+ resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"]
under_test.run_tests(tests_by_task, resmoke_cmd)
@@ -637,7 +637,7 @@ class RunTests(unittest.TestCase):
def test_run_tests_some_test(self, check_call_mock):
n_tasks = 3
tests_by_task = create_tests_by_task_mock(n_tasks, 5)
- resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"]
+ resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"]
under_test.run_tests(tests_by_task, resmoke_cmd)
@@ -649,7 +649,7 @@ class RunTests(unittest.TestCase):
error_code = 42
n_tasks = 3
tests_by_task = create_tests_by_task_mock(n_tasks, 5)
- resmoke_cmd = ["python", "buildscripts/resmoke.py", "--continueOnFailure"]
+ resmoke_cmd = ["python", "buildscripts/resmoke.py", "run", "--continueOnFailure"]
check_call_mock.side_effect = subprocess.CalledProcessError(error_code, "err1")
exit_mock.side_effect = ValueError('exiting')
diff --git a/etc/drivers_nightly.yml b/etc/drivers_nightly.yml
index 83511382080..844f252e530 100644
--- a/etc/drivers_nightly.yml
+++ b/etc/drivers_nightly.yml
@@ -188,7 +188,7 @@ functions:
fi
${rlp_environment} \
- ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json
+ ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py run ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json
"cleanup environment":
command: shell.exec
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index d9f3f370b5c..eced82ced9e 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1251,7 +1251,7 @@ functions:
${san_symbolizer} \
${snmp_config_path} \
${resmoke_wrapper} \
- $python buildscripts/evergreen_run_tests.py \
+ $python buildscripts/resmoke.py run \
$resmoke_cmd_args \
$extra_args \
${test_flags} \
diff --git a/jstests/noPassthrough/libs/backup_restore.js b/jstests/noPassthrough/libs/backup_restore.js
index 37411b9d061..d15b30fe046 100644
--- a/jstests/noPassthrough/libs/backup_restore.js
+++ b/jstests/noPassthrough/libs/backup_restore.js
@@ -122,7 +122,7 @@ var BackupRestoreTest = function(options) {
function _fsmClient(host) {
// Launch FSM client
const suite = 'concurrency_replication_for_backup_restore';
- const resmokeCmd = 'python buildscripts/resmoke.py --shuffle --continueOnFailure' +
+ const resmokeCmd = 'python buildscripts/resmoke.py run --shuffle --continueOnFailure' +
' --repeat=99999 --mongo=' + MongoRunner.mongoShellPath +
' --shellConnString=mongodb://' + host + ' --suites=' + suite;
diff --git a/pytests/powertest.py b/pytests/powertest.py
index 32c6cbe551a..24b826e8963 100755
--- a/pytests/powertest.py
+++ b/pytests/powertest.py
@@ -1708,6 +1708,7 @@ def resmoke_client( # pylint: disable=too-many-arguments
log_output = ">> {} 2>&1".format(log_file) if log_file else ""
cmds = ("cd {}; "
"python buildscripts/resmoke.py"
+ "run"
" --mongo {}"
" --suites {}"
" --shellConnString mongodb://{}"
diff --git a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
index 49ff11d44f6..f8efe461c68 100644
--- a/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
+++ b/src/mongo/gotools/src/github.com/mongodb/mongo-tools/common.yml
@@ -350,7 +350,7 @@ functions:
cd test/qa-tests
chmod 400 jstests/libs/key*
- python buildscripts/resmoke.py --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+ python buildscripts/resmoke.py run --suite=${resmoke_suite} --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
"build tool":
command: shell.exec
@@ -1412,7 +1412,7 @@ tasks:
chmod +x ../../bin/*
mv ../../bin/* .
chmod 400 jstests/libs/key*
- python buildscripts/resmoke.py --suite=native_cert_ssl --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
+ python buildscripts/resmoke.py run --suite=native_cert_ssl --continueOnFailure --log=buildlogger --reportFile=../../report.json ${resmoke_args} --excludeWithAnyTags="${excludes}"
- name: qa-dump-restore-with-archiving-current
depends_on: