diff options
-rw-r--r-- | buildscripts/burn_in_tests.py | 2 | ||||
-rwxr-xr-x | buildscripts/evergreen_run_tests.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/config.py | 14 | ||||
-rw-r--r-- | buildscripts/resmokelib/parser.py | 17 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/executor.py | 19 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/report.py | 12 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/suite.py | 7 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/testcases/interface.py | 7 | ||||
-rw-r--r-- | etc/evergreen.yml | 4 |
9 files changed, 54 insertions, 31 deletions
diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py index 690303d3b12..cae3ee9baca 100644 --- a/buildscripts/burn_in_tests.py +++ b/buildscripts/burn_in_tests.py @@ -337,7 +337,7 @@ def main(): # If a resmoke.py command wasn't passed in, use a simple version. if not args: - args = ["python", "buildscripts/resmoke.py", "--repeat=2"] + args = ["python", "buildscripts/resmoke.py", "--repeatSuites=2"] # Load the dict of tests to run. if values.test_list_file: diff --git a/buildscripts/evergreen_run_tests.py b/buildscripts/evergreen_run_tests.py index 78f208b6c90..ac6a88c2637 100755 --- a/buildscripts/evergreen_run_tests.py +++ b/buildscripts/evergreen_run_tests.py @@ -40,7 +40,8 @@ class Main(resmoke.Resmoke): tag_name="retry_on_failure", evergreen_aware=True, suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace( # type: ignore - fail_fast=False, num_repeats=2, report_failure_status="silentfail")) + fail_fast=False, num_repeat_suites=2, num_repeat_tests=1, + report_failure_status="silentfail")) @staticmethod def _make_evergreen_aware_tags(tag_name): diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index dd2e0f8622a..c2f092a8154 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -63,7 +63,8 @@ DEFAULTS = { "no_journal": False, "num_clients_per_fixture": 1, "perf_report_file": None, - "repeat": 1, + "repeat_suites": 1, + "repeat_tests": 1, "report_failure_status": "fail", "report_file": None, "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. @@ -109,7 +110,8 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [ "fail_fast", "include_tags", "num_jobs", - "num_repeats", + "num_repeat_suites", + "num_repeat_tests", "report_failure_status", ]) @@ -171,7 +173,8 @@ class SuiteOptions(_SuiteOptions): FAIL_FAST, include_tags, JOBS, - REPEAT, + REPEAT_SUITES, + REPEAT_TESTS, REPORT_FAILURE_STATUS, ])) @@ -299,7 +302,10 @@ PERF_REPORT_FILE = None RANDOM_SEED = None # If set, then each suite is repeated the specified number of times. -REPEAT = None +REPEAT_SUITES = None + +# If set, then each test is repeated the specified number of times inside the suites. +REPEAT_TESTS = None # Controls if the test failure status should be reported as failed or be silently ignored. REPORT_FAILURE_STATUS = None diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 9ddcc914428..48d8ab89bde 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -145,9 +145,14 @@ def _make_parser(): # pylint: disable=too-many-statements " existing MongoDB cluster with the URL mongodb://localhost:[PORT]." " This is useful for connecting to a server running in a debugger.") - parser.add_option("--repeat", type="int", dest="repeat", metavar="N", + parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N", help="Repeats the given suite(s) N times, or until one fails.") + parser.add_option("--repeatTests", type="int", dest="repeat_tests", metavar="N", + help="Repeats the tests inside each suite N times. This applies to tests" + " defined in the suite configuration as well as tests defined on the command" + " line.") + parser.add_option("--reportFailureStatus", type="choice", action="store", dest="report_failure_status", choices=("fail", "silentfail"), metavar="STATUS", @@ -340,10 +345,11 @@ def validate_benchmark_options(): :return: None """ - if _config.REPEAT > 1: + if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1: raise optparse.OptionValueError( - "--repeat cannot be used with benchmark tests. Please use --benchmarkMinTimeSecs to " - "increase the runtime of a single benchmark configuration.") + "--repeatSuites/--repeatTests cannot be used with benchmark tests. " + "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark " + "configuration.") if _config.JOBS > 1: raise optparse.OptionValueError( @@ -390,7 +396,8 @@ def _update_config_vars(values): # pylint: disable=too-many-statements _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture") _config.PERF_REPORT_FILE = config.pop("perf_report_file") _config.RANDOM_SEED = config.pop("seed") - _config.REPEAT = config.pop("repeat") + _config.REPEAT_SUITES = config.pop("repeat_suites") + _config.REPEAT_TESTS = config.pop("repeat_tests") _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status") _config.REPORT_FILE = config.pop("report_file") _config.SERVICE_EXECUTOR = config.pop("service_executor") diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 79ccb17786d..6d67b51da44 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -53,7 +53,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes # Only start as many jobs as we need. Note this means that the number of jobs we run may # not actually be _config.JOBS or self._suite.options.num_jobs. jobs_to_start = self._suite.options.num_jobs - self.num_tests = len(suite.tests) + self.num_tests = len(suite.tests) * self._suite.options.num_repeat_tests if self.num_tests < jobs_to_start: self.logger.info( @@ -80,8 +80,8 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes return_code = 2 return - num_repeats = self._suite.options.num_repeats - while num_repeats > 0: + num_repeat_suites = self._suite.options.num_repeat_suites + while num_repeat_suites > 0: test_queue = self._make_test_queue() partial_reports = [job.report for job in self._jobs] @@ -91,7 +91,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes # finish running their last test. This avoids having a large number of processes # still running if an Evergreen task were to time out from a hang/deadlock being # triggered. - teardown_flag = threading.Event() if num_repeats == 1 else None + teardown_flag = threading.Event() if num_repeat_suites == 1 else None (report, interrupted) = self._run_tests(test_queue, teardown_flag) self._suite.record_test_end(report) @@ -122,7 +122,7 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes # Clear the report so it can be reused for the next execution. for job in self._jobs: job.report.reset() - num_repeats -= 1 + num_repeat_suites -= 1 finally: if not teardown_flag: if not self._teardown_fixtures(): @@ -269,10 +269,11 @@ class TestSuiteExecutor(object): # pylint: disable=too-many-instance-attributes test_queue_logger = self.logger.new_testqueue_logger(self._suite.test_kind) # Put all the test cases in a queue. queue = _queue.Queue() - for test_name in self._suite.tests: - test_case = testcases.make_test_case(self._suite.test_kind, test_queue_logger, - test_name, **self.test_config) - queue.put(test_case) + for _ in range(self._suite.options.num_repeat_tests): + for test_name in self._suite.tests: + test_case = testcases.make_test_case(self._suite.test_kind, test_queue_logger, + test_name, **self.test_config) + queue.put(test_case) # Add sentinel value for each job to indicate when there are no more items to process. for _ in xrange(len(self._jobs)): diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index c968449c8a2..c1ba466f681 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -94,7 +94,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr unittest.TestResult.startTest(self, test) - test_info = _TestInfo(test.id(), dynamic) + test_info = _TestInfo(test.id(), test.test_name, dynamic) test_info.start_time = time.time() basename = test.basename() @@ -262,7 +262,7 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr with self._lock: for test_info in self.test_infos: result = { - "test_file": test_info.test_id, + "test_file": test_info.test_file, "status": test_info.evergreen_status, "exit_code": test_info.return_code, "start": test_info.start_time, @@ -292,7 +292,10 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr for result in report_dict["results"]: # By convention, dynamic tests are named "<basename>:<hook name>". is_dynamic = ":" in result["test_file"] - test_info = _TestInfo(result["test_file"], is_dynamic) + test_file = result["test_file"] + # Using test_file as the test id is ok here since the test id only needs to be unique + # during suite execution. + test_info = _TestInfo(test_file, test_file, is_dynamic) test_info.url_endpoint = result.get("url") test_info.status = result["status"] test_info.evergreen_status = test_info.status @@ -341,10 +344,11 @@ class TestReport(unittest.TestResult): # pylint: disable=too-many-instance-attr class _TestInfo(object): # pylint: disable=too-many-instance-attributes """Holder for the test status and timing information.""" - def __init__(self, test_id, dynamic): + def __init__(self, test_id, test_file, dynamic): """Initialize the _TestInfo instance.""" self.test_id = test_id + self.test_file = test_file self.dynamic = dynamic self.start_time = None diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py index 1a57b6c7716..87fad83692f 100644 --- a/buildscripts/resmokelib/testing/suite.py +++ b/buildscripts/resmokelib/testing/suite.py @@ -271,7 +271,8 @@ class Suite(object): # pylint: disable=too-many-instance-attributes # cannot be said to have succeeded. num_failed = report.num_failed + report.num_interrupted num_run = report.num_succeeded + report.num_errored + num_failed - num_skipped = len(self.tests) + report.num_dynamic - num_run + num_tests = len(self.tests) * self.options.num_repeat_tests + num_skipped = num_tests + report.num_dynamic - num_run if report.num_succeeded == num_run and num_skipped == 0: sb.append("All %d test(s) passed in %0.2f seconds." % (num_run, time_taken)) @@ -286,12 +287,12 @@ class Suite(object): # pylint: disable=too-many-instance-attributes if num_failed > 0: sb.append("The following tests failed (with exit code):") for test_info in itertools.chain(report.get_failed(), report.get_interrupted()): - sb.append(" %s (%d)" % (test_info.test_id, test_info.return_code)) + sb.append(" %s (%d)" % (test_info.test_file, test_info.return_code)) if report.num_errored > 0: sb.append("The following tests had errors:") for test_info in report.get_errored(): - sb.append(" %s" % (test_info.test_id)) + sb.append(" %s" % (test_info.test_file)) return summary diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py index 183e69f9d36..f27df970b34 100644 --- a/buildscripts/resmokelib/testing/testcases/interface.py +++ b/buildscripts/resmokelib/testing/testcases/interface.py @@ -8,6 +8,7 @@ from __future__ import absolute_import import os import os.path import unittest +import uuid from ... import logging from ...utils import registry @@ -22,7 +23,7 @@ def make_test_case(test_kind, *args, **kwargs): return _TEST_CASES[test_kind](*args, **kwargs) -class TestCase(unittest.TestCase): +class TestCase(unittest.TestCase): # pylint: disable=too-many-instance-attributes """A test case to execute.""" __metaclass__ = registry.make_registry_metaclass(_TEST_CASES) # type: ignore @@ -42,6 +43,8 @@ class TestCase(unittest.TestCase): if not isinstance(test_name, basestring): raise TypeError("test_name must be a string") + self._id = uuid.uuid4() + # When the TestCase is created by the TestSuiteExecutor (through a call to make_test_case()) # logger is an instance of TestQueueLogger. When the TestCase is created by a hook # implementation it is an instance of BaseLogger. @@ -71,7 +74,7 @@ class TestCase(unittest.TestCase): def id(self): """Return the id of the test.""" - return self.test_name + return self._id def short_description(self): """Return the short_description of the test.""" diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 88054083a15..cd24606dc84 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -4027,7 +4027,7 @@ tasks: vars: task_path_suffix: /data/multiversion:$HOME resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json - resmoke_args: --repeat=2 + resmoke_args: --repeatSuites=2 run_multiple_jobs: true - <<: *benchmark_template @@ -7756,7 +7756,7 @@ buildvariants: expansions: compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars --enable-free-mon=off --enable-http-client=off num_jobs_available: $(grep -c ^processor /proc/cpuinfo) - test_flags: --repeat=10 --shuffle + test_flags: --repeatSuites=10 --shuffle scons_cache_scope: shared gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10' test_flags: --excludeWithAnyTags=requires_http_client |