SERVER-34593 New resmoke option to repeat tests

9 files changed, 54 insertions, 31 deletions

diff --git a/buildscripts/burn_in_tests.py b/buildscripts/burn_in_tests.py
index 690303d3b12..cae3ee9baca 100644
--- a/buildscripts/burn_in_tests.py
+++ b/buildscripts/burn_in_tests.py
@@ -337,7 +337,7 @@ def main():
 
     # If a resmoke.py command wasn't passed in, use a simple version.
     if not args:
-        args = ["python", "buildscripts/resmoke.py", "--repeat=2"]
+        args = ["python", "buildscripts/resmoke.py", "--repeatSuites=2"]
 
     # Load the dict of tests to run.
     if values.test_list_file:
diff --git a/buildscripts/evergreen_run_tests.py b/buildscripts/evergreen_run_tests.py
index 78f208b6c90..ac6a88c2637 100755
--- a/buildscripts/evergreen_run_tests.py
+++ b/buildscripts/evergreen_run_tests.py
@@ -40,7 +40,8 @@ class Main(resmoke.Resmoke):
         tag_name="retry_on_failure",
         evergreen_aware=True,
         suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace(  # type: ignore
-            fail_fast=False, num_repeats=2, report_failure_status="silentfail"))
+            fail_fast=False, num_repeat_suites=2, num_repeat_tests=1,
+            report_failure_status="silentfail"))
 
     @staticmethod
     def _make_evergreen_aware_tags(tag_name):
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index dd2e0f8622a..c2f092a8154 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -63,7 +63,8 @@ DEFAULTS = {
     "no_journal": False,
     "num_clients_per_fixture": 1,
     "perf_report_file": None,
-    "repeat": 1,
+    "repeat_suites": 1,
+    "repeat_tests": 1,
     "report_failure_status": "fail",
     "report_file": None,
     "seed": long(time.time() * 256),  # Taken from random.py code in Python 2.7.
@@ -109,7 +110,8 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [
     "fail_fast",
     "include_tags",
     "num_jobs",
-    "num_repeats",
+    "num_repeat_suites",
+    "num_repeat_tests",
     "report_failure_status",
 ])
 
@@ -171,7 +173,8 @@ class SuiteOptions(_SuiteOptions):
                 FAIL_FAST,
                 include_tags,
                 JOBS,
-                REPEAT,
+                REPEAT_SUITES,
+                REPEAT_TESTS,
                 REPORT_FAILURE_STATUS,
             ]))
 
@@ -299,7 +302,10 @@ PERF_REPORT_FILE = None
 RANDOM_SEED = None
 
 # If set, then each suite is repeated the specified number of times.
-REPEAT = None
+REPEAT_SUITES = None
+
+# If set, then each test is repeated the specified number of times inside the suites.
+REPEAT_TESTS = None
 
 # Controls if the test failure status should be reported as failed or be silently ignored.
 REPORT_FAILURE_STATUS = None
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 9ddcc914428..48d8ab89bde 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -145,9 +145,14 @@ def _make_parser():  # pylint: disable=too-many-statements
                       " existing MongoDB cluster with the URL mongodb://localhost:[PORT]."
                       " This is useful for connecting to a server running in a debugger.")
 
-    parser.add_option("--repeat", type="int", dest="repeat", metavar="N",
+    parser.add_option("--repeat", "--repeatSuites", type="int", dest="repeat_suites", metavar="N",
                       help="Repeats the given suite(s) N times, or until one fails.")
 
+    parser.add_option("--repeatTests", type="int", dest="repeat_tests", metavar="N",
+                      help="Repeats the tests inside each suite N times. This applies to tests"
+                      " defined in the suite configuration as well as tests defined on the command"
+                      " line.")
+
     parser.add_option("--reportFailureStatus", type="choice", action="store",
                       dest="report_failure_status", choices=("fail",
                                                              "silentfail"), metavar="STATUS",
@@ -340,10 +345,11 @@ def validate_benchmark_options():
     :return: None
     """
 
-    if _config.REPEAT > 1:
+    if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1:
         raise optparse.OptionValueError(
-            "--repeat cannot be used with benchmark tests. Please use --benchmarkMinTimeSecs to "
-            "increase the runtime of a single benchmark configuration.")
+            "--repeatSuites/--repeatTests cannot be used with benchmark tests. "
+            "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark "
+            "configuration.")
 
     if _config.JOBS > 1:
         raise optparse.OptionValueError(
@@ -390,7 +396,8 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements
     _config.NUM_CLIENTS_PER_FIXTURE = config.pop("num_clients_per_fixture")
     _config.PERF_REPORT_FILE = config.pop("perf_report_file")
     _config.RANDOM_SEED = config.pop("seed")
-    _config.REPEAT = config.pop("repeat")
+    _config.REPEAT_SUITES = config.pop("repeat_suites")
+    _config.REPEAT_TESTS = config.pop("repeat_tests")
     _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
     _config.REPORT_FILE = config.pop("report_file")
     _config.SERVICE_EXECUTOR = config.pop("service_executor")
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index 79ccb17786d..6d67b51da44 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -53,7 +53,7 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
         # Only start as many jobs as we need. Note this means that the number of jobs we run may
         # not actually be _config.JOBS or self._suite.options.num_jobs.
         jobs_to_start = self._suite.options.num_jobs
-        self.num_tests = len(suite.tests)
+        self.num_tests = len(suite.tests) * self._suite.options.num_repeat_tests
 
         if self.num_tests < jobs_to_start:
             self.logger.info(
@@ -80,8 +80,8 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
                 return_code = 2
                 return
 
-            num_repeats = self._suite.options.num_repeats
-            while num_repeats > 0:
+            num_repeat_suites = self._suite.options.num_repeat_suites
+            while num_repeat_suites > 0:
                 test_queue = self._make_test_queue()
 
                 partial_reports = [job.report for job in self._jobs]
@@ -91,7 +91,7 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
                 # finish running their last test. This avoids having a large number of processes
                 # still running if an Evergreen task were to time out from a hang/deadlock being
                 # triggered.
-                teardown_flag = threading.Event() if num_repeats == 1 else None
+                teardown_flag = threading.Event() if num_repeat_suites == 1 else None
                 (report, interrupted) = self._run_tests(test_queue, teardown_flag)
 
                 self._suite.record_test_end(report)
@@ -122,7 +122,7 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
                 # Clear the report so it can be reused for the next execution.
                 for job in self._jobs:
                     job.report.reset()
-                num_repeats -= 1
+                num_repeat_suites -= 1
         finally:
             if not teardown_flag:
                 if not self._teardown_fixtures():
@@ -269,10 +269,11 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
         test_queue_logger = self.logger.new_testqueue_logger(self._suite.test_kind)
         # Put all the test cases in a queue.
         queue = _queue.Queue()
-        for test_name in self._suite.tests:
-            test_case = testcases.make_test_case(self._suite.test_kind, test_queue_logger,
-                                                 test_name, **self.test_config)
-            queue.put(test_case)
+        for _ in range(self._suite.options.num_repeat_tests):
+            for test_name in self._suite.tests:
+                test_case = testcases.make_test_case(self._suite.test_kind, test_queue_logger,
+                                                     test_name, **self.test_config)
+                queue.put(test_case)
 
         # Add sentinel value for each job to indicate when there are no more items to process.
         for _ in xrange(len(self._jobs)):
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index c968449c8a2..c1ba466f681 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -94,7 +94,7 @@ class TestReport(unittest.TestResult):  # pylint: disable=too-many-instance-attr
 
         unittest.TestResult.startTest(self, test)
 
-        test_info = _TestInfo(test.id(), dynamic)
+        test_info = _TestInfo(test.id(), test.test_name, dynamic)
         test_info.start_time = time.time()
 
         basename = test.basename()
@@ -262,7 +262,7 @@ class TestReport(unittest.TestResult):  # pylint: disable=too-many-instance-attr
         with self._lock:
             for test_info in self.test_infos:
                 result = {
-                    "test_file": test_info.test_id,
+                    "test_file": test_info.test_file,
                     "status": test_info.evergreen_status,
                     "exit_code": test_info.return_code,
                     "start": test_info.start_time,
@@ -292,7 +292,10 @@ class TestReport(unittest.TestResult):  # pylint: disable=too-many-instance-attr
         for result in report_dict["results"]:
             # By convention, dynamic tests are named "<basename>:<hook name>".
             is_dynamic = ":" in result["test_file"]
-            test_info = _TestInfo(result["test_file"], is_dynamic)
+            test_file = result["test_file"]
+            # Using test_file as the test id is ok here since the test id only needs to be unique
+            # during suite execution.
+            test_info = _TestInfo(test_file, test_file, is_dynamic)
             test_info.url_endpoint = result.get("url")
             test_info.status = result["status"]
             test_info.evergreen_status = test_info.status
@@ -341,10 +344,11 @@ class TestReport(unittest.TestResult):  # pylint: disable=too-many-instance-attr
 class _TestInfo(object):  # pylint: disable=too-many-instance-attributes
     """Holder for the test status and timing information."""
 
-    def __init__(self, test_id, dynamic):
+    def __init__(self, test_id, test_file, dynamic):
         """Initialize the _TestInfo instance."""
 
         self.test_id = test_id
+        self.test_file = test_file
         self.dynamic = dynamic
 
         self.start_time = None
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index 1a57b6c7716..87fad83692f 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -271,7 +271,8 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         # cannot be said to have succeeded.
         num_failed = report.num_failed + report.num_interrupted
         num_run = report.num_succeeded + report.num_errored + num_failed
-        num_skipped = len(self.tests) + report.num_dynamic - num_run
+        num_tests = len(self.tests) * self.options.num_repeat_tests
+        num_skipped = num_tests + report.num_dynamic - num_run
 
         if report.num_succeeded == num_run and num_skipped == 0:
             sb.append("All %d test(s) passed in %0.2f seconds." % (num_run, time_taken))
@@ -286,12 +287,12 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         if num_failed > 0:
             sb.append("The following tests failed (with exit code):")
             for test_info in itertools.chain(report.get_failed(), report.get_interrupted()):
-                sb.append("    %s (%d)" % (test_info.test_id, test_info.return_code))
+                sb.append("    %s (%d)" % (test_info.test_file, test_info.return_code))
 
         if report.num_errored > 0:
             sb.append("The following tests had errors:")
             for test_info in report.get_errored():
-                sb.append("    %s" % (test_info.test_id))
+                sb.append("    %s" % (test_info.test_file))
 
         return summary
 
diff --git a/buildscripts/resmokelib/testing/testcases/interface.py b/buildscripts/resmokelib/testing/testcases/interface.py
index 183e69f9d36..f27df970b34 100644
--- a/buildscripts/resmokelib/testing/testcases/interface.py
+++ b/buildscripts/resmokelib/testing/testcases/interface.py
@@ -8,6 +8,7 @@ from __future__ import absolute_import
 import os
 import os.path
 import unittest
+import uuid
 
 from ... import logging
 from ...utils import registry
@@ -22,7 +23,7 @@ def make_test_case(test_kind, *args, **kwargs):
     return _TEST_CASES[test_kind](*args, **kwargs)
 
 
-class TestCase(unittest.TestCase):
+class TestCase(unittest.TestCase):  # pylint: disable=too-many-instance-attributes
     """A test case to execute."""
 
     __metaclass__ = registry.make_registry_metaclass(_TEST_CASES)  # type: ignore
@@ -42,6 +43,8 @@ class TestCase(unittest.TestCase):
         if not isinstance(test_name, basestring):
             raise TypeError("test_name must be a string")
 
+        self._id = uuid.uuid4()
+
         # When the TestCase is created by the TestSuiteExecutor (through a call to make_test_case())
         # logger is an instance of TestQueueLogger. When the TestCase is created by a hook
         # implementation it is an instance of BaseLogger.
@@ -71,7 +74,7 @@ class TestCase(unittest.TestCase):
 
     def id(self):
         """Return the id of the test."""
-        return self.test_name
+        return self._id
 
     def short_description(self):
         """Return the short_description of the test."""
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 88054083a15..cd24606dc84 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -4027,7 +4027,7 @@ tasks:
     vars:
       task_path_suffix: /data/multiversion:$HOME
       resmoke_wrapper: $python buildscripts/burn_in_tests.py --testListFile=jstests/new_tests.json
-      resmoke_args: --repeat=2
+      resmoke_args: --repeatSuites=2
       run_multiple_jobs: true
 
 - <<: *benchmark_template
@@ -7756,7 +7756,7 @@ buildvariants:
   expansions:
     compile_flags: -j$(grep -c ^processor /proc/cpuinfo) --variables-files=etc/scons/mongodbtoolchain_gcc.vars --enable-free-mon=off --enable-http-client=off
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
-    test_flags: --repeat=10 --shuffle
+    test_flags: --repeatSuites=10 --shuffle
     scons_cache_scope: shared
     gorootvars: 'PATH="/opt/golang/go1.10/bin:/opt/mongodbtoolchain/v2/bin/:$PATH" GOROOT=/opt/golang/go1.10'
     test_flags: --excludeWithAnyTags=requires_http_client