SERVER-39305 Add --repeatTestsSecs, --repeatTestsMin & --repeatTestsMax options to resmoke

11 files changed, 496 insertions, 18 deletions

diff --git a/buildscripts/resmokeconfig/suites/buildscripts_test.yml b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
index 3184bb1fed9..3e29cd214a3 100644
--- a/buildscripts/resmokeconfig/suites/buildscripts_test.yml
+++ b/buildscripts/resmokeconfig/suites/buildscripts_test.yml
@@ -5,8 +5,8 @@ selector:
   - buildscripts/tests/**/test_*.py
   - buildscripts/idl/tests/**/test_*.py
   exclude_files:
-  - buildscripts/tests/resmokelib/test_archival.py # Requires boto3.
   - buildscripts/tests/resmokelib/test_selector.py # Test assumes POSIX path.
+  - buildscripts/tests/resmokelib/utils/test_archival.py # Requires boto3.
   - buildscripts/tests/test_aws_ec2.py # Requires boto3.
   - buildscripts/tests/test_remote_operations.py # Requires ssh to be enabled locally.
   - buildscripts/tests/test_update_test_lifecycle.py # Test assumes POSIX path.
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index b18120c561f..50af5c5ccdd 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -70,6 +70,9 @@ DEFAULTS = {
     "perf_report_file": None,
     "repeat_suites": 1,
     "repeat_tests": 1,
+    "repeat_tests_max": None,
+    "repeat_tests_min": None,
+    "repeat_tests_secs": None,
     "report_failure_status": "fail",
     "report_file": None,
     "seed": long(time.time() * 256),  # Taken from random.py code in Python 2.7.
@@ -119,6 +122,9 @@ _SuiteOptions = collections.namedtuple("_SuiteOptions", [
     "num_jobs",
     "num_repeat_suites",
     "num_repeat_tests",
+    "num_repeat_tests_max",
+    "num_repeat_tests_min",
+    "time_repeat_tests_secs",
     "report_failure_status",
 ])
 
@@ -182,6 +188,9 @@ class SuiteOptions(_SuiteOptions):
                 JOBS,
                 REPEAT_SUITES,
                 REPEAT_TESTS,
+                REPEAT_TESTS_MAX,
+                REPEAT_TESTS_MIN,
+                REPEAT_TESTS_SECS,
                 REPORT_FAILURE_STATUS,
             ]))
 
@@ -320,6 +329,17 @@ REPEAT_SUITES = None
 # If set, then each test is repeated the specified number of times inside the suites.
 REPEAT_TESTS = None
 
+# If set and REPEAT_TESTS_SECS is set, then each test is repeated up to specified number of
+# times inside the suites.
+REPEAT_TESTS_MAX = None
+
+# If set and REPEAT_TESTS_SECS is set, then each test is repeated at least specified number of
+# times inside the suites.
+REPEAT_TESTS_MIN = None
+
+# If set, then each test is repeated the specified time (seconds) inside the suites.
+REPEAT_TESTS_SECS = None
+
 # Controls if the test failure status should be reported as failed or be silently ignored.
 REPORT_FAILURE_STATUS = None
 
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 3c7969a3f19..6a93f3593eb 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -164,6 +164,23 @@ def _make_parser():  # pylint: disable=too-many-statements
                       " defined in the suite configuration as well as tests defined on the command"
                       " line.")
 
+    parser.add_option("--repeatTestsMax", type="int", dest="repeat_tests_max", metavar="N",
+                      help="Repeats the tests inside each suite no more than N time when"
+                      " --repeatTestsSecs is specified. This applies to tests defined in the suite"
+                      " configuration as well as tests defined on the command line.")
+
+    parser.add_option("--repeatTestsMin", type="int", dest="repeat_tests_min", metavar="N",
+                      help="Repeats the tests inside each suite at least N times when"
+                      " --repeatTestsSecs is specified. This applies to tests defined in the suite"
+                      " configuration as well as tests defined on the command line.")
+
+    parser.add_option("--repeatTestsSecs", type="float", dest="repeat_tests_secs",
+                      metavar="SECONDS",
+                      help="Repeats the tests inside each suite this amount of time. Note that"
+                      " this option is mutually exclusive with --repeatTests. This applies to"
+                      " tests defined in the suite configuration as well as tests defined on the"
+                      " command line.")
+
     parser.add_option("--reportFailureStatus", type="choice", action="store",
                       dest="report_failure_status", choices=("fail",
                                                              "silentfail"), metavar="STATUS",
@@ -337,6 +354,7 @@ def parse_command_line():
 
     _validate_options(parser, options, args)
     _update_config_vars(options)
+    _validate_config(parser)
 
     return ResmokeConfig(list_suites=options.list_suites, find_suites=options.find_suites,
                          dry_run=options.dry_run, suite_files=options.suite_files.split(","),
@@ -355,13 +373,30 @@ def _validate_options(parser, options, args):
                          options.executor_file, " ".join(args)))
 
 
+def _validate_config(parser):
+    """Do validation on the config settings."""
+
+    if _config.REPEAT_TESTS_MAX:
+        if not _config.REPEAT_TESTS_SECS:
+            parser.error("Must specify --repeatTestsSecs with --repeatTestsMax")
+
+        if _config.REPEAT_TESTS_MIN > _config.REPEAT_TESTS_MAX:
+            parser.error("--repeatTestsSecsMin > --repeatTestsMax")
+
+    if _config.REPEAT_TESTS_MIN and not _config.REPEAT_TESTS_SECS:
+        parser.error("Must specify --repeatTestsSecs with --repeatTestsMin")
+
+    if _config.REPEAT_TESTS > 1 and _config.REPEAT_TESTS_SECS:
+        parser.error("Cannot specify --repeatTests and --repeatTestsSecs")
+
+
 def validate_benchmark_options():
     """Error out early if any options are incompatible with benchmark test suites.
 
     :return: None
     """
 
-    if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1:
+    if _config.REPEAT_SUITES > 1 or _config.REPEAT_TESTS > 1 or _config.REPEAT_TESTS_SECS:
         raise optparse.OptionValueError(
             "--repeatSuites/--repeatTests cannot be used with benchmark tests. "
             "Please use --benchmarkMinTimeSecs to increase the runtime of a single benchmark "
@@ -419,6 +454,9 @@ def _update_config_vars(values):  # pylint: disable=too-many-statements
     _config.RANDOM_SEED = config.pop("seed")
     _config.REPEAT_SUITES = config.pop("repeat_suites")
     _config.REPEAT_TESTS = config.pop("repeat_tests")
+    _config.REPEAT_TESTS_MAX = config.pop("repeat_tests_max")
+    _config.REPEAT_TESTS_MIN = config.pop("repeat_tests_min")
+    _config.REPEAT_TESTS_SECS = config.pop("repeat_tests_secs")
     _config.REPORT_FAILURE_STATUS = config.pop("report_failure_status")
     _config.REPORT_FILE = config.pop("report_file")
     _config.SERVICE_EXECUTOR = config.pop("service_executor")
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index aeb7f8b6f3c..024cce166a4 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -9,6 +9,7 @@ from . import fixtures
 from . import hook_test_archival as archival
 from . import hooks as _hooks
 from . import job as _job
+from . import queue_element
 from . import report as _report
 from . import testcases
 from .. import config as _config
@@ -178,8 +179,10 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
             user_interrupted = True
         else:
             # Only wait for all the Job instances if not interrupted by the user.
+            self.logger.debug("Waiting for threads to complete")
             for thr in threads:
                 thr.join()
+            self.logger.debug("Threads are completed!")
 
         reports = [job.report for job in self._jobs]
         combined_report = _report.TestReport.combine(*reports)
@@ -252,15 +255,16 @@ class TestSuiteExecutor(object):  # pylint: disable=too-many-instance-attributes
         """
 
         # Put all the test cases in a queue.
+        if self._suite.options.time_repeat_tests_secs:
+            queue_method = queue_element.QueueElemRepeatTime
+        else:
+            queue_method = queue_element.QueueElemRepeatNum
+
         queue = _queue.Queue()
-        for _ in range(self._suite.options.num_repeat_tests):
-            for test_name in self._suite.tests:
-                test_case = testcases.make_test_case(self._suite.test_kind, self.test_queue_logger,
-                                                     test_name, **self.test_config)
-                queue.put(test_case)
-
-        # Add sentinel value for each job to indicate when there are no more items to process.
-        for _ in xrange(len(self._jobs)):
-            queue.put(None)
+
+        for test_name in self._suite.tests:
+            test_case = testcases.make_test_case(self._suite.test_kind, self.test_queue_logger,
+                                                 test_name, **self.test_config)
+            queue.put(queue_method(test_case, self.test_config, self._suite.options))
 
         return queue
diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index cfff5ccec58..908ae85832a 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -3,7 +3,10 @@
 from __future__ import absolute_import
 
 import sys
+import time
 
+from . import queue_element
+from . import testcases
 from .. import config
 from .. import errors
 from ..testing.fixtures import interface as _fixtures
@@ -107,19 +110,48 @@ class Job(object):  # pylint: disable=too-many-instance-attributes
         for hook in self.hooks:
             hook.before_suite(self.report)
 
-        while not interrupt_flag.is_set():
-            test = queue.get_nowait()
+        while not queue.empty() and not interrupt_flag.is_set():
+            queue_elem = queue.get_nowait()
+            test_time_start = time.time()
             try:
-                if test is None:
-                    # Sentinel value received, so exit.
-                    break
+                test = queue_elem.testcase
                 self._execute_test(test)
             finally:
+                queue_elem.job_completed(time.time() - test_time_start)
                 queue.task_done()
 
+            self._requeue_test(queue, queue_elem, interrupt_flag)
+
         for hook in self.hooks:
             hook.after_suite(self.report)
 
+    def _log_requeue_test(self, queue_elem):
+        """Log the requeue of a test."""
+
+        if self.suite_options.time_repeat_tests_secs:
+            progress = "{} of ({}/{}/{:2.2f} min/max/time)".format(
+                queue_elem.repeat_num + 1, self.suite_options.num_repeat_tests_min,
+                self.suite_options.num_repeat_tests_max, self.suite_options.time_repeat_tests_secs)
+        else:
+            progress = "{} of {}".format(queue_elem.repeat_num + 1,
+                                         self.suite_options.num_repeat_tests)
+        self.logger.info(("Requeueing test %s %s, cumulative time elapsed %0.2f"),
+                         queue_elem.testcase.test_name, progress, queue_elem.repeat_time_elapsed)
+
+    def _requeue_test(self, queue, queue_elem, interrupt_flag):
+        """Requeue a test if it needs to be repeated."""
+
+        if not queue_elem.should_requeue():
+            return
+
+        queue_elem.testcase = testcases.make_test_case(
+            queue_elem.testcase.REGISTERED_NAME, queue_elem.testcase.logger,
+            queue_elem.testcase.test_name, **queue_elem.test_config)
+
+        if not interrupt_flag.is_set():
+            self._log_requeue_test(queue_elem)
+            queue.put(queue_elem)
+
     def _execute_test(self, test):
         """Call the before/after test hooks and execute 'test'."""
 
diff --git a/buildscripts/resmokelib/testing/queue_element.py b/buildscripts/resmokelib/testing/queue_element.py
new file mode 100644
index 00000000000..052864c9180
--- /dev/null
+++ b/buildscripts/resmokelib/testing/queue_element.py
@@ -0,0 +1,48 @@
+"""Queue entry interface."""
+
+
+class QueueElemRepeatNum(object):
+    """Base class for an element on the queue."""
+
+    def __init__(self, testcase, test_config, suite_options):
+        """Initialize QueueElemRepeatNum class."""
+        self.testcase = testcase
+        self.test_config = test_config
+        self.repeat_num_min = suite_options.num_repeat_tests
+        self.repeat_time_elapsed = 0
+        self.repeat_num = 0
+
+    def job_completed(self, job_time):
+        """Increment values when the job completes."""
+        self.repeat_num += 1
+        self.repeat_time_elapsed += job_time
+
+    def should_requeue(self):
+        """Return True if the queue element should be requeued."""
+        return self.repeat_num < self.repeat_num_min
+
+
+class QueueElemRepeatTime(QueueElemRepeatNum):
+    """Queue element for repeat time."""
+
+    def __init__(self, testcase, config, suite_options):
+        """Initialize QueueElemRepeatTime class."""
+        super(QueueElemRepeatTime, self).__init__(testcase, config, suite_options)
+        self.repeat_num_min = suite_options.num_repeat_tests_min
+        self.repeat_num_max = suite_options.num_repeat_tests_max
+        self.repeat_secs = suite_options.time_repeat_tests_secs
+
+    def should_requeue(self):
+        """Return True if the queue element should be requeued."""
+        avg_time = 0 if self.repeat_num == 0 else self.repeat_time_elapsed / self.repeat_num
+
+        # Minumim number of tests has not been run.
+        if self.repeat_num_min and self.repeat_num < self.repeat_num_min:
+            return True
+
+        # Maximum number of tests has been run or elapsed time has been exceeded.
+        if ((self.repeat_num_max and self.repeat_num >= self.repeat_num_max)
+                or self.repeat_time_elapsed + avg_time > self.repeat_secs):
+            return False
+
+        return True
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index e179854cb15..e0f2dda5151 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -306,8 +306,13 @@ class Suite(object):  # pylint: disable=too-many-instance-attributes
         # cannot be said to have succeeded.
         num_failed = report.num_failed + report.num_interrupted
         num_run = report.num_succeeded + report.num_errored + num_failed
-        num_tests = len(self.tests) * self.options.num_repeat_tests
-        num_skipped = num_tests + report.num_dynamic - num_run
+        # The number of skipped tests is only known if self.options.time_repeat_tests_secs
+        # is not specified.
+        if self.options.time_repeat_tests_secs:
+            num_skipped = 0
+        else:
+            num_tests = len(self.tests) * self.options.num_repeat_tests
+            num_skipped = num_tests + report.num_dynamic - num_run
 
         if report.num_succeeded == num_run and num_skipped == 0:
             sb.append("All %d test(s) passed in %0.2f seconds." % (num_run, time_taken))
diff --git a/buildscripts/tests/resmokelib/testing/test_executor.py b/buildscripts/tests/resmokelib/testing/test_executor.py
new file mode 100644
index 00000000000..33331c22313
--- /dev/null
+++ b/buildscripts/tests/resmokelib/testing/test_executor.py
@@ -0,0 +1,53 @@
+"""Unit tests for the resmokelib.testing.executor module."""
+import logging
+import unittest
+
+import mock
+
+from buildscripts.resmokelib.testing import executor
+from buildscripts.resmokelib.testing import queue_element
+
+# pylint: disable=missing-docstring,protected-access
+
+
+class TestTestSuiteExecutor(unittest.TestCase):
+    @staticmethod
+    def mock_suite():
+        suite = mock.Mock()
+        suite.test_kind = "js_test"
+        suite.tests = ["jstests/core/and.js", "jstests/core/and2.js"]
+        suite.options = mock.Mock()
+        return suite
+
+    def test__make_test_queue_time_repeat(self):
+        suite = self.mock_suite()
+        suite.options.time_repeat_tests_secs = 30
+        executor_object = UnitTestExecutor(suite, {})
+        test_queue = executor_object._make_test_queue()
+        self.assertFalse(test_queue.empty())
+        self.assertEqual(test_queue.qsize(), len(suite.tests))
+        for suite_test in suite.tests:
+            test_element = test_queue.get_nowait()
+            self.assertIsInstance(test_element, queue_element.QueueElemRepeatTime)
+            self.assertEqual(test_element.testcase.test_name, suite_test)
+        self.assertTrue(test_queue.empty())
+
+    def test__make_test_queue_num_repeat(self):
+        suite = self.mock_suite()
+        suite.options.time_repeat_tests_secs = None
+        executor_object = UnitTestExecutor(suite, {})
+        test_queue = executor_object._make_test_queue()
+        self.assertFalse(test_queue.empty())
+        self.assertEqual(test_queue.qsize(), len(suite.tests))
+        for suite_test in suite.tests:
+            test_element = test_queue.get_nowait()
+            self.assertIsInstance(test_element, queue_element.QueueElemRepeatNum)
+            self.assertEqual(test_element.testcase.test_name, suite_test)
+        self.assertTrue(test_queue.empty())
+
+
+class UnitTestExecutor(executor.TestSuiteExecutor):
+    def __init__(self, suite, config):  # pylint: disable=super-init-not-called
+        self._suite = suite
+        self.test_queue_logger = logging.getLogger("executor_unittest")
+        self.test_config = config
diff --git a/buildscripts/tests/resmokelib/testing/test_job.py b/buildscripts/tests/resmokelib/testing/test_job.py
new file mode 100644
index 00000000000..4d16ea65faf
--- /dev/null
+++ b/buildscripts/tests/resmokelib/testing/test_job.py
@@ -0,0 +1,186 @@
+"""Unit tests for the resmokelib.testing.executor module."""
+from __future__ import division
+
+import logging
+import time
+import unittest
+
+import mock
+
+from buildscripts.resmokelib.testing import job
+from buildscripts.resmokelib.testing import queue_element
+from buildscripts.resmokelib.utils import queue as _queue
+
+# pylint: disable=missing-docstring,protected-access
+
+
+class TestJob(unittest.TestCase):
+
+    TESTS = ["jstests/core/and.js", "jstests/core/or.js"]
+
+    @staticmethod
+    def mock_testcase(test_name):
+        testcase = mock.Mock()
+        testcase.test_name = test_name
+        testcase.REGISTERED_NAME = "js_test"
+        testcase.logger = logging.getLogger("job_unittest")
+        return testcase
+
+    @staticmethod
+    def mock_interrupt_flag():
+        interrupt_flag = mock.Mock()
+        interrupt_flag.is_set = lambda: False
+        return interrupt_flag
+
+    @staticmethod
+    def get_suite_options(num_repeat_tests=None, time_repeat_tests_secs=None,
+                          num_repeat_tests_min=None, num_repeat_tests_max=None):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests = num_repeat_tests
+        suite_options.time_repeat_tests_secs = time_repeat_tests_secs
+        suite_options.num_repeat_tests_min = num_repeat_tests_min
+        suite_options.num_repeat_tests_max = num_repeat_tests_max
+        return suite_options
+
+    @staticmethod
+    def queue_tests(tests, queue, queue_elem_type, suite_options):
+        for test in tests:
+            queue_elem = queue_elem_type(TestJob.mock_testcase(test), {}, suite_options)
+            queue.put(queue_elem)
+
+    @staticmethod
+    def expected_run_num(time_repeat_tests_secs, test_time_secs):
+        """Return the number of times a test is expected to run.
+
+        Note this result should be used as an approximation as the test_time_secs relies on
+        time.sleep(), which does not guarantee an exact pause, plus the overhead of other functions
+        running for each test.
+        """
+        return (time_repeat_tests_secs / test_time_secs) - 1
+
+    def test__run_num_repeat(self):
+        num_repeat_tests = 3
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(num_repeat_tests=num_repeat_tests)
+        job_object = UnitJob(suite_options, {})
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatNum, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertEqual(job_object.total_test_num, num_repeat_tests * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertEqual(job_object.tests[test], num_repeat_tests)
+
+    def test__run_time_repeat_time_no_min_max(self):
+        delay = 0.1
+        time_repeat_tests_secs = 1
+        expected_tests_run = self.expected_run_num(time_repeat_tests_secs, delay)
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertGreaterEqual(job_object.total_test_num, expected_tests_run * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertGreaterEqual(job_object.tests[test], expected_tests_run)
+
+    def test__run_time_repeat_time_no_min(self):
+        delay = 0.1
+        time_repeat_tests_secs = 1
+        num_repeat_tests_max = 100
+        expected_tests_run = self.expected_run_num(time_repeat_tests_secs, delay)
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertLess(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertGreaterEqual(job_object.tests[test], expected_tests_run)
+
+    def test__run_time_repeat_time_no_max(self):
+        delay = 0.1
+        time_repeat_tests_secs = 1
+        num_repeat_tests_min = 1
+        expected_tests_run = self.expected_run_num(time_repeat_tests_secs, delay)
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs,
+                                               num_repeat_tests_min=num_repeat_tests_min)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertGreater(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertGreaterEqual(job_object.tests[test], expected_tests_run)
+
+    def test__run_time_repeat_time(self):
+        delay = 0.1
+        time_repeat_tests_secs = 1
+        num_repeat_tests_min = 1
+        num_repeat_tests_max = 100
+        expected_tests_run = self.expected_run_num(time_repeat_tests_secs, delay)
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs,
+                                               num_repeat_tests_min=num_repeat_tests_min,
+                                               num_repeat_tests_max=num_repeat_tests_max)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertGreater(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS))
+        self.assertLess(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertGreaterEqual(job_object.tests[test], expected_tests_run)
+
+    def test__run_time_repeat_min(self):
+        delay = 0.1
+        time_repeat_tests_secs = 0.05
+        num_repeat_tests_min = 3
+        num_repeat_tests_max = 100
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs,
+                                               num_repeat_tests_min=num_repeat_tests_min,
+                                               num_repeat_tests_max=num_repeat_tests_max)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertEqual(job_object.total_test_num, num_repeat_tests_min * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertEqual(job_object.tests[test], num_repeat_tests_min)
+
+    def test__run_time_repeat_max(self):
+        delay = 0.1
+        time_repeat_tests_secs = 30
+        num_repeat_tests_min = 1
+        num_repeat_tests_max = 10
+        expected_time_repeat_tests = self.expected_run_num(time_repeat_tests_secs, delay)
+        queue = _queue.Queue()
+        suite_options = self.get_suite_options(time_repeat_tests_secs=time_repeat_tests_secs,
+                                               num_repeat_tests_min=num_repeat_tests_min,
+                                               num_repeat_tests_max=num_repeat_tests_max)
+        job_object = UnitJob(suite_options, {}, delay=delay)
+        self.queue_tests(self.TESTS, queue, queue_element.QueueElemRepeatTime, suite_options)
+        job_object._run(queue, self.mock_interrupt_flag())
+        self.assertEqual(job_object.total_test_num, num_repeat_tests_max * len(self.TESTS))
+        for test in self.TESTS:
+            self.assertEqual(job_object.tests[test], num_repeat_tests_max)
+            self.assertLess(job_object.tests[test], expected_time_repeat_tests)
+
+
+class UnitJob(job.Job):  # pylint: disable=too-many-instance-attributes
+    def __init__(self, suite_options, _, delay=0):  #pylint: disable=super-init-not-called
+        self.job_num = 0
+        self.logger = logging.getLogger("job_unittest")
+        self.fixture = None
+        self.hooks = []
+        self.report = None
+        self.archival = None
+        self.suite_options = suite_options
+        self.test_queue_logger = logging.getLogger("job_unittest")
+        self.total_test_num = 0
+        self.delay = delay
+        self.tests = {}
+
+    def _execute_test(self, test):
+        self.total_test_num += 1
+        if test.test_name not in self.tests:
+            self.tests[test.test_name] = 0
+        self.tests[test.test_name] += 1
+        time.sleep(self.delay)
diff --git a/buildscripts/tests/resmokelib/testing/test_queue_element.py b/buildscripts/tests/resmokelib/testing/test_queue_element.py
new file mode 100644
index 00000000000..a54ed350b8d
--- /dev/null
+++ b/buildscripts/tests/resmokelib/testing/test_queue_element.py
@@ -0,0 +1,92 @@
+"""Unit tests for the resmokelib.testing.executor module."""
+import unittest
+
+import mock
+
+from buildscripts.resmokelib.testing import queue_element
+
+# pylint: disable=missing-docstring,protected-access
+
+
+class TestQueueElemRepeatNum(unittest.TestCase):
+    def test_norequeue(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests = 1
+        queue_elem = queue_element.QueueElemRepeatNum(None, None, suite_options)
+        queue_elem.job_completed(1)
+        self.assertFalse(queue_elem.should_requeue())
+
+    def test_requeue(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests = 2
+        queue_elem = queue_element.QueueElemRepeatNum(None, None, suite_options)
+        queue_elem.job_completed(1)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(1)
+        self.assertFalse(queue_elem.should_requeue())
+
+
+class TestQueueElemRepeatTime(unittest.TestCase):
+    def test_requeue_time_only(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests_min = None
+        suite_options.num_repeat_tests_max = None
+        suite_options.time_repeat_tests_secs = 7
+        queue_elem = queue_element.QueueElemRepeatTime(None, None, suite_options)
+        job_time = 3
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertFalse(queue_elem.should_requeue())
+
+    def test_should_requeue_time_exact_avg(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests_min = None
+        suite_options.num_repeat_tests_max = None
+        suite_options.time_repeat_tests_secs = 9
+        queue_elem = queue_element.QueueElemRepeatTime(None, None, suite_options)
+        job_time = 3
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertFalse(queue_elem.should_requeue())
+
+    def test_should_requeue_time_with_min(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests_min = 3
+        suite_options.num_repeat_tests_max = None
+        suite_options.time_repeat_tests_secs = 5
+        queue_elem = queue_element.QueueElemRepeatTime(None, None, suite_options)
+        job_time = 3
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertFalse(queue_elem.should_requeue())
+
+    def test_should_requeue_time_with_max(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests_min = None
+        suite_options.num_repeat_tests_max = 2
+        suite_options.time_repeat_tests_secs = 10
+        queue_elem = queue_element.QueueElemRepeatTime(None, None, suite_options)
+        job_time = 2
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertFalse(queue_elem.should_requeue())
+
+    def test_should_requeue_time_with_min_max(self):
+        suite_options = mock.Mock()
+        suite_options.num_repeat_tests_min = 1
+        suite_options.num_repeat_tests_max = 2
+        suite_options.time_repeat_tests_secs = 10
+        queue_elem = queue_element.QueueElemRepeatTime(None, None, suite_options)
+        job_time = 1
+        queue_elem.job_completed(job_time)
+        self.assertTrue(queue_elem.should_requeue())
+        queue_elem.job_completed(job_time)
+        self.assertFalse(queue_elem.should_requeue())
diff --git a/buildscripts/tests/resmokelib/test_archival.py b/buildscripts/tests/resmokelib/utils/test_archival.py
index 43a63fa17c6..43a63fa17c6 100644
--- a/buildscripts/tests/resmokelib/test_archival.py
+++ b/buildscripts/tests/resmokelib/utils/test_archival.py