SERVER-31470 Move "run tests" logic into evergreen_run_tests.py.

9 files changed, 256 insertions, 61 deletions

diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index 456926adafb..1dcd7d77932 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -4,6 +4,8 @@ Configuration options for resmoke.py.
 
 from __future__ import absolute_import
 
+import collections
+import itertools
 import os
 import os.path
 import time
@@ -38,6 +40,7 @@ DEFAULTS = {
     "continueOnFailure": False,
     "dbpathPrefix": None,
     "dbtest": None,
+    "distroId": None,
     "dryRun": None,
     "excludeWithAnyTags": None,
     "includeWithAnyTags": None,
@@ -51,6 +54,7 @@ DEFAULTS = {
     "numClientsPerFixture": 1,
     "shellPort": None,
     "shellConnString": None,
+    "patchBuild": False,
     "repeat": 1,
     "reportFailureStatus": "fail",
     "reportFile": None,
@@ -64,13 +68,99 @@ DEFAULTS = {
     "storageEngineCacheSizeGB": None,
     "tagFile": None,
     "taskId": None,
+    "taskName": None,
     "transportLayer": None,
+    "variantName": None,
     "wiredTigerCollectionConfigString": None,
     "wiredTigerEngineConfigString": None,
     "wiredTigerIndexConfigString": None
 }
 
 
+_SuiteOptions = collections.namedtuple("_SuiteOptions", [
+    "description",
+    "fail_fast",
+    "include_tags",
+    "num_jobs",
+    "num_repeats",
+    "report_failure_status",
+])
+
+
+class SuiteOptions(_SuiteOptions):
+    """
+    A class for representing top-level options to resmoke.py that can also be set at the
+    suite-level.
+    """
+
+    INHERIT = object()
+    ALL_INHERITED = None
+
+    @classmethod
+    def combine(cls, *suite_options_list):
+        """
+        Returns a SuiteOptions instance representing the combination of all SuiteOptions in
+        'suite_options_list'.
+        """
+
+        combined_options = cls.ALL_INHERITED._asdict()
+        include_tags_list = []
+
+        for suite_options in suite_options_list:
+            for field in cls._fields:
+                value = getattr(suite_options, field)
+                if value is cls.INHERIT:
+                    continue
+
+                if field == "description":
+                    # We discard the description of each of the individual SuiteOptions when they
+                    # are combined.
+                    continue
+
+                if field == "include_tags":
+                    if value is not None:
+                        include_tags_list.append(value)
+                    continue
+
+                combined_value = combined_options[field]
+                if combined_value is not cls.INHERIT and combined_value != value:
+                    raise ValueError("Attempted to set '{}' option multiple times".format(field))
+                combined_options[field] = value
+
+        if include_tags_list:
+            combined_options["include_tags"] = {"$allOf": include_tags_list}
+
+        return cls(**combined_options)
+
+    def resolve(self):
+        """
+        Returns a SuiteOptions instance representing the options overridden at the suite-level and
+        the inherited options from the top-level.
+        """
+
+        description = None
+        include_tags = None
+        parent = dict(zip(SuiteOptions._fields, [
+            description,
+            FAIL_FAST,
+            include_tags,
+            JOBS,
+            REPEAT,
+            REPORT_FAILURE_STATUS,
+        ]))
+
+        options = self._asdict()
+        for field in SuiteOptions._fields:
+            if options[field] is SuiteOptions.INHERIT:
+                options[field] = parent[field]
+
+        return SuiteOptions(**options)
+
+
+SuiteOptions.ALL_INHERITED = SuiteOptions(**dict(zip(SuiteOptions._fields,
+                                                     itertools.repeat(SuiteOptions.INHERIT))))
+
+
 ##
 # Variables that are set by the user at the command line or with --options.
 ##
@@ -93,6 +183,22 @@ DBTEST_EXECUTABLE = None
 # actually running them).
 DRY_RUN = None
 
+# The identifier for the Evergreen distro that resmoke.py is being run on.
+EVERGREEN_DISTRO_ID = None
+
+# If true, then resmoke.py is being run as part of a patch build in Evergreen.
+EVERGREEN_PATCH_BUILD = None
+
+# The identifier for the Evergreen task that resmoke.py is being run under. If set, then the
+# Evergreen task id value will be transmitted to logkeeper when creating builds and tests.
+EVERGREEN_TASK_ID = None
+
+# The name of the Evergreen task that resmoke.py is being run for.
+EVERGREEN_TASK_NAME = None
+
+# The name of the Evergreen build variant that resmoke.py is being run on.
+EVERGREEN_VARIANT_NAME = None
+
 # If set, then any jstests that have any of the specified tags will be excluded from the suite(s).
 EXCLUDE_WITH_ANY_TAGS = None
 
@@ -163,7 +269,7 @@ SHELL_WRITE_MODE = None
 SHUFFLE = None
 
 # If true, the launching of jobs is staggered in resmoke.py.
-STAGGER_JOBS = None 
+STAGGER_JOBS = None
 
 # If set, then all mongod's started by resmoke.py and by the mongo shell will use the specified
 # storage engine.
@@ -176,11 +282,7 @@ STORAGE_ENGINE_CACHE_SIZE = None
 # The tag file to use that associates tests with tags.
 TAG_FILE = None
 
-# If set, then the Evergreen task Id value will be transmitted to logkeeper when creating builds and
-# tests.
-TASK_ID = None
-
-# IF set, then mongod/mongos's started by resmoke.py will use the specified transport layer
+# If set, then mongod/mongos's started by resmoke.py will use the specified transport layer.
 TRANSPORT_LAYER = None
 
 # If set, then all mongod's started by resmoke.py and by the mongo shell will use the specified
@@ -208,7 +310,6 @@ DEFAULT_INTEGRATION_TEST_LIST = "build/integration_tests.txt"
 
 # External files or executables, used as suite selectors, that are created during the build and
 # therefore might not be available when creating a test membership map.
-EXTERNAL_SUITE_SELECTORS = [DEFAULT_UNIT_TEST_LIST,
+EXTERNAL_SUITE_SELECTORS = (DEFAULT_UNIT_TEST_LIST,
                             DEFAULT_INTEGRATION_TEST_LIST,
-                            DEFAULT_DBTEST_EXECUTABLE]
-
+                            DEFAULT_DBTEST_EXECUTABLE)
diff --git a/buildscripts/resmokelib/core/network.py b/buildscripts/resmokelib/core/network.py
index 44e54667a67..396da4e4935 100644
--- a/buildscripts/resmokelib/core/network.py
+++ b/buildscripts/resmokelib/core/network.py
@@ -112,3 +112,15 @@ class PortAllocator(object):
         """
         next_range_start = config.BASE_PORT + ((job_num + 1) * cls._PORTS_PER_JOB)
         return next_range_start - 1
+
+    @classmethod
+    def reset(cls):
+        """
+        Resets the internal state of the PortAllocator.
+
+        This method is intended to be called each time resmoke.py starts
+        a new test suite.
+        """
+
+        with cls._NUM_USED_PORTS_LOCK:
+            cls._NUM_USED_PORTS = collections.defaultdict(int)
diff --git a/buildscripts/resmokelib/logging/buildlogger.py b/buildscripts/resmokelib/logging/buildlogger.py
index b844d5371b7..a577d64e3f0 100644
--- a/buildscripts/resmokelib/logging/buildlogger.py
+++ b/buildscripts/resmokelib/logging/buildlogger.py
@@ -235,7 +235,7 @@ class BuildloggerServer(object):
         response = handler.post(CREATE_BUILD_ENDPOINT, data={
             "builder": builder,
             "buildnum": build_num,
-            "task_id": _config.TASK_ID,
+            "task_id": _config.EVERGREEN_TASK_ID,
         })
 
         return response["id"]
@@ -255,7 +255,7 @@ class BuildloggerServer(object):
             "test_filename": test_filename,
             "command": test_command,
             "phase": self.config.get("build_phase", "unknown"),
-            "task_id": _config.TASK_ID,
+            "task_id": _config.EVERGREEN_TASK_ID,
         })
 
         return response["id"]
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index b637549c060..1f5f79cba30 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -24,6 +24,7 @@ DEST_TO_CONFIG = {
     "continue_on_failure": "continueOnFailure",
     "dbpath_prefix": "dbpathPrefix",
     "dbtest_executable": "dbtest",
+    "distro_id": "distroId",
     "dry_run": "dryRun",
     "exclude_with_any_tags": "excludeWithAnyTags",
     "include_with_any_tags": "includeWithAnyTags",
@@ -35,6 +36,7 @@ DEST_TO_CONFIG = {
     "mongos_parameters": "mongosSetParameters",
     "no_journal": "nojournal",
     "num_clients_per_fixture": "numClientsPerFixture",
+    "patch_build": "patchBuild",
     "prealloc_journal": "preallocJournal",
     "repeat": "repeat",
     "report_failure_status": "reportFailureStatus",
@@ -51,7 +53,9 @@ DEST_TO_CONFIG = {
     "storage_engine_cache_size": "storageEngineCacheSizeGB",
     "tag_file": "tagFile",
     "task_id": "taskId",
+    "task_name": "taskName",
     "transport_layer": "transportLayer",
+    "variant_name": "variantName",
     "wt_coll_config": "wiredTigerCollectionConfigString",
     "wt_engine_config": "wiredTigerEngineConfigString",
     "wt_index_config": "wiredTigerIndexConfigString"
@@ -232,9 +236,6 @@ def parse_command_line():
     parser.add_option("--tagFile", dest="tag_file", metavar="OPTIONS",
                       help="A YAML file that associates tests and tags.")
 
-    parser.add_option("--taskId", dest="task_id", metavar="TASK_ID",
-                      help="Set the Id of the Evergreen task running the tests.")
-
     parser.add_option("--wiredTigerCollectionConfigString", dest="wt_coll_config", metavar="CONFIG",
                       help="Set the WiredTiger collection configuration setting for all mongod's.")
 
@@ -248,6 +249,31 @@ def parse_command_line():
                       help="OBSOLETE: Superceded by --suites; specify --suites=SUITE path/to/test"
                            " to run a particular test under a particular suite configuration.")
 
+    evergreen_options = optparse.OptionGroup(
+        parser,
+        title="Evergreen options",
+        description=("Options used to propagate information about the Evergreen task running this"
+                     " script."))
+    parser.add_option_group(evergreen_options)
+
+    evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID",
+                                 help=("Set the identifier for the Evergreen distro running the"
+                                       " tests."))
+
+    evergreen_options.add_option("--patchBuild", action="store_true", dest="patch_build",
+                                 help=("Indicate that the Evergreen task running the tests is a"
+                                       " patch build."))
+
+    evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME",
+                                 help="Set the name of the Evergreen task running the tests.")
+
+    evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID",
+                                 help="Set the Id of the Evergreen task running the tests.")
+
+    evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME",
+                                 help=("Set the name of the Evergreen build variant running the"
+                                       " tests."))
+
     parser.set_defaults(logger_file="console",
                         dry_run="off",
                         find_suites=False,
@@ -300,6 +326,11 @@ def update_config_vars(values):
     _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix"))
     _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest"))
     _config.DRY_RUN = config.pop("dryRun")
+    _config.EVERGREEN_DISTRO_ID = config.pop("distroId")
+    _config.EVERGREEN_PATCH_BUILD = config.pop("patchBuild")
+    _config.EVERGREEN_TASK_ID = config.pop("taskId")
+    _config.EVERGREEN_TASK_NAME = config.pop("taskName")
+    _config.EVERGREEN_VARIANT_NAME = config.pop("variantName")
     _config.EXCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("excludeWithAnyTags"))
     _config.FAIL_FAST = not config.pop("continueOnFailure")
     _config.INCLUDE_WITH_ANY_TAGS = _tags_from_list(config.pop("includeWithAnyTags"))
@@ -323,7 +354,6 @@ def update_config_vars(values):
     _config.STORAGE_ENGINE = config.pop("storageEngine")
     _config.STORAGE_ENGINE_CACHE_SIZE = config.pop("storageEngineCacheSizeGB")
     _config.TAG_FILE = config.pop("tagFile")
-    _config.TASK_ID = config.pop("taskId")
     _config.TRANSPORT_LAYER = config.pop("transportLayer")
     _config.WT_COLL_CONFIG = config.pop("wiredTigerCollectionConfigString")
     _config.WT_ENGINE_CONFIG = config.pop("wiredTigerEngineConfigString")
diff --git a/buildscripts/resmokelib/reportfile.py b/buildscripts/resmokelib/reportfile.py
index 11e43018087..7dcf5623a6d 100644
--- a/buildscripts/resmokelib/reportfile.py
+++ b/buildscripts/resmokelib/reportfile.py
@@ -23,6 +23,6 @@ def write(suites):
     for suite in suites:
         reports.extend(suite.get_reports())
 
-    combined_report_dict = _report.TestReport.combine(*reports).as_dict(convert_failures=True)
+    combined_report_dict = _report.TestReport.combine(*reports).as_dict()
     with open(config.REPORT_FILE, "w") as fp:
         json.dump(combined_report_dict, fp)
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index 62dee25be7f..cc665568f05 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -15,6 +15,7 @@ from . import testcases
 from .. import config as _config
 from .. import errors
 from .. import utils
+from ..core import network
 from ..utils import queue as _queue
 
 
@@ -52,14 +53,14 @@ class TestSuiteExecutor(object):
         self._suite = suite
 
         # Only start as many jobs as we need. Note this means that the number of jobs we run may not
-        # actually be _config.JOBS.
-        jobs_to_start = _config.JOBS
+        # actually be _config.JOBS or self._suite.options.num_jobs.
+        jobs_to_start = self._suite.options.num_jobs
         num_tests = len(suite.tests)
 
         if num_tests < jobs_to_start:
-            self.logger.info("Reducing the number of jobs from %d to %d since there are only %d "
-                             "test(s) to run.",
-                _config.JOBS, num_tests, num_tests)
+            self.logger.info(
+                "Reducing the number of jobs from %d to %d since there are only %d test(s) to run.",
+                self._suite.options.num_jobs, num_tests, num_tests)
             jobs_to_start = num_tests
 
         # Must be done after getting buildlogger configuration.
@@ -82,7 +83,7 @@ class TestSuiteExecutor(object):
                 return_code = 2
                 return
 
-            num_repeats = _config.REPEAT
+            num_repeats = self._suite.options.num_repeats
             while num_repeats > 0:
                 test_queue = self._make_test_queue()
 
@@ -111,7 +112,7 @@ class TestSuiteExecutor(object):
 
                 if not report.wasSuccessful():
                     return_code = 1
-                    if _config.FAIL_FAST:
+                    if self._suite.options.fail_fast:
                         break
 
                 # Clear the report so it can be reused for the next execution.
@@ -128,6 +129,12 @@ class TestSuiteExecutor(object):
         """
         Sets up a fixture for each job.
         """
+
+        # We reset the internal state of the PortAllocator before calling job.fixture.setup() so
+        # that ports used by the fixture during a test suite run earlier can be reused during this
+        # current test suite.
+        network.PortAllocator.reset()
+
         for job in self._jobs:
             try:
                 job.fixture.setup()
@@ -260,9 +267,9 @@ class TestSuiteExecutor(object):
         fixture = self._make_fixture(job_num, job_logger)
         hooks = self._make_hooks(job_num, fixture)
 
-        report = _report.TestReport(job_logger)
+        report = _report.TestReport(job_logger, self._suite.options)
 
-        return _job.Job(job_logger, fixture, hooks, report)
+        return _job.Job(job_logger, fixture, hooks, report, self._suite.options)
 
     def _make_test_queue(self):
         """
diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index 4d17152b17d..9841c071ce7 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -9,7 +9,6 @@ import sys
 
 from .. import config
 from .. import errors
-from .. import logging
 from ..utils import queue as _queue
 
 
@@ -18,7 +17,7 @@ class Job(object):
     Runs tests from a queue.
     """
 
-    def __init__(self, logger, fixture, hooks, report):
+    def __init__(self, logger, fixture, hooks, report, suite_options):
         """
         Initializes the job with the specified fixture and custom
         behaviors.
@@ -28,6 +27,7 @@ class Job(object):
         self.fixture = fixture
         self.hooks = hooks
         self.report = report
+        self.suite_options = suite_options
 
     def __call__(self, queue, interrupt_flag, teardown_flag=None):
         """
@@ -98,7 +98,7 @@ class Job(object):
         self._run_hooks_before_tests(test)
 
         test(self.report)
-        if config.FAIL_FAST and not self.report.wasSuccessful():
+        if self.suite_options.fail_fast and not self.report.wasSuccessful():
             self.logger.info("%s failed, so stopping..." % (test.shortDescription()))
             raise errors.StopExecution("%s failed" % (test.shortDescription()))
 
@@ -137,7 +137,7 @@ class Job(object):
             self.logger.exception("%s marked as a failure by a hook's before_test.",
                                   test.shortDescription())
             self._fail_test(test, sys.exc_info(), return_code=1)
-            if config.FAIL_FAST:
+            if self.suite_options.fail_fast:
                 raise errors.StopExecution("A hook's before_test failed")
 
         except:
@@ -171,7 +171,7 @@ class Job(object):
             self.logger.exception("%s marked as a failure by a hook's after_test.",
                                   test.shortDescription())
             self.report.setFailure(test, return_code=1)
-            if config.FAIL_FAST:
+            if self.suite_options.fail_fast:
                 raise errors.StopExecution("A hook's after_test failed")
 
         except:
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index d9270528440..71a8e5e60c3 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -19,7 +19,7 @@ class TestReport(unittest.TestResult):
     Records test status and timing information.
     """
 
-    def __init__(self, job_logger):
+    def __init__(self, job_logger, suite_options):
         """
         Initializes the TestReport with the buildlogger configuration.
         """
@@ -27,6 +27,7 @@ class TestReport(unittest.TestResult):
         unittest.TestResult.__init__(self)
 
         self.job_logger = job_logger
+        self.suite_options = suite_options
 
         self._lock = threading.Lock()
 
@@ -45,7 +46,8 @@ class TestReport(unittest.TestResult):
 
         # TestReports that are used when running tests need a JobLogger but combined reports don't
         # use the logger.
-        combined_report = cls(logging.loggers.EXECUTOR_LOGGER)
+        combined_report = cls(logging.loggers.EXECUTOR_LOGGER,
+                              _config.SuiteOptions.ALL_INHERITED.resolve())
         combining_time = time.time()
 
         for report in reports:
@@ -64,7 +66,14 @@ class TestReport(unittest.TestResult):
                     if test_info.status is None or test_info.return_code is None:
                         # Mark the test as having timed out if it was interrupted. It might have
                         # passed if the suite ran to completion, but we wouldn't know for sure.
+                        #
+                        # Until EVG-1536 is completed, we shouldn't distinguish between failures and
+                        # interrupted tests in the report.json file. In Evergreen, the behavior to
+                        # sort tests with the "timeout" test status after tests with the "pass" test
+                        # status effectively hides interrupted tests from the test results sidebar
+                        # unless sorting by the time taken.
                         test_info.status = "timeout"
+                        test_info.evergreen_status = "fail"
                         test_info.return_code = -2
 
                     # TestReport.stopTest() may not have been called.
@@ -154,8 +163,10 @@ class TestReport(unittest.TestResult):
         with self._lock:
             self.num_errored += 1
 
+            # We don't distinguish between test failures and Python errors in Evergreen.
             test_info = self._find_test_info(test)
             test_info.status = "error"
+            test_info.evergreen_status = "fail"
             test_info.return_code = test.return_code
 
     def setError(self, test):
@@ -168,7 +179,9 @@ class TestReport(unittest.TestResult):
             if test_info.end_time is None:
                 raise ValueError("stopTest was not called on %s" % (test.basename()))
 
+            # We don't distinguish between test failures and Python errors in Evergreen.
             test_info.status = "error"
+            test_info.evergreen_status = "fail"
             test_info.return_code = 2
 
         # Recompute number of success, failures, and errors.
@@ -190,6 +203,12 @@ class TestReport(unittest.TestResult):
 
             test_info = self._find_test_info(test)
             test_info.status = "fail"
+            if test_info.dynamic:
+                # Dynamic tests are used for data consistency checks, so the failures are never
+                # silenced.
+                test_info.evergreen_status = "fail"
+            else:
+                test_info.evergreen_status = self.suite_options.report_failure_status
             test_info.return_code = test.return_code
 
     def setFailure(self, test, return_code=1):
@@ -203,6 +222,12 @@ class TestReport(unittest.TestResult):
                 raise ValueError("stopTest was not called on %s" % (test.basename()))
 
             test_info.status = "fail"
+            if test_info.dynamic:
+                # Dynamic tests are used for data consistency checks, so the failures are never
+                # silenced.
+                test_info.evergreen_status = "fail"
+            else:
+                test_info.evergreen_status = self.suite_options.report_failure_status
             test_info.return_code = return_code
 
         # Recompute number of success, failures, and errors.
@@ -223,6 +248,7 @@ class TestReport(unittest.TestResult):
 
             test_info = self._find_test_info(test)
             test_info.status = "pass"
+            test_info.evergreen_status = "pass"
             test_info.return_code = test.return_code
 
     def wasSuccessful(self):
@@ -249,8 +275,7 @@ class TestReport(unittest.TestResult):
         """
 
         with self._lock:
-            return [test_info for test_info in self.test_infos
-                    if test_info.status in ("fail", "silentfail")]
+            return [test_info for test_info in self.test_infos if test_info.status == "fail"]
 
     def get_errored(self):
         """
@@ -270,40 +295,19 @@ class TestReport(unittest.TestResult):
         with self._lock:
             return [test_info for test_info in self.test_infos if test_info.status == "timeout"]
 
-    def as_dict(self, convert_failures=False):
+    def as_dict(self):
         """
         Return the test result information as a dictionary.
 
         Used to create the report.json file.
-
-        If 'convert_failures' is true, then "error" and "fail" test statuses are replaced with
-        _config.REPORT_FAILURE_STATUS in the returned dictionary.
         """
 
         results = []
         with self._lock:
             for test_info in self.test_infos:
-                status = test_info.status
-                if convert_failures:
-                    if status == "error" or status == "fail":
-                        # Don't distinguish between failures and errors.
-                        if test_info.dynamic:
-                            # Dynamic tests are used for data consistency checks, so the failures
-                            # are not silenced.
-                            status = "fail"
-                        else:
-                            status = _config.REPORT_FAILURE_STATUS
-                    elif status == "timeout":
-                        # Until EVG-1536 is completed, we shouldn't distinguish between failures and
-                        # interrupted tests in the report.json file. In Evergreen, the behavior to
-                        # sort tests with the "timeout" test status after tests with the "pass" test
-                        # status effectively hides interrupted tests from the test results sidebar
-                        # unless sorting by the time taken.
-                        status = "fail"
-
                 result = {
                     "test_file": test_info.test_id,
-                    "status": status,
+                    "status": test_info.evergreen_status,
                     "exit_code": test_info.return_code,
                     "start": test_info.start_time,
                     "end": test_info.end_time,
@@ -329,13 +333,14 @@ class TestReport(unittest.TestResult):
         Used when combining reports instances.
         """
 
-        report = cls(logging.loggers.EXECUTOR_LOGGER)
+        report = cls(logging.loggers.EXECUTOR_LOGGER, _config.SuiteOptions.ALL_INHERITED.resolve())
         for result in report_dict["results"]:
             # By convention, dynamic tests are named "<basename>:<hook name>".
             is_dynamic = ":" in result["test_file"]
             test_info = _TestInfo(result["test_file"], is_dynamic)
             test_info.url_endpoint = result.get("url")
             test_info.status = result["status"]
+            test_info.evergreen_status = test_info.status
             test_info.return_code = result["exit_code"]
             test_info.start_time = result["start"]
             test_info.end_time = result["end"]
@@ -403,5 +408,6 @@ class _TestInfo(object):
         self.start_time = None
         self.end_time = None
         self.status = None
+        self.evergreen_status = None
         self.return_code = None
         self.url_endpoint = None
diff --git a/buildscripts/resmokelib/testing/suite.py b/buildscripts/resmokelib/testing/suite.py
index 29276f5aa18..132a2d70d9d 100644
--- a/buildscripts/resmokelib/testing/suite.py
+++ b/buildscripts/resmokelib/testing/suite.py
@@ -31,7 +31,7 @@ class Suite(object):
     A suite of tests of a particular kind (e.g. C++ unit tests, dbtests, jstests).
     """
 
-    def __init__(self, suite_name, suite_config):
+    def __init__(self, suite_name, suite_config, suite_options=_config.SuiteOptions.ALL_INHERITED):
         """
         Initializes the suite with the specified name and configuration.
         """
@@ -39,6 +39,7 @@ class Suite(object):
 
         self._suite_name = suite_name
         self._suite_config = suite_config
+        self._suite_options = suite_options
 
         self.test_kind = self.get_test_kind_config()
         self.tests = self._get_tests_for_kind(self.test_kind)
@@ -83,11 +84,38 @@ class Suite(object):
         """
         return self._suite_name
 
+    def get_display_name(self):
+        """
+        Returns the name of the test suite with a unique identifier for its SuiteOptions.
+        """
+
+        if self.options.description is None:
+            return self.get_name()
+
+        return "{} ({})".format(self.get_name(), self.options.description)
+
     def get_selector_config(self):
         """
         Returns the "selector" section of the YAML configuration.
         """
-        return self._suite_config["selector"]
+
+        selector = self._suite_config["selector"].copy()
+
+        if self.options.include_tags is not None:
+            if "include_tags" in selector:
+                selector["include_tags"] = {"$allOf": [
+                    selector["include_tags"],
+                    self.options.include_tags,
+                ]}
+            elif "exclude_tags" in selector:
+                selector["exclude_tags"] = {"$anyOf": [
+                    selector["exclude_tags"],
+                    {"$not": self.options.include_tags},
+                ]}
+            else:
+                selector["include_tags"] = self.options.include_tags
+
+        return selector
 
     def get_executor_config(self):
         """
@@ -101,6 +129,17 @@ class Suite(object):
         """
         return self._suite_config["test_kind"]
 
+    @property
+    def options(self):
+        return self._suite_options.resolve()
+
+    def with_options(self, suite_options):
+        """
+        Returns a Suite instance with the specified resmokelib.config.SuiteOptions.
+        """
+
+        return Suite(self._suite_name, self._suite_config, suite_options)
+
     @synchronized
     def record_suite_start(self):
         """
@@ -299,7 +338,7 @@ class Suite(object):
         for suite in suites:
             suite_sb = []
             suite.summarize(suite_sb)
-            sb.append("    %s: %s" % (suite.get_name(), "\n    ".join(suite_sb)))
+            sb.append("    %s: %s" % (suite.get_display_name(), "\n    ".join(suite_sb)))
 
         logger.info("=" * 80)
         logger.info("\n".join(sb))