SERVER-26884 Support archiving data files in Evergreen on test failure

14 files changed, 368 insertions, 87 deletions

diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py
index 7aa13277384..3fa24bd6553 100755
--- a/buildscripts/resmoke.py
+++ b/buildscripts/resmoke.py
@@ -54,10 +54,19 @@ def _execute_suite(suite, logging_config):
             logger.info("Skipping %ss, no tests to run", group.test_kind)
             continue
 
+        archive = None
+        if resmokelib.config.ARCHIVE_FILE:
+            archive = resmokelib.utils.archival.Archival(
+                archival_json_file=resmokelib.config.ARCHIVE_FILE,
+                execution=resmokelib.config.EVERGREEN_EXECUTION,
+                limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB,
+                limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS,
+                logger=logger)
         group_config = suite.get_executor_config().get(group.test_kind, {})
         executor = resmokelib.testing.executor.TestGroupExecutor(logger,
                                                                  group,
                                                                  logging_config,
+                                                                 archive_instance=archive,
                                                                  **group_config)
 
         try:
@@ -73,6 +82,9 @@ def _execute_suite(suite, logging_config):
                              group.test_kind, suite.get_name())
             suite.return_code = 2
             return False
+        finally:
+            if archive:
+                archive.exit()
 
 
 def _log_summary(logger, suites, time_taken):
diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py
index b75753398a5..8f636ddbadd 100644
--- a/buildscripts/resmokelib/config.py
+++ b/buildscripts/resmokelib/config.py
@@ -33,14 +33,20 @@ MONGO_RUNNER_SUBDIR = "mongorunner"
 
 # Names below correspond to how they are specified via the command line or in the options YAML file.
 DEFAULTS = {
+    "archiveFile": None,
+    "archiveLimitMb": 5000,
+    "archiveLimitTests": 10,
     "basePort": 20000,
     "buildloggerUrl": "https://logkeeper.mongodb.org",
     "continueOnFailure": False,
     "dbpathPrefix": None,
     "dbtest": None,
+    "distroId": None,
     "dryRun": None,
     "excludeWithAllTags": None,
     "excludeWithAnyTags": None,
+    "executionNumber": 0,
+    "gitRevision": None,
     "includeWithAllTags": None,
     "includeWithAnyTags": None,
     "jobs": 1,
@@ -51,6 +57,7 @@ DEFAULTS = {
     "mongosSetParameters": None,
     "nojournal": False,
     "numClientsPerFixture": 1,
+    "projectName": "mongodb-mongo-master",
     "repeat": 1,
     "reportFile": None,
     "seed": long(time.time() * 256),  # Taken from random.py code in Python 2.7.
@@ -59,6 +66,9 @@ DEFAULTS = {
     "shuffle": False,
     "storageEngine": None,
     "storageEngineCacheSizeGB": None,
+    "taskId": None,
+    "taskName": None,
+    "variantName": None,
     "wiredTigerCollectionConfigString": None,
     "wiredTigerEngineConfigString": None,
     "wiredTigerIndexConfigString": None
@@ -69,6 +79,15 @@ DEFAULTS = {
 # Variables that are set by the user at the command line or with --options.
 ##
 
+# The name of the archive JSON file used to associate S3 archives to an Evergreen task.
+ARCHIVE_FILE = None
+
+# The limit size of all archive files for an Evergreen task.
+ARCHIVE_LIMIT_MB = None
+
+# The limit number of tests to archive for an Evergreen task.
+ARCHIVE_LIMIT_TESTS = None
+
 # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the
 # mongo shell.
 BASE_PORT = None
@@ -87,6 +106,28 @@ DBTEST_EXECUTABLE = None
 # actually running them).
 DRY_RUN = None
 
+# The identifier for the Evergreen distro that resmoke.py is being run on.
+EVERGREEN_DISTRO_ID = None
+
+# The number of the Evergreen execution that resmoke.py is being run on.
+EVERGREEN_EXECUTION = None
+
+# The name of the Evergreen project that resmoke.py is being run on.
+EVERGREEN_PROJECT_NAME = None
+
+# The git revision of the Evergreen task that resmoke.py is being run on.
+EVERGREEN_REVISION = None
+
+# The identifier for the Evergreen task that resmoke.py is being run under. If set, then the
+# Evergreen task id value will be transmitted to logkeeper when creating builds and tests.
+EVERGREEN_TASK_ID = None
+
+# The name of the Evergreen task that resmoke.py is being run for.
+EVERGREEN_TASK_NAME = None
+
+# The name of the Evergreen build variant that resmoke.py is being run on.
+EVERGREEN_VARIANT_NAME = None
+
 # If set, then any jstests that have all of the specified tags will be excluded from the suite(s).
 EXCLUDE_WITH_ALL_TAGS = None
 
@@ -177,6 +218,9 @@ WT_INDEX_CONFIG = None
 # Internally used configuration options that aren't exposed to the user
 ##
 
+# S3 Bucket to upload archive files.
+ARCHIVE_BUCKET = "mongodatafiles"
+
 # Default sort order for test execution. Will only be changed if --suites wasn't specified.
 ORDER_TESTS_BY_NAME = True
 
diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py
index 870e0e9145c..59b4585a273 100644
--- a/buildscripts/resmokelib/parser.py
+++ b/buildscripts/resmokelib/parser.py
@@ -18,14 +18,20 @@ from .. import resmokeconfig
 # Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options
 # YAML configuration file. Most should only be converting from snake_case to camelCase.
 DEST_TO_CONFIG = {
+    "archive_file": "archiveFile",
+    "archive_limit_mb": "archiveLimitMb",
+    "archive_limit_tests": "archiveLimitTests",
     "base_port": "basePort",
     "buildlogger_url": "buildloggerUrl",
     "continue_on_failure": "continueOnFailure",
     "dbpath_prefix": "dbpathPrefix",
     "dbtest_executable": "dbtest",
+    "distro_id": "distroId",
     "dry_run": "dryRun",
     "exclude_with_all_tags": "excludeWithAllTags",
     "exclude_with_any_tags": "excludeWithAnyTags",
+    "execution_number": "executionNumber",
+    "git_revision": "gitRevision",
     "include_with_all_tags": "includeWithAllTags",
     "include_with_any_tags": "includeWithAnyTags",
     "jobs": "jobs",
@@ -37,6 +43,7 @@ DEST_TO_CONFIG = {
     "no_journal": "nojournal",
     "num_clients_per_fixture": "numClientsPerFixture",
     "prealloc_journal": "preallocJournal",
+    "project_name": "projectName",
     "repeat": "repeat",
     "report_file": "reportFile",
     "seed": "seed",
@@ -45,6 +52,9 @@ DEST_TO_CONFIG = {
     "shuffle": "shuffle",
     "storage_engine": "storageEngine",
     "storage_engine_cache_size": "storageEngineCacheSizeGB",
+    "task_id": "taskId",
+    "task_name": "taskName",
+    "variant_name": "variantName",
     "wt_coll_config": "wiredTigerCollectionConfigString",
     "wt_engine_config": "wiredTigerEngineConfigString",
     "wt_index_config": "wiredTigerIndexConfigString"
@@ -79,6 +89,23 @@ def parse_command_line():
     parser.add_option("--options", dest="options_file", metavar="OPTIONS",
                       help="A YAML file that specifies global options to resmoke.py.")
 
+    parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE",
+                      help=("Sets the archive file name for the Evergreen task running the tests."
+                            " The archive file is JSON format containing a list of tests that were"
+                            " successfully archived to S3. If unspecified, no data files from tests"
+                            " will be archived in S3. Tests can be designated for archival in the"
+                            " task suite configuration file."))
+
+    parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb",
+                      metavar="ARCHIVE_LIMIT_MB",
+                      help=("Sets the limit (in MB) for archived files to S3. A value of 0"
+                            " indicates there is no limit."))
+
+    parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests",
+                      metavar="ARCHIVE_LIMIT_TESTS",
+                      help=("Sets the maximum number of tests to archive to S3. A value"
+                            " of 0 indicates there is no limit."))
+
     parser.add_option("--basePort", dest="base_port", metavar="PORT",
                       help=("The starting port number to use for mongod and mongos processes"
                             " spawned by resmoke.py or the tests themselves. Each fixture and Job"
@@ -209,6 +236,42 @@ def parse_command_line():
     parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG",
                       help="Set the WiredTiger index configuration setting for all mongod's.")
 
+
+    evergreen_options = optparse.OptionGroup(
+        parser, title="Evergreen options",
+        description=("Options used to propagate information about the Evergreen task running this"
+                     " script."))
+    parser.add_option_group(evergreen_options)
+
+    evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID",
+                                 help="Sets the build ID of the task.")
+
+    evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID",
+                                 help=("Sets the identifier for the Evergreen distro running the"
+                                       " tests."))
+
+    evergreen_options.add_option("--executionNumber", type="int", dest="execution_number",
+                                 metavar="EXECUTION_NUMBER",
+                                 help=("Sets the number for the Evergreen execution running the"
+                                       " tests."))
+
+    evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION",
+                                 help=("Sets the git revision for the Evergreen task running the"
+                                       " tests."))
+
+    evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME",
+                                 help=("Sets the name of the Evergreen project running the tests."))
+
+    evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME",
+                                 help="Sets the name of the Evergreen task running the tests.")
+
+    evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID",
+                                 help="Sets the Id of the Evergreen task running the tests.")
+
+    evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME",
+                                 help=("Sets the name of the Evergreen build variant running the"
+                                       " tests."))
+
     parser.set_defaults(executor_file="with_server",
                         logger_file="console",
                         dry_run="off",
@@ -237,11 +300,21 @@ def update_config_vars(values):
         if values[dest] is not None:
             config[config_var] = values[dest]
 
+    _config.ARCHIVE_FILE = config.pop("archiveFile")
+    _config.ARCHIVE_LIMIT_MB = config.pop("archiveLimitMb")
+    _config.ARCHIVE_LIMIT_TESTS = config.pop("archiveLimitTests")
     _config.BASE_PORT = int(config.pop("basePort"))
     _config.BUILDLOGGER_URL = config.pop("buildloggerUrl")
     _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix"))
     _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest"))
     _config.DRY_RUN = config.pop("dryRun")
+    _config.EVERGREEN_DISTRO_ID = config.pop("distroId")
+    _config.EVERGREEN_EXECUTION = config.pop("executionNumber")
+    _config.EVERGREEN_PROJECT_NAME = config.pop("projectName")
+    _config.EVERGREEN_REVISION = config.pop("gitRevision")
+    _config.EVERGREEN_TASK_ID = config.pop("taskId")
+    _config.EVERGREEN_TASK_NAME = config.pop("taskName")
+    _config.EVERGREEN_VARIANT_NAME = config.pop("variantName")
     _config.EXCLUDE_WITH_ALL_TAGS = config.pop("excludeWithAllTags")
     _config.EXCLUDE_WITH_ANY_TAGS = config.pop("excludeWithAnyTags")
     _config.FAIL_FAST = not config.pop("continueOnFailure")
diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py
index 438a06b9c8e..e88eed6bc02 100644
--- a/buildscripts/resmokelib/testing/executor.py
+++ b/buildscripts/resmokelib/testing/executor.py
@@ -8,6 +8,7 @@ import threading
 import time
 
 from . import fixtures
+from . import hook_test_archival as archival
 from . import hooks as _hooks
 from . import job as _job
 from . import report as _report
@@ -35,7 +36,9 @@ class TestGroupExecutor(object):
                  logging_config,
                  config=None,
                  fixture=None,
-                 hooks=None):
+                 hooks=None,
+                 archive_instance=None,
+                 archive=None):
         """
         Initializes the TestGroupExecutor with the test group to run.
         """
@@ -49,6 +52,11 @@ class TestGroupExecutor(object):
         self.hooks_config = utils.default_if_none(hooks, [])
         self.test_config = utils.default_if_none(config, {})
 
+        self.archival = None
+        if archive_instance:
+            self.archival = archival.HookTestArchival(test_group, self.hooks_config,
+                                                      archive_instance, archive)
+
         self._test_group = test_group
 
         self._using_buildlogger = logging.config.using_buildlogger(logging_config)
@@ -299,7 +307,7 @@ class TestGroupExecutor(object):
                                     build_id=build_id,
                                     build_config=build_config)
 
-        return _job.Job(logger, fixture, hooks, report)
+        return _job.Job(logger, fixture, hooks, report, self.archival)
 
     def _make_test_queue(self):
         """
diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py
index fb444cfe097..66924d3e3d9 100644
--- a/buildscripts/resmokelib/testing/fixtures/masterslave.py
+++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py
@@ -62,6 +62,9 @@ class MasterSlaveFixture(interface.ReplFixture):
             self.slave = self._new_mongod_slave()
         self.slave.setup()
 
+    def get_dbpath(self):
+        return self._dbpath_prefix
+
     def await_ready(self):
         self.master.await_ready()
         self.slave.await_ready()
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 0ec3de30280..26079cd9e02 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -186,6 +186,9 @@ class ReplicaSetFixture(interface.ReplFixture):
                     raise
                 time.sleep(5)  # Wait a little bit before trying again.
 
+    def get_dbpath(self):
+        return self._dbpath_prefix
+
     def await_ready(self):
         self._await_primary()
         self._await_secondaries()
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index 2e2db535d6d..ca788cc8dde 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -88,6 +88,9 @@ class ShardedClusterFixture(interface.Fixture):
         for shard in self.shards:
             shard.setup()
 
+    def get_dbpath(self):
+        return self._dbpath_prefix
+
     def await_ready(self):
         # Wait for the config server
         if self.configsvr is not None:
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index bc69775c285..ffc94ceb4aa 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -85,6 +85,9 @@ class MongoDFixture(interface.Fixture):
 
         self.mongod = mongod
 
+    def get_dbpath(self):
+        return self._dbpath
+
     def await_ready(self):
         deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS
 
diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py
new file mode 100644
index 00000000000..1873485cdbf
--- /dev/null
+++ b/buildscripts/resmokelib/testing/hook_test_archival.py
@@ -0,0 +1,116 @@
+"""
+Enables supports for archiving tests or hooks.
+"""
+
+from __future__ import absolute_import
+
+import os
+import threading
+
+from .. import config
+from .. import utils
+from ..utils import globstar
+
+
+class HookTestArchival(object):
+    """
+    Archives hooks and tests to S3.
+    """
+
+    def __init__(self, suite, hooks, archive_instance, archive_config):
+        self.archive_instance = archive_instance
+        archive_config = utils.default_if_none(archive_config, {})
+
+        self.on_success = archive_config.get("on_success", False)
+
+        self.tests = []
+        if "tests" in archive_config:
+            # 'tests' is either a list of tests to archive or a bool (archive all if True).
+            if not isinstance(archive_config["tests"], bool):
+                for test in archive_config["tests"]:
+                    self.tests += globstar.glob(test)
+            elif archive_config["tests"]:
+                self.tests = suite.tests
+
+        self.hooks = []
+        if "hooks" in archive_config:
+            # 'hooks' is either a list of hooks to archive or a bool (archive all if True).
+            if not isinstance(archive_config["hooks"], bool):
+                self.hooks = archive_config["hooks"]
+            elif archive_config["hooks"]:
+                for hook in hooks:
+                    self.hooks.append(hook["class"])
+
+        self._tests_repeat = {}
+        self._lock = threading.Lock()
+
+    def _should_archive(self, success):
+        """ Return True if failed test or 'on_success' is True. """
+        return not success or self.on_success
+
+    def _archive_hook(self, logger, hook, test, success):
+        """ Helper to archive hooks. """
+        hook_match = hook.__class__.__name__ in self.hooks
+        if not hook_match or not self._should_archive(success):
+            return
+
+        test_name = "{}:{}".format(test.short_name(), hook.__class__.__name__)
+        self._archive_hook_or_test(logger, test_name, test)
+
+    def _archive_test(self, logger, test, success):
+        """ Helper to archive tests. """
+        test_name = test.test_name
+        test_match = False
+        for arch_test in self.tests:
+            # Ensure that the test_name is in the same format as the arch_test.
+            if os.path.normpath(test_name) == os.path.normpath(arch_test):
+                test_match = True
+                break
+        if not test_match or not self._should_archive(success):
+            return
+
+        self._archive_hook_or_test(logger, test_name, test)
+
+    def archive(self, logger, test, success, hook=None):
+        """ Archives data files for hooks or tests. """
+        if not config.ARCHIVE_FILE or not self.archive_instance:
+            return
+        if hook:
+            self._archive_hook(logger, hook, test, success)
+        else:
+            self._archive_test(logger, test, success)
+
+    def _archive_hook_or_test(self, logger, test_name, test):
+        """ Trigger archive of data files for a test or hook. """
+
+        with self._lock:
+            # Test repeat number is how many times the particular test has been archived.
+            if test_name not in self._tests_repeat:
+                self._tests_repeat[test_name] = 0
+            else:
+                self._tests_repeat[test_name] += 1
+        logger.info("Archiving data files for test %s", test_name)
+        # Normalize test path from a test or hook name.
+        test_path = \
+            test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_")
+        file_name = "mongo-data-{}-{}-{}-{}.tgz".format(
+            config.EVERGREEN_TASK_ID,
+            test_path,
+            config.EVERGREEN_EXECUTION,
+            self._tests_repeat[test_name])
+        # Retrieve root directory for all dbPaths from fixture.
+        input_files = test.fixture.get_dbpath()
+        s3_bucket = config.ARCHIVE_BUCKET
+        s3_path = "{}/{}/{}/datafiles/{}".format(
+            config.EVERGREEN_PROJECT_NAME,
+            config.EVERGREEN_VARIANT_NAME,
+            config.EVERGREEN_REVISION,
+            file_name)
+        display_name = "Data files {} - Execution {} Repetition {}".format(
+            test_name,
+            config.EVERGREEN_EXECUTION,
+            self._tests_repeat[test_name])
+        status, message = self.archive_instance.archive_files_to_s3(
+            display_name, input_files, s3_bucket, s3_path)
+        if status:
+            logger.warning("Archive failed for %s: %s", test_name, message)
diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py
index 4d17152b17d..c8be906dd3a 100644
--- a/buildscripts/resmokelib/testing/job.py
+++ b/buildscripts/resmokelib/testing/job.py
@@ -18,7 +18,7 @@ class Job(object):
     Runs tests from a queue.
     """
 
-    def __init__(self, logger, fixture, hooks, report):
+    def __init__(self, logger, fixture, hooks, report, archival):
         """
         Initializes the job with the specified fixture and custom
         behaviors.
@@ -28,6 +28,7 @@ class Job(object):
         self.fixture = fixture
         self.hooks = hooks
         self.report = report
+        self.archival = archival
 
     def __call__(self, queue, interrupt_flag, teardown_flag=None):
         """
@@ -98,20 +99,38 @@ class Job(object):
         self._run_hooks_before_tests(test)
 
         test(self.report)
-        if config.FAIL_FAST and not self.report.wasSuccessful():
-            self.logger.info("%s failed, so stopping..." % (test.shortDescription()))
-            raise errors.StopExecution("%s failed" % (test.shortDescription()))
-
-        if not self.fixture.is_running():
-            self.logger.error("%s marked as a failure because the fixture crashed during the test.",
-                              test.shortDescription())
-            self.report.setFailure(test, return_code=2)
-            # Always fail fast if the fixture fails.
-            raise errors.StopExecution("%s not running after %s" %
-                                       (self.fixture, test.shortDescription()))
+        try:
+            if config.FAIL_FAST and not self.report.wasSuccessful():
+                self.logger.info("%s failed, so stopping..." % (test.shortDescription()))
+                raise errors.StopExecution("%s failed" % (test.shortDescription()))
+
+            if not self.fixture.is_running():
+                self.logger.error(
+                    "%s marked as a failure because the fixture crashed during the test.",
+                    test.shortDescription())
+                self.report.setFailure(test, return_code=2)
+                # Always fail fast if the fixture fails.
+                raise errors.StopExecution("%s not running after %s" %
+                                           (self.fixture, test.shortDescription()))
+
+        finally:
+            success = self.report.find_test_info(test).status == "pass"
+            self.archival.archive(self.logger, test, success)
+            if self.archival:
+                self.archival.archive(self.logger, test, success)
 
         self._run_hooks_after_tests(test)
 
+    def _run_hook(self, hook, hook_function, test):
+        """Provide helper to run hook and archival."""
+        try:
+            success = False
+            hook_function(test, self.report)
+            success = True
+        finally:
+            if self.archival:
+                self.archival.archive(self.logger, test, success, hook=hook)
+
     def _run_hooks_before_tests(self, test):
         """
         Runs the before_test method on each of the hooks.
@@ -122,7 +141,7 @@ class Job(object):
 
         try:
             for hook in self.hooks:
-                hook.before_test(test, self.report)
+                self._run_hook(hook, hook.before_test, test)
 
         except errors.StopExecution:
             raise
@@ -156,7 +175,7 @@ class Job(object):
         """
         try:
             for hook in self.hooks:
-                hook.after_test(test, self.report)
+                self._run_hook(hook, hook.after_test, test)
 
         except errors.StopExecution:
             raise
diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py
index b9cccccc14f..e3c68393a4d 100644
--- a/buildscripts/resmokelib/testing/report.py
+++ b/buildscripts/resmokelib/testing/report.py
@@ -147,7 +147,7 @@ class TestReport(unittest.TestResult):
         unittest.TestResult.stopTest(self, test)
 
         with self._lock:
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             test_info.end_time = time.time()
 
         time_taken = test_info.end_time - test_info.start_time
@@ -177,7 +177,7 @@ class TestReport(unittest.TestResult):
         with self._lock:
             self.num_errored += 1
 
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             test_info.status = "error"
             test_info.return_code = test.return_code
 
@@ -187,7 +187,7 @@ class TestReport(unittest.TestResult):
         """
 
         with self._lock:
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             if test_info.end_time is None:
                 raise ValueError("stopTest was not called on %s" % (test.basename()))
 
@@ -211,7 +211,7 @@ class TestReport(unittest.TestResult):
         with self._lock:
             self.num_failed += 1
 
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             test_info.status = "fail"
             test_info.return_code = test.return_code
 
@@ -221,7 +221,7 @@ class TestReport(unittest.TestResult):
         """
 
         with self._lock:
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             if test_info.end_time is None:
                 raise ValueError("stopTest was not called on %s" % (test.basename()))
 
@@ -244,7 +244,7 @@ class TestReport(unittest.TestResult):
         with self._lock:
             self.num_succeeded += 1
 
-            test_info = self._find_test_info(test)
+            test_info = self.find_test_info(test)
             test_info.status = "pass"
             test_info.return_code = test.return_code
 
@@ -352,7 +352,7 @@ class TestReport(unittest.TestResult):
         # protecting it with the lock.
         self.__original_loggers = {}
 
-    def _find_test_info(self, test):
+    def find_test_info(self, test):
         """
         Returns the status and timing information associated with
         'test'.
diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py
index df387cc3323..bdd169af132 100644
--- a/buildscripts/resmokelib/utils/__init__.py
+++ b/buildscripts/resmokelib/utils/__init__.py
@@ -9,6 +9,8 @@ import os.path
 import pymongo
 import yaml
 
+from . import archival
+
 
 def default_if_none(value, default):
     return value if value is not None else default
diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py
index baafe90778c..9d31e053846 100644
--- a/buildscripts/resmokelib/utils/archival.py
+++ b/buildscripts/resmokelib/utils/archival.py
@@ -9,11 +9,14 @@ import collections
 import json
 import math
 import os
+import sys
 import tarfile
 import tempfile
 import threading
 import time
 
+_IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin"
+
 UploadArgs = collections.namedtuple(
     "UploadArgs",
     ["archival_file",
@@ -128,6 +131,10 @@ class Archival(object):
         Returns status and message, where message contains information if status is non-0.
         """
 
+        # TODO: Support archival on Windows (SERVER-33144).
+        if _IS_WINDOWS:
+            return 1, "Archival not supported on Windows"
+
         start_time = time.time()
         with self._lock:
             if not input_files:
@@ -135,10 +142,10 @@ class Archival(object):
                 message = "No input_files specified"
             elif self.limit_size_mb and self.size_mb >= self.limit_size_mb:
                 status = 1
-                message = "Files not archived, limit size {}MB reached".format(self.limit_size_mb)
+                message = "Files not archived, {}MB size limit reached".format(self.limit_size_mb)
             elif self.limit_files and self.num_files >= self.limit_files:
                 status = 1
-                message = "Files not archived, limit files {} reached".format(self.limit_files)
+                message = "Files not archived, {} file limit reached".format(self.limit_files)
             else:
                 status, message, file_size_mb = self._archive_files(
                     display_name,
@@ -203,7 +210,10 @@ class Archival(object):
                 logger.exception("Upload to S3 error %s", err)
 
             if upload_args.delete_file:
-                os.remove(upload_args.local_file)
+                try:
+                    os.remove(upload_args.local_file)
+                except Exception as err:
+                    logger.exception("Upload to S3 file removal error %s", err)
 
             remote_file = "https://s3.amazonaws.com/{}/{}".format(
                 upload_args.s3_bucket, upload_args.s3_path)
@@ -232,28 +242,27 @@ class Archival(object):
         size_mb = 0
 
         # Tar/gzip to a temporary file.
-        temp_file = tempfile.NamedTemporaryFile(suffix=".tgz", delete=False)
-        local_file = temp_file.name
+        _, temp_file = tempfile.mkstemp(suffix=".tgz")
 
         # Check if there is sufficient space for the temporary tgz file.
-        if file_list_size(input_files) > free_space(local_file):
-            os.remove(local_file)
+        if file_list_size(input_files) > free_space(temp_file):
+            os.remove(temp_file)
             return 1, "Insufficient space for {}".format(message), 0
 
         try:
-            with tarfile.open(local_file, "w:gz") as tar_handle:
+            with tarfile.open(temp_file, "w:gz") as tar_handle:
                 for input_file in input_files:
                     tar_handle.add(input_file)
         except (IOError, tarfile.TarError) as err:
-            message = str(err)
-            status = 1
+            os.remove(temp_file)
+            return 1, str(err), 0
 
-        # Round up the size of archive.
-        size_mb = int(math.ceil(float(file_list_size(local_file)) / (1024 * 1024)))
+        # Round up the size of the archive.
+        size_mb = int(math.ceil(float(file_list_size(temp_file)) / (1024 * 1024)))
         self._upload_queue.put(UploadArgs(
             self.archival_json_file,
             display_name,
-            local_file,
+            temp_file,
             "application/x-gzip",
             s3_bucket,
             s3_path,
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 502280c9912..83d75f410f9 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -483,16 +483,16 @@ functions:
     command: shell.exec
     type: test
     params:
-        working_dir: src
         script: |
           # exit immediately if virtualenv is not found
           set -o errexit
           set -o verbose
-          if [ ${build_variant} == enterprise-suse12-64 ]; then
-              virtualenv ./venv
-          else
-              virtualenv --system-site-packages ./venv
+          python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
+          if [ "Windows_NT" = "$OS" ]; then
+            python_loc=$(cygpath -w $python_loc)
           fi
+          # Set up virtualenv in ${workdir}
+          virtualenv --python "$python_loc" --system-site-packages "${workdir}/venv"
 
   "run tests" :
     - command: expansions.update
@@ -520,14 +520,8 @@ functions:
           export TMPDIR="${workdir}/tmp"
           mkdir -p $TMPDIR
 
-          # check if virtualenv is set up
-          if [ -d "venv" ]; then
-            if [ "Windows_NT" = "$OS" ]; then
-              . ./venv/Scripts/activate
-            else
-              . ./venv/bin/activate
-            fi
-          fi
+          ${activate_virtualenv}
+          pip install boto3
 
           if [ -f /proc/self/coredump_filter ]; then
             # Set the shell process (and its children processes) to dump ELF headers (bit 4),
@@ -608,10 +602,18 @@ functions:
           ${lang_environment}                \
           ${san_options}                     \
           ${rlp_environment}                 \
-          ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py  \
+          $python buildscripts/resmoke.py    \
             ${resmoke_args}                  \
             $extra_args ${test_flags}        \
             --log=buildlogger                \
+            --taskId=${task_id}              \
+            --taskName=${task_name}          \
+            --executionNumber=${execution}   \
+            --projectName=${project}         \
+            --variantName=${build_variant}   \
+            --distroId=${distro_id}          \
+            --gitRevision=${revision}        \
+            --archiveFile=archive.json       \
             --reportFile=report.json
 
 
@@ -912,9 +914,28 @@ pre:
   - command: shell.track
   - func: "kill processes"
   - func: "cleanup environment"
+  - func: "set up virtualenv"
   - command: expansions.update
     params:
        updates:
+       - key: activate_virtualenv
+         value: |
+          # check if virtualenv is set up
+          if [ -d "${workdir}/venv" ]; then
+            if [ "Windows_NT" = "$OS" ]; then
+              # Need to quote the path on Windows to preserve the separator.
+              . "${workdir}/venv/Scripts/activate" 2> /tmp/activate_error.log
+            else
+              . ${workdir}/venv/bin/activate 2> /tmp/activate_error.log
+            fi
+            if [ $? -ne 0 ]; then
+              echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)"
+            fi
+            python=python
+          else
+            python=${python|/opt/mongodbtoolchain/v2/bin/python2}
+          fi
+          echo "python set to $(which $python)"
        - key: set_sudo
          value: |
           set -o | grep errexit | grep on
@@ -986,32 +1007,6 @@ post:
           echo "No OOM (Out of memory) killed processes detected"
         fi
 
-  # Gather and archive FTDC data.
-  - command: shell.exec
-    params:
-      working_dir: src
-      script: |
-        # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories.
-        # The archive.targz_pack command is not being used here because the command's glob support
-        # did not allow us to gather all directories.
-        if [ -d /data/db ]; then
-            file_list=$(cd /data/db && find . -type d -name diagnostic.data)
-            if [ -n "$file_list" ]; then
-                ${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list
-            fi
-        fi
-  - command: s3.put
-    params:
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      local_file: src/diagnostic-data.tgz
-      remote_file: mongodb-mongo-v3.4/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz
-      bucket: mciuploads
-      permissions: public-read
-      content_type: ${content_type|application/x-gzip}
-      display_name: FTDC Diagnostic Data - Execution ${execution}
-      optional: true
-
   # Process and save coverage data.
   - command: shell.exec
     params:
@@ -1477,15 +1472,11 @@ tasks:
         enterprise: ${enterprise_rev}
   - command: shell.exec
   - func: "do setup"
-  - func: "set up virtualenv"
   - command: shell.exec
     params:
       working_dir: burn_in_tests_clonedir
       script: |
         set -o errexit
-        # Create a symbolic link to the venv in the src directory so activate_virtualenv can use it
-        # if it exists.
-        ln -s ../src/venv venv
         ${activate_virtualenv}
         set -o verbose
         # If this is a scheduled build, we check for changes against the last scheduled commit.
@@ -1496,7 +1487,7 @@ tasks:
         # list of dbtest suites.
         cp ../src/dbtest${exe} .
         # Capture a list of new and modified tests.
-        ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args
+        $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args
         # Copy the results to the src dir.
         cp jstests/new_tests.json ../src/jstests/new_tests.json
   - func: "do multiversion setup"
@@ -1937,7 +1928,6 @@ tasks:
   name: ese_WT
   commands:
   - func: "do setup"
-  - func: "set up virtualenv"
   - command: shell.exec
     type: test
     params:
@@ -1946,11 +1936,7 @@ tasks:
         set -o errexit
         set -o verbose
 
-        if [ "Windows_NT" = "$OS" ]; then
-          . ./venv/Scripts/activate
-        else
-          . ./venv/bin/activate
-        fi
+        ${activate_virtualenv}
         # we should go back to using pip when it works on SLES
         easy_install --upgrade `cat src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt`
   - func: "run tests"