SERVER-31562 Archival for test failures from suites not using a resmoke fixture

10 files changed, 37 insertions, 45 deletions

diff --git a/buildscripts/resmokeconfig/suites/generational_fuzzer_replication.yml b/buildscripts/resmokeconfig/suites/generational_fuzzer_replication.yml
index 2a121525ea6..66605b58496 100644
--- a/buildscripts/resmokeconfig/suites/generational_fuzzer_replication.yml
+++ b/buildscripts/resmokeconfig/suites/generational_fuzzer_replication.yml
@@ -5,6 +5,8 @@ selector:
   - jstestfuzz/out/*.js
 
 executor:
+  archive:
+    tests: true
   config:
     shell_options:
       nodb: ''
diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml
index ac0c08980a0..bd87e5e56e0 100644
--- a/buildscripts/resmokeconfig/suites/no_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml
@@ -9,6 +9,10 @@ selector:
 
 # noPassthrough tests start their own mongod's.
 executor:
+  archive:
+    tests:
+      - jstests/noPassthrough/backup*.js
+      - jstests/noPassthrough/wt_unclean_shutdown.js
   config:
     shell_options:
       nodb: ''
diff --git a/buildscripts/resmokeconfig/suites/rollback_fuzzer.yml b/buildscripts/resmokeconfig/suites/rollback_fuzzer.yml
index 0ca3c7f33f6..a1a2f2c3928 100644
--- a/buildscripts/resmokeconfig/suites/rollback_fuzzer.yml
+++ b/buildscripts/resmokeconfig/suites/rollback_fuzzer.yml
@@ -5,6 +5,8 @@ selector:
   - jstestfuzz/out/*.js
 
 executor:
+  archive:
+    tests: true
   config:
     shell_options:
       nodb: ''
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index 6dffa24e430..af694ba3e01 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -4,13 +4,16 @@ Interface of the different fixtures for executing JSTests against.
 
 from __future__ import absolute_import
 
+import os.path
 import time
 
 import pymongo
 import pymongo.errors
 
+from ... import config
 from ... import errors
 from ... import logging
+from ... import utils
 from ...utils import registry
 
 
@@ -38,7 +41,7 @@ class Fixture(object):
     # is defined for all subclasses of Fixture.
     REGISTERED_NAME = "Fixture"
 
-    def __init__(self, logger, job_num):
+    def __init__(self, logger, job_num, dbpath_prefix=None):
         """
         Initializes the fixture with a logger instance.
         """
@@ -54,6 +57,10 @@ class Fixture(object):
         self.logger = logger
         self.job_num = job_num
 
+        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
+        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
+        self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num))
+
     def setup(self):
         """
         Creates the fixture.
@@ -98,6 +105,9 @@ class Fixture(object):
         """
         return True
 
+    def get_dbpath_prefix(self):
+        return self._dbpath_prefix
+
     def get_internal_connection_string(self):
         """
         Returns the connection string for this fixture. This is NOT a
diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py
index 2b9c48de3e7..d63205363ef 100644
--- a/buildscripts/resmokelib/testing/fixtures/masterslave.py
+++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py
@@ -31,7 +31,7 @@ class MasterSlaveFixture(interface.ReplFixture):
                  dbpath_prefix=None,
                  preserve_dbpath=False):
 
-        interface.ReplFixture.__init__(self, logger, job_num)
+        interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options:
             raise ValueError("Cannot specify mongod_options.dbpath")
@@ -42,12 +42,7 @@ class MasterSlaveFixture(interface.ReplFixture):
         self.slave_options = utils.default_if_none(slave_options, {})
         self.preserve_dbpath = preserve_dbpath
 
-        # Command line options override the YAML configuration.
-        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-        self._dbpath_prefix = os.path.join(dbpath_prefix,
-                                           "job{}".format(self.job_num),
-                                           config.FIXTURE_SUBDIR)
+        self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.master = None
         self.slave = None
@@ -120,9 +115,6 @@ class MasterSlaveFixture(interface.ReplFixture):
     def get_secondaries(self):
         return [self.slave]
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def _new_mongod(self, mongod_logger, mongod_options):
         """
         Returns a standalone.MongoDFixture with the specified logger and
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 67d3e18f3aa..37925aae889 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -41,7 +41,7 @@ class ReplicaSetFixture(interface.ReplFixture):
                  all_nodes_electable=False,
                  use_replica_set_connection_string=None):
 
-        interface.ReplFixture.__init__(self, logger, job_num)
+        interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         self.mongod_executable = mongod_executable
         self.mongod_options = utils.default_if_none(mongod_options, {})
@@ -71,12 +71,7 @@ class ReplicaSetFixture(interface.ReplFixture):
         if "dbpath" in self.mongod_options:
             self._dbpath_prefix = self.mongod_options.pop("dbpath")
         else:
-            # Command line options override the YAML configuration.
-            dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-            dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-            self._dbpath_prefix = os.path.join(dbpath_prefix,
-                                               "job{}".format(self.job_num),
-                                               config.FIXTURE_SUBDIR)
+            self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.nodes = []
         self.replset_name = None
@@ -310,9 +305,6 @@ class ReplicaSetFixture(interface.ReplFixture):
     def get_initial_sync_node(self):
         return self.initial_sync_node
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def _new_mongod(self, index, replset_name):
         """
         Returns a standalone.MongoDFixture configured to be used as a
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index a4c053b77a2..9a8c5be1efa 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -51,7 +51,7 @@ class ShardedClusterFixture(interface.Fixture):
         the mongod and mongos processes.
         """
 
-        interface.Fixture.__init__(self, logger, job_num)
+        interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options:
             raise ValueError("Cannot specify mongod_options.dbpath")
@@ -70,11 +70,7 @@ class ShardedClusterFixture(interface.Fixture):
         self.configsvr_options = utils.default_if_none(configsvr_options, {})
         self.shard_options = utils.default_if_none(shard_options, {})
 
-        # Command line options override the YAML configuration.
-        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-        self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num),
-                                           config.FIXTURE_SUBDIR)
+        self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.configsvr = None
         self.mongos = None
@@ -102,9 +98,6 @@ class ShardedClusterFixture(interface.Fixture):
         for shard in self.shards:
             shard.setup()
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def await_ready(self):
         # Wait for the config server
         if self.configsvr is not None:
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index 4b99381d6ca..28a8c00c486 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -35,7 +35,7 @@ class MongoDFixture(interface.Fixture):
                  dbpath_prefix=None,
                  preserve_dbpath=False):
 
-        interface.Fixture.__init__(self, logger, job_num)
+        interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options and dbpath_prefix is not None:
             raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix")
@@ -49,12 +49,8 @@ class MongoDFixture(interface.Fixture):
         # The dbpath in mongod_options takes precedence over other settings to make it easier for
         # users to specify a dbpath containing data to test against.
         if "dbpath" not in self.mongod_options:
-            # Command line options override the YAML configuration.
-            dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-            dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-            self.mongod_options["dbpath"] = os.path.join(dbpath_prefix,
-                                                         "job{}".format(self.job_num),
-                                                         config.FIXTURE_SUBDIR)
+            self.mongod_options["dbpath"] = os.path.join(
+                self._dbpath_prefix, config.FIXTURE_SUBDIR)
         self._dbpath = self.mongod_options["dbpath"]
 
         self.mongod = None
@@ -87,9 +83,6 @@ class MongoDFixture(interface.Fixture):
 
         self.mongod = mongod
 
-    def get_dbpath(self):
-        return self._dbpath
-
     def await_ready(self):
         deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS
 
@@ -152,6 +145,10 @@ class MongoDFixture(interface.Fixture):
     def is_running(self):
         return self.mongod is not None and self.mongod.poll() is None
 
+    def get_dbpath_prefix(self):
+        """ Returns the _dbpath, as this is the root of the data directory. """
+        return self._dbpath
+
     def get_internal_connection_string(self):
         if self.mongod is None:
             raise ValueError("Must call setup() before calling get_internal_connection_string()")
diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py
index 0503d7be50d..05445208673 100644
--- a/buildscripts/resmokelib/testing/hook_test_archival.py
+++ b/buildscripts/resmokelib/testing/hook_test_archival.py
@@ -89,7 +89,6 @@ class HookTestArchival(object):
                 self._tests_repeat[test_name] = 0
             else:
                 self._tests_repeat[test_name] += 1
-        logger.info("Archiving data files for test %s", test_name)
         # Normalize test path from a test or hook name.
         test_path = \
             test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_")
@@ -99,7 +98,7 @@ class HookTestArchival(object):
             config.EVERGREEN_EXECUTION,
             self._tests_repeat[test_name])
         # Retrieve root directory for all dbPaths from fixture.
-        input_files = test.fixture.get_dbpath()
+        input_files = test.fixture.get_dbpath_prefix()
         s3_bucket = config.ARCHIVE_BUCKET
         s3_path = "{}/{}/{}/datafiles/{}".format(
             config.EVERGREEN_PROJECT_NAME,
@@ -110,6 +109,7 @@ class HookTestArchival(object):
             test_name,
             config.EVERGREEN_EXECUTION,
             self._tests_repeat[test_name])
+        logger.info("Archiving data files for test %s from %s", test_name, input_files)
         status, message = self.archive_instance.archive_files_to_s3(
             display_name, input_files, s3_bucket, s3_path)
         if status:
diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py
index 5d3545159d3..221b54ecd35 100644
--- a/buildscripts/resmokelib/utils/archival.py
+++ b/buildscripts/resmokelib/utils/archival.py
@@ -52,7 +52,7 @@ def file_list_size(files):
 def directory_size(directory):
     """ Return size (in bytes) of files in 'directory' tree. """
     dir_bytes = 0
-    for root_dir, _, files in os.walk(directory):
+    for root_dir, _, files in os.walk(unicode(directory)):
         for name in files:
             full_name = os.path.join(root_dir, name)
             try:
@@ -277,10 +277,10 @@ class Archival(object):
                 for input_file in input_files:
                     try:
                         tar_handle.add(input_file)
-                    except (IOError, tarfile.TarError) as err:
+                    except (IOError, OSError, tarfile.TarError) as err:
                         message = "{}; Unable to add {} to archive file: {}".format(
                             message, input_file, err)
-        except (IOError, tarfile.TarError) as err:
+        except (IOError, OSError, tarfile.TarError) as err:
             status, message = remove_file(temp_file)
             if status:
                 self.logger.warning("Removing tarfile due to creation failure - %s", message)