SERVER-31562 Archival for test failures from suites not using a resmoke fixture

(cherry picked from commit 9fd34c78b7471a3cec40e7cdc221d10b1a100ad3)

8 files changed, 34 insertions, 47 deletions

diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml
index 2afbe6ae044..8012bfaf155 100644
--- a/buildscripts/resmokeconfig/suites/no_passthrough.yml
+++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml
@@ -9,6 +9,10 @@ selector:
 # noPassthrough tests start their own mongod's.
 executor:
   js_test:
+    archive:
+      tests:
+        - jstests/noPassthrough/backup*.js
+        - jstests/noPassthrough/wt_unclean_shutdown.js
     config:
       shell_options:
         nodb: ''
diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py
index f2cb1b9c8b8..5da95175008 100644
--- a/buildscripts/resmokelib/testing/fixtures/interface.py
+++ b/buildscripts/resmokelib/testing/fixtures/interface.py
@@ -4,12 +4,16 @@ Interface of the different fixtures for executing JSTests against.
 
 from __future__ import absolute_import
 
+import os.path
 import time
 
 import pymongo
+import pymongo.errors
 
+from ... import config
 from ... import errors
 from ... import logging
+from ... import utils
 
 
 class Fixture(object):
@@ -17,7 +21,7 @@ class Fixture(object):
     Base class for all fixtures.
     """
 
-    def __init__(self, logger, job_num):
+    def __init__(self, logger, job_num, dbpath_prefix=None):
         """
         Initializes the fixture with a logger instance.
         """
@@ -32,9 +36,12 @@ class Fixture(object):
 
         self.logger = logger
         self.job_num = job_num
-
         self.port = None  # Port that the mongo shell should connect to.
 
+        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
+        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
+        self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num))
+
     def setup(self):
         """
         Creates the fixture.
@@ -79,6 +86,9 @@ class Fixture(object):
         """
         return True
 
+    def get_dbpath_prefix(self):
+        return self._dbpath_prefix
+
     def get_internal_connection_string(self):
         """
         Returns the connection string for this fixture. This is NOT a
diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py
index 66924d3e3d9..f8bf4417451 100644
--- a/buildscripts/resmokelib/testing/fixtures/masterslave.py
+++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py
@@ -31,7 +31,7 @@ class MasterSlaveFixture(interface.ReplFixture):
                  dbpath_prefix=None,
                  preserve_dbpath=False):
 
-        interface.ReplFixture.__init__(self, logger, job_num)
+        interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options:
             raise ValueError("Cannot specify mongod_options.dbpath")
@@ -42,12 +42,7 @@ class MasterSlaveFixture(interface.ReplFixture):
         self.slave_options = utils.default_if_none(slave_options, {})
         self.preserve_dbpath = preserve_dbpath
 
-        # Command line options override the YAML configuration.
-        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-        self._dbpath_prefix = os.path.join(dbpath_prefix,
-                                           "job%d" % (self.job_num),
-                                           config.FIXTURE_SUBDIR)
+        self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.master = None
         self.slave = None
@@ -62,9 +57,6 @@ class MasterSlaveFixture(interface.ReplFixture):
             self.slave = self._new_mongod_slave()
         self.slave.setup()
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def await_ready(self):
         self.master.await_ready()
         self.slave.await_ready()
diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py
index 26079cd9e02..b185ee7f8f2 100644
--- a/buildscripts/resmokelib/testing/fixtures/replicaset.py
+++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py
@@ -40,7 +40,7 @@ class ReplicaSetFixture(interface.ReplFixture):
                  voting_secondaries=False,
                  use_replica_set_connection_string=False):
 
-        interface.ReplFixture.__init__(self, logger, job_num)
+        interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         self.mongod_executable = mongod_executable
         self.mongod_options = utils.default_if_none(mongod_options, {})
@@ -59,12 +59,7 @@ class ReplicaSetFixture(interface.ReplFixture):
         if "dbpath" in self.mongod_options:
             self._dbpath_prefix = self.mongod_options.pop("dbpath")
         else:
-            # Command line options override the YAML configuration.
-            dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-            dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-            self._dbpath_prefix = os.path.join(dbpath_prefix,
-                                               "job%d" % (self.job_num),
-                                               config.FIXTURE_SUBDIR)
+            self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.nodes = []
         self.replset_name = None
@@ -186,9 +181,6 @@ class ReplicaSetFixture(interface.ReplFixture):
                     raise
                 time.sleep(5)  # Wait a little bit before trying again.
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def await_ready(self):
         self._await_primary()
         self._await_secondaries()
diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
index ca788cc8dde..c0c3624b69a 100644
--- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
+++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py
@@ -47,7 +47,7 @@ class ShardedClusterFixture(interface.Fixture):
         the mongod and mongos processes.
         """
 
-        interface.Fixture.__init__(self, logger, job_num)
+        interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options:
             raise ValueError("Cannot specify mongod_options.dbpath")
@@ -62,12 +62,7 @@ class ShardedClusterFixture(interface.Fixture):
         self.enable_sharding = utils.default_if_none(enable_sharding, [])
         self.auth_options = auth_options
 
-        # Command line options override the YAML configuration.
-        dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-        dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-        self._dbpath_prefix = os.path.join(dbpath_prefix,
-                                           "job%d" % (self.job_num),
-                                           config.FIXTURE_SUBDIR)
+        self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR)
 
         self.configsvr = None
         self.mongos = None
@@ -88,9 +83,6 @@ class ShardedClusterFixture(interface.Fixture):
         for shard in self.shards:
             shard.setup()
 
-    def get_dbpath(self):
-        return self._dbpath_prefix
-
     def await_ready(self):
         # Wait for the config server
         if self.configsvr is not None:
diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py
index ffc94ceb4aa..9da26559a05 100644
--- a/buildscripts/resmokelib/testing/fixtures/standalone.py
+++ b/buildscripts/resmokelib/testing/fixtures/standalone.py
@@ -34,7 +34,7 @@ class MongoDFixture(interface.Fixture):
                  dbpath_prefix=None,
                  preserve_dbpath=False):
 
-        interface.Fixture.__init__(self, logger, job_num)
+        interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix)
 
         if "dbpath" in mongod_options and dbpath_prefix is not None:
             raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix")
@@ -48,12 +48,8 @@ class MongoDFixture(interface.Fixture):
         # The dbpath in mongod_options takes precedence over other settings to make it easier for
         # users to specify a dbpath containing data to test against.
         if "dbpath" not in self.mongod_options:
-            # Command line options override the YAML configuration.
-            dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix)
-            dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX)
-            self.mongod_options["dbpath"] = os.path.join(dbpath_prefix,
-                                                         "job%d" % (self.job_num),
-                                                         config.FIXTURE_SUBDIR)
+            self.mongod_options["dbpath"] = os.path.join(
+                self._dbpath_prefix, config.FIXTURE_SUBDIR)
         self._dbpath = self.mongod_options["dbpath"]
 
         self.mongod = None
@@ -85,9 +81,6 @@ class MongoDFixture(interface.Fixture):
 
         self.mongod = mongod
 
-    def get_dbpath(self):
-        return self._dbpath
-
     def await_ready(self):
         deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS
 
@@ -148,6 +141,10 @@ class MongoDFixture(interface.Fixture):
     def is_running(self):
         return self.mongod is not None and self.mongod.poll() is None
 
+    def get_dbpath_prefix(self):
+        """ Returns the _dbpath, as this is the root of the data directory. """
+        return self._dbpath
+
     def get_internal_connection_string(self):
         if self.mongod is None:
             raise ValueError("Must call setup() before calling get_internal_connection_string()")
diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py
index 1873485cdbf..bde562cf617 100644
--- a/buildscripts/resmokelib/testing/hook_test_archival.py
+++ b/buildscripts/resmokelib/testing/hook_test_archival.py
@@ -89,7 +89,6 @@ class HookTestArchival(object):
                 self._tests_repeat[test_name] = 0
             else:
                 self._tests_repeat[test_name] += 1
-        logger.info("Archiving data files for test %s", test_name)
         # Normalize test path from a test or hook name.
         test_path = \
             test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_")
@@ -99,7 +98,7 @@ class HookTestArchival(object):
             config.EVERGREEN_EXECUTION,
             self._tests_repeat[test_name])
         # Retrieve root directory for all dbPaths from fixture.
-        input_files = test.fixture.get_dbpath()
+        input_files = test.fixture.get_dbpath_prefix()
         s3_bucket = config.ARCHIVE_BUCKET
         s3_path = "{}/{}/{}/datafiles/{}".format(
             config.EVERGREEN_PROJECT_NAME,
@@ -110,6 +109,7 @@ class HookTestArchival(object):
             test_name,
             config.EVERGREEN_EXECUTION,
             self._tests_repeat[test_name])
+        logger.info("Archiving data files for test %s from %s", test_name, input_files)
         status, message = self.archive_instance.archive_files_to_s3(
             display_name, input_files, s3_bucket, s3_path)
         if status:
diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py
index 5d3545159d3..221b54ecd35 100644
--- a/buildscripts/resmokelib/utils/archival.py
+++ b/buildscripts/resmokelib/utils/archival.py
@@ -52,7 +52,7 @@ def file_list_size(files):
 def directory_size(directory):
     """ Return size (in bytes) of files in 'directory' tree. """
     dir_bytes = 0
-    for root_dir, _, files in os.walk(directory):
+    for root_dir, _, files in os.walk(unicode(directory)):
         for name in files:
             full_name = os.path.join(root_dir, name)
             try:
@@ -277,10 +277,10 @@ class Archival(object):
                 for input_file in input_files:
                     try:
                         tar_handle.add(input_file)
-                    except (IOError, tarfile.TarError) as err:
+                    except (IOError, OSError, tarfile.TarError) as err:
                         message = "{}; Unable to add {} to archive file: {}".format(
                             message, input_file, err)
-        except (IOError, tarfile.TarError) as err:
+        except (IOError, OSError, tarfile.TarError) as err:
             status, message = remove_file(temp_file)
             if status:
                 self.logger.warning("Removing tarfile due to creation failure - %s", message)