From fd5fc15bbc8021899b12e83bca11e2ec9c2a1163 Mon Sep 17 00:00:00 2001 From: Jonathan Abrahams Date: Wed, 14 Feb 2018 11:38:17 -0500 Subject: SERVER-31562 Archival for test failures from suites not using a resmoke fixture (cherry picked from commit 9fd34c78b7471a3cec40e7cdc221d10b1a100ad3) --- buildscripts/resmokeconfig/suites/no_passthrough.yml | 4 ++++ buildscripts/resmokelib/testing/fixtures/interface.py | 14 ++++++++++++-- buildscripts/resmokelib/testing/fixtures/masterslave.py | 12 ++---------- buildscripts/resmokelib/testing/fixtures/replicaset.py | 12 ++---------- .../resmokelib/testing/fixtures/shardedcluster.py | 12 ++---------- buildscripts/resmokelib/testing/fixtures/standalone.py | 17 +++++++---------- buildscripts/resmokelib/testing/hook_test_archival.py | 4 ++-- buildscripts/resmokelib/utils/archival.py | 6 +++--- 8 files changed, 34 insertions(+), 47 deletions(-) diff --git a/buildscripts/resmokeconfig/suites/no_passthrough.yml b/buildscripts/resmokeconfig/suites/no_passthrough.yml index 2afbe6ae044..8012bfaf155 100644 --- a/buildscripts/resmokeconfig/suites/no_passthrough.yml +++ b/buildscripts/resmokeconfig/suites/no_passthrough.yml @@ -9,6 +9,10 @@ selector: # noPassthrough tests start their own mongod's. executor: js_test: + archive: + tests: + - jstests/noPassthrough/backup*.js + - jstests/noPassthrough/wt_unclean_shutdown.js config: shell_options: nodb: '' diff --git a/buildscripts/resmokelib/testing/fixtures/interface.py b/buildscripts/resmokelib/testing/fixtures/interface.py index f2cb1b9c8b8..5da95175008 100644 --- a/buildscripts/resmokelib/testing/fixtures/interface.py +++ b/buildscripts/resmokelib/testing/fixtures/interface.py @@ -4,12 +4,16 @@ Interface of the different fixtures for executing JSTests against. from __future__ import absolute_import +import os.path import time import pymongo +import pymongo.errors +from ... import config from ... import errors from ... import logging +from ... import utils class Fixture(object): @@ -17,7 +21,7 @@ class Fixture(object): Base class for all fixtures. """ - def __init__(self, logger, job_num): + def __init__(self, logger, job_num, dbpath_prefix=None): """ Initializes the fixture with a logger instance. """ @@ -32,9 +36,12 @@ class Fixture(object): self.logger = logger self.job_num = job_num - self.port = None # Port that the mongo shell should connect to. + dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) + dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) + self._dbpath_prefix = os.path.join(dbpath_prefix, "job{}".format(self.job_num)) + def setup(self): """ Creates the fixture. @@ -79,6 +86,9 @@ class Fixture(object): """ return True + def get_dbpath_prefix(self): + return self._dbpath_prefix + def get_internal_connection_string(self): """ Returns the connection string for this fixture. This is NOT a diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index 66924d3e3d9..f8bf4417451 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -31,7 +31,7 @@ class MasterSlaveFixture(interface.ReplFixture): dbpath_prefix=None, preserve_dbpath=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -42,12 +42,7 @@ class MasterSlaveFixture(interface.ReplFixture): self.slave_options = utils.default_if_none(slave_options, {}) self.preserve_dbpath = preserve_dbpath - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.master = None self.slave = None @@ -62,9 +57,6 @@ class MasterSlaveFixture(interface.ReplFixture): self.slave = self._new_mongod_slave() self.slave.setup() - def get_dbpath(self): - return self._dbpath_prefix - def await_ready(self): self.master.await_ready() self.slave.await_ready() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 26079cd9e02..b185ee7f8f2 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -40,7 +40,7 @@ class ReplicaSetFixture(interface.ReplFixture): voting_secondaries=False, use_replica_set_connection_string=False): - interface.ReplFixture.__init__(self, logger, job_num) + interface.ReplFixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) self.mongod_executable = mongod_executable self.mongod_options = utils.default_if_none(mongod_options, {}) @@ -59,12 +59,7 @@ class ReplicaSetFixture(interface.ReplFixture): if "dbpath" in self.mongod_options: self._dbpath_prefix = self.mongod_options.pop("dbpath") else: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.nodes = [] self.replset_name = None @@ -186,9 +181,6 @@ class ReplicaSetFixture(interface.ReplFixture): raise time.sleep(5) # Wait a little bit before trying again. - def get_dbpath(self): - return self._dbpath_prefix - def await_ready(self): self._await_primary() self._await_secondaries() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index ca788cc8dde..c0c3624b69a 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -47,7 +47,7 @@ class ShardedClusterFixture(interface.Fixture): the mongod and mongos processes. """ - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options: raise ValueError("Cannot specify mongod_options.dbpath") @@ -62,12 +62,7 @@ class ShardedClusterFixture(interface.Fixture): self.enable_sharding = utils.default_if_none(enable_sharding, []) self.auth_options = auth_options - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self._dbpath_prefix = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self._dbpath_prefix = os.path.join(self._dbpath_prefix, config.FIXTURE_SUBDIR) self.configsvr = None self.mongos = None @@ -88,9 +83,6 @@ class ShardedClusterFixture(interface.Fixture): for shard in self.shards: shard.setup() - def get_dbpath(self): - return self._dbpath_prefix - def await_ready(self): # Wait for the config server if self.configsvr is not None: diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index ffc94ceb4aa..9da26559a05 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -34,7 +34,7 @@ class MongoDFixture(interface.Fixture): dbpath_prefix=None, preserve_dbpath=False): - interface.Fixture.__init__(self, logger, job_num) + interface.Fixture.__init__(self, logger, job_num, dbpath_prefix=dbpath_prefix) if "dbpath" in mongod_options and dbpath_prefix is not None: raise ValueError("Cannot specify both mongod_options.dbpath and dbpath_prefix") @@ -48,12 +48,8 @@ class MongoDFixture(interface.Fixture): # The dbpath in mongod_options takes precedence over other settings to make it easier for # users to specify a dbpath containing data to test against. if "dbpath" not in self.mongod_options: - # Command line options override the YAML configuration. - dbpath_prefix = utils.default_if_none(config.DBPATH_PREFIX, dbpath_prefix) - dbpath_prefix = utils.default_if_none(dbpath_prefix, config.DEFAULT_DBPATH_PREFIX) - self.mongod_options["dbpath"] = os.path.join(dbpath_prefix, - "job%d" % (self.job_num), - config.FIXTURE_SUBDIR) + self.mongod_options["dbpath"] = os.path.join( + self._dbpath_prefix, config.FIXTURE_SUBDIR) self._dbpath = self.mongod_options["dbpath"] self.mongod = None @@ -85,9 +81,6 @@ class MongoDFixture(interface.Fixture): self.mongod = mongod - def get_dbpath(self): - return self._dbpath - def await_ready(self): deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS @@ -148,6 +141,10 @@ class MongoDFixture(interface.Fixture): def is_running(self): return self.mongod is not None and self.mongod.poll() is None + def get_dbpath_prefix(self): + """ Returns the _dbpath, as this is the root of the data directory. """ + return self._dbpath + def get_internal_connection_string(self): if self.mongod is None: raise ValueError("Must call setup() before calling get_internal_connection_string()") diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py index 1873485cdbf..bde562cf617 100644 --- a/buildscripts/resmokelib/testing/hook_test_archival.py +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -89,7 +89,6 @@ class HookTestArchival(object): self._tests_repeat[test_name] = 0 else: self._tests_repeat[test_name] += 1 - logger.info("Archiving data files for test %s", test_name) # Normalize test path from a test or hook name. test_path = \ test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_") @@ -99,7 +98,7 @@ class HookTestArchival(object): config.EVERGREEN_EXECUTION, self._tests_repeat[test_name]) # Retrieve root directory for all dbPaths from fixture. - input_files = test.fixture.get_dbpath() + input_files = test.fixture.get_dbpath_prefix() s3_bucket = config.ARCHIVE_BUCKET s3_path = "{}/{}/{}/datafiles/{}".format( config.EVERGREEN_PROJECT_NAME, @@ -110,6 +109,7 @@ class HookTestArchival(object): test_name, config.EVERGREEN_EXECUTION, self._tests_repeat[test_name]) + logger.info("Archiving data files for test %s from %s", test_name, input_files) status, message = self.archive_instance.archive_files_to_s3( display_name, input_files, s3_bucket, s3_path) if status: diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py index 5d3545159d3..221b54ecd35 100644 --- a/buildscripts/resmokelib/utils/archival.py +++ b/buildscripts/resmokelib/utils/archival.py @@ -52,7 +52,7 @@ def file_list_size(files): def directory_size(directory): """ Return size (in bytes) of files in 'directory' tree. """ dir_bytes = 0 - for root_dir, _, files in os.walk(directory): + for root_dir, _, files in os.walk(unicode(directory)): for name in files: full_name = os.path.join(root_dir, name) try: @@ -277,10 +277,10 @@ class Archival(object): for input_file in input_files: try: tar_handle.add(input_file) - except (IOError, tarfile.TarError) as err: + except (IOError, OSError, tarfile.TarError) as err: message = "{}; Unable to add {} to archive file: {}".format( message, input_file, err) - except (IOError, tarfile.TarError) as err: + except (IOError, OSError, tarfile.TarError) as err: status, message = remove_file(temp_file) if status: self.logger.warning("Removing tarfile due to creation failure - %s", message) -- cgit v1.2.1