diff options
-rwxr-xr-x | buildscripts/resmoke.py | 12 | ||||
-rw-r--r-- | buildscripts/resmokelib/config.py | 44 | ||||
-rw-r--r-- | buildscripts/resmokelib/parser.py | 73 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/executor.py | 12 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/masterslave.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/replicaset.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/shardedcluster.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/fixtures/standalone.py | 3 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/hook_test_archival.py | 116 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/job.py | 47 | ||||
-rw-r--r-- | buildscripts/resmokelib/testing/report.py | 14 | ||||
-rw-r--r-- | buildscripts/resmokelib/utils/__init__.py | 2 | ||||
-rw-r--r-- | buildscripts/resmokelib/utils/archival.py | 35 | ||||
-rw-r--r-- | etc/evergreen.yml | 88 |
14 files changed, 368 insertions, 87 deletions
diff --git a/buildscripts/resmoke.py b/buildscripts/resmoke.py index 7aa13277384..3fa24bd6553 100755 --- a/buildscripts/resmoke.py +++ b/buildscripts/resmoke.py @@ -54,10 +54,19 @@ def _execute_suite(suite, logging_config): logger.info("Skipping %ss, no tests to run", group.test_kind) continue + archive = None + if resmokelib.config.ARCHIVE_FILE: + archive = resmokelib.utils.archival.Archival( + archival_json_file=resmokelib.config.ARCHIVE_FILE, + execution=resmokelib.config.EVERGREEN_EXECUTION, + limit_size_mb=resmokelib.config.ARCHIVE_LIMIT_MB, + limit_files=resmokelib.config.ARCHIVE_LIMIT_TESTS, + logger=logger) group_config = suite.get_executor_config().get(group.test_kind, {}) executor = resmokelib.testing.executor.TestGroupExecutor(logger, group, logging_config, + archive_instance=archive, **group_config) try: @@ -73,6 +82,9 @@ def _execute_suite(suite, logging_config): group.test_kind, suite.get_name()) suite.return_code = 2 return False + finally: + if archive: + archive.exit() def _log_summary(logger, suites, time_taken): diff --git a/buildscripts/resmokelib/config.py b/buildscripts/resmokelib/config.py index b75753398a5..8f636ddbadd 100644 --- a/buildscripts/resmokelib/config.py +++ b/buildscripts/resmokelib/config.py @@ -33,14 +33,20 @@ MONGO_RUNNER_SUBDIR = "mongorunner" # Names below correspond to how they are specified via the command line or in the options YAML file. DEFAULTS = { + "archiveFile": None, + "archiveLimitMb": 5000, + "archiveLimitTests": 10, "basePort": 20000, "buildloggerUrl": "https://logkeeper.mongodb.org", "continueOnFailure": False, "dbpathPrefix": None, "dbtest": None, + "distroId": None, "dryRun": None, "excludeWithAllTags": None, "excludeWithAnyTags": None, + "executionNumber": 0, + "gitRevision": None, "includeWithAllTags": None, "includeWithAnyTags": None, "jobs": 1, @@ -51,6 +57,7 @@ DEFAULTS = { "mongosSetParameters": None, "nojournal": False, "numClientsPerFixture": 1, + "projectName": "mongodb-mongo-master", "repeat": 1, "reportFile": None, "seed": long(time.time() * 256), # Taken from random.py code in Python 2.7. @@ -59,6 +66,9 @@ DEFAULTS = { "shuffle": False, "storageEngine": None, "storageEngineCacheSizeGB": None, + "taskId": None, + "taskName": None, + "variantName": None, "wiredTigerCollectionConfigString": None, "wiredTigerEngineConfigString": None, "wiredTigerIndexConfigString": None @@ -69,6 +79,15 @@ DEFAULTS = { # Variables that are set by the user at the command line or with --options. ## +# The name of the archive JSON file used to associate S3 archives to an Evergreen task. +ARCHIVE_FILE = None + +# The limit size of all archive files for an Evergreen task. +ARCHIVE_LIMIT_MB = None + +# The limit number of tests to archive for an Evergreen task. +ARCHIVE_LIMIT_TESTS = None + # The starting port number to use for mongod and mongos processes spawned by resmoke.py and the # mongo shell. BASE_PORT = None @@ -87,6 +106,28 @@ DBTEST_EXECUTABLE = None # actually running them). DRY_RUN = None +# The identifier for the Evergreen distro that resmoke.py is being run on. +EVERGREEN_DISTRO_ID = None + +# The number of the Evergreen execution that resmoke.py is being run on. +EVERGREEN_EXECUTION = None + +# The name of the Evergreen project that resmoke.py is being run on. +EVERGREEN_PROJECT_NAME = None + +# The git revision of the Evergreen task that resmoke.py is being run on. +EVERGREEN_REVISION = None + +# The identifier for the Evergreen task that resmoke.py is being run under. If set, then the +# Evergreen task id value will be transmitted to logkeeper when creating builds and tests. +EVERGREEN_TASK_ID = None + +# The name of the Evergreen task that resmoke.py is being run for. +EVERGREEN_TASK_NAME = None + +# The name of the Evergreen build variant that resmoke.py is being run on. +EVERGREEN_VARIANT_NAME = None + # If set, then any jstests that have all of the specified tags will be excluded from the suite(s). EXCLUDE_WITH_ALL_TAGS = None @@ -177,6 +218,9 @@ WT_INDEX_CONFIG = None # Internally used configuration options that aren't exposed to the user ## +# S3 Bucket to upload archive files. +ARCHIVE_BUCKET = "mongodatafiles" + # Default sort order for test execution. Will only be changed if --suites wasn't specified. ORDER_TESTS_BY_NAME = True diff --git a/buildscripts/resmokelib/parser.py b/buildscripts/resmokelib/parser.py index 870e0e9145c..59b4585a273 100644 --- a/buildscripts/resmokelib/parser.py +++ b/buildscripts/resmokelib/parser.py @@ -18,14 +18,20 @@ from .. import resmokeconfig # Mapping of the attribute of the parsed arguments (dest) to its key as it appears in the options # YAML configuration file. Most should only be converting from snake_case to camelCase. DEST_TO_CONFIG = { + "archive_file": "archiveFile", + "archive_limit_mb": "archiveLimitMb", + "archive_limit_tests": "archiveLimitTests", "base_port": "basePort", "buildlogger_url": "buildloggerUrl", "continue_on_failure": "continueOnFailure", "dbpath_prefix": "dbpathPrefix", "dbtest_executable": "dbtest", + "distro_id": "distroId", "dry_run": "dryRun", "exclude_with_all_tags": "excludeWithAllTags", "exclude_with_any_tags": "excludeWithAnyTags", + "execution_number": "executionNumber", + "git_revision": "gitRevision", "include_with_all_tags": "includeWithAllTags", "include_with_any_tags": "includeWithAnyTags", "jobs": "jobs", @@ -37,6 +43,7 @@ DEST_TO_CONFIG = { "no_journal": "nojournal", "num_clients_per_fixture": "numClientsPerFixture", "prealloc_journal": "preallocJournal", + "project_name": "projectName", "repeat": "repeat", "report_file": "reportFile", "seed": "seed", @@ -45,6 +52,9 @@ DEST_TO_CONFIG = { "shuffle": "shuffle", "storage_engine": "storageEngine", "storage_engine_cache_size": "storageEngineCacheSizeGB", + "task_id": "taskId", + "task_name": "taskName", + "variant_name": "variantName", "wt_coll_config": "wiredTigerCollectionConfigString", "wt_engine_config": "wiredTigerEngineConfigString", "wt_index_config": "wiredTigerIndexConfigString" @@ -79,6 +89,23 @@ def parse_command_line(): parser.add_option("--options", dest="options_file", metavar="OPTIONS", help="A YAML file that specifies global options to resmoke.py.") + parser.add_option("--archiveFile", dest="archive_file", metavar="ARCHIVE_FILE", + help=("Sets the archive file name for the Evergreen task running the tests." + " The archive file is JSON format containing a list of tests that were" + " successfully archived to S3. If unspecified, no data files from tests" + " will be archived in S3. Tests can be designated for archival in the" + " task suite configuration file.")) + + parser.add_option("--archiveLimitMb", type="int", dest="archive_limit_mb", + metavar="ARCHIVE_LIMIT_MB", + help=("Sets the limit (in MB) for archived files to S3. A value of 0" + " indicates there is no limit.")) + + parser.add_option("--archiveLimitTests", type="int", dest="archive_limit_tests", + metavar="ARCHIVE_LIMIT_TESTS", + help=("Sets the maximum number of tests to archive to S3. A value" + " of 0 indicates there is no limit.")) + parser.add_option("--basePort", dest="base_port", metavar="PORT", help=("The starting port number to use for mongod and mongos processes" " spawned by resmoke.py or the tests themselves. Each fixture and Job" @@ -209,6 +236,42 @@ def parse_command_line(): parser.add_option("--wiredTigerIndexConfigString", dest="wt_index_config", metavar="CONFIG", help="Set the WiredTiger index configuration setting for all mongod's.") + + evergreen_options = optparse.OptionGroup( + parser, title="Evergreen options", + description=("Options used to propagate information about the Evergreen task running this" + " script.")) + parser.add_option_group(evergreen_options) + + evergreen_options.add_option("--buildId", dest="build_id", metavar="BUILD_ID", + help="Sets the build ID of the task.") + + evergreen_options.add_option("--distroId", dest="distro_id", metavar="DISTRO_ID", + help=("Sets the identifier for the Evergreen distro running the" + " tests.")) + + evergreen_options.add_option("--executionNumber", type="int", dest="execution_number", + metavar="EXECUTION_NUMBER", + help=("Sets the number for the Evergreen execution running the" + " tests.")) + + evergreen_options.add_option("--gitRevision", dest="git_revision", metavar="GIT_REVISION", + help=("Sets the git revision for the Evergreen task running the" + " tests.")) + + evergreen_options.add_option("--projectName", dest="project_name", metavar="PROJECT_NAME", + help=("Sets the name of the Evergreen project running the tests.")) + + evergreen_options.add_option("--taskName", dest="task_name", metavar="TASK_NAME", + help="Sets the name of the Evergreen task running the tests.") + + evergreen_options.add_option("--taskId", dest="task_id", metavar="TASK_ID", + help="Sets the Id of the Evergreen task running the tests.") + + evergreen_options.add_option("--variantName", dest="variant_name", metavar="VARIANT_NAME", + help=("Sets the name of the Evergreen build variant running the" + " tests.")) + parser.set_defaults(executor_file="with_server", logger_file="console", dry_run="off", @@ -237,11 +300,21 @@ def update_config_vars(values): if values[dest] is not None: config[config_var] = values[dest] + _config.ARCHIVE_FILE = config.pop("archiveFile") + _config.ARCHIVE_LIMIT_MB = config.pop("archiveLimitMb") + _config.ARCHIVE_LIMIT_TESTS = config.pop("archiveLimitTests") _config.BASE_PORT = int(config.pop("basePort")) _config.BUILDLOGGER_URL = config.pop("buildloggerUrl") _config.DBPATH_PREFIX = _expand_user(config.pop("dbpathPrefix")) _config.DBTEST_EXECUTABLE = _expand_user(config.pop("dbtest")) _config.DRY_RUN = config.pop("dryRun") + _config.EVERGREEN_DISTRO_ID = config.pop("distroId") + _config.EVERGREEN_EXECUTION = config.pop("executionNumber") + _config.EVERGREEN_PROJECT_NAME = config.pop("projectName") + _config.EVERGREEN_REVISION = config.pop("gitRevision") + _config.EVERGREEN_TASK_ID = config.pop("taskId") + _config.EVERGREEN_TASK_NAME = config.pop("taskName") + _config.EVERGREEN_VARIANT_NAME = config.pop("variantName") _config.EXCLUDE_WITH_ALL_TAGS = config.pop("excludeWithAllTags") _config.EXCLUDE_WITH_ANY_TAGS = config.pop("excludeWithAnyTags") _config.FAIL_FAST = not config.pop("continueOnFailure") diff --git a/buildscripts/resmokelib/testing/executor.py b/buildscripts/resmokelib/testing/executor.py index 438a06b9c8e..e88eed6bc02 100644 --- a/buildscripts/resmokelib/testing/executor.py +++ b/buildscripts/resmokelib/testing/executor.py @@ -8,6 +8,7 @@ import threading import time from . import fixtures +from . import hook_test_archival as archival from . import hooks as _hooks from . import job as _job from . import report as _report @@ -35,7 +36,9 @@ class TestGroupExecutor(object): logging_config, config=None, fixture=None, - hooks=None): + hooks=None, + archive_instance=None, + archive=None): """ Initializes the TestGroupExecutor with the test group to run. """ @@ -49,6 +52,11 @@ class TestGroupExecutor(object): self.hooks_config = utils.default_if_none(hooks, []) self.test_config = utils.default_if_none(config, {}) + self.archival = None + if archive_instance: + self.archival = archival.HookTestArchival(test_group, self.hooks_config, + archive_instance, archive) + self._test_group = test_group self._using_buildlogger = logging.config.using_buildlogger(logging_config) @@ -299,7 +307,7 @@ class TestGroupExecutor(object): build_id=build_id, build_config=build_config) - return _job.Job(logger, fixture, hooks, report) + return _job.Job(logger, fixture, hooks, report, self.archival) def _make_test_queue(self): """ diff --git a/buildscripts/resmokelib/testing/fixtures/masterslave.py b/buildscripts/resmokelib/testing/fixtures/masterslave.py index fb444cfe097..66924d3e3d9 100644 --- a/buildscripts/resmokelib/testing/fixtures/masterslave.py +++ b/buildscripts/resmokelib/testing/fixtures/masterslave.py @@ -62,6 +62,9 @@ class MasterSlaveFixture(interface.ReplFixture): self.slave = self._new_mongod_slave() self.slave.setup() + def get_dbpath(self): + return self._dbpath_prefix + def await_ready(self): self.master.await_ready() self.slave.await_ready() diff --git a/buildscripts/resmokelib/testing/fixtures/replicaset.py b/buildscripts/resmokelib/testing/fixtures/replicaset.py index 0ec3de30280..26079cd9e02 100644 --- a/buildscripts/resmokelib/testing/fixtures/replicaset.py +++ b/buildscripts/resmokelib/testing/fixtures/replicaset.py @@ -186,6 +186,9 @@ class ReplicaSetFixture(interface.ReplFixture): raise time.sleep(5) # Wait a little bit before trying again. + def get_dbpath(self): + return self._dbpath_prefix + def await_ready(self): self._await_primary() self._await_secondaries() diff --git a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py index 2e2db535d6d..ca788cc8dde 100644 --- a/buildscripts/resmokelib/testing/fixtures/shardedcluster.py +++ b/buildscripts/resmokelib/testing/fixtures/shardedcluster.py @@ -88,6 +88,9 @@ class ShardedClusterFixture(interface.Fixture): for shard in self.shards: shard.setup() + def get_dbpath(self): + return self._dbpath_prefix + def await_ready(self): # Wait for the config server if self.configsvr is not None: diff --git a/buildscripts/resmokelib/testing/fixtures/standalone.py b/buildscripts/resmokelib/testing/fixtures/standalone.py index bc69775c285..ffc94ceb4aa 100644 --- a/buildscripts/resmokelib/testing/fixtures/standalone.py +++ b/buildscripts/resmokelib/testing/fixtures/standalone.py @@ -85,6 +85,9 @@ class MongoDFixture(interface.Fixture): self.mongod = mongod + def get_dbpath(self): + return self._dbpath + def await_ready(self): deadline = time.time() + MongoDFixture.AWAIT_READY_TIMEOUT_SECS diff --git a/buildscripts/resmokelib/testing/hook_test_archival.py b/buildscripts/resmokelib/testing/hook_test_archival.py new file mode 100644 index 00000000000..1873485cdbf --- /dev/null +++ b/buildscripts/resmokelib/testing/hook_test_archival.py @@ -0,0 +1,116 @@ +""" +Enables supports for archiving tests or hooks. +""" + +from __future__ import absolute_import + +import os +import threading + +from .. import config +from .. import utils +from ..utils import globstar + + +class HookTestArchival(object): + """ + Archives hooks and tests to S3. + """ + + def __init__(self, suite, hooks, archive_instance, archive_config): + self.archive_instance = archive_instance + archive_config = utils.default_if_none(archive_config, {}) + + self.on_success = archive_config.get("on_success", False) + + self.tests = [] + if "tests" in archive_config: + # 'tests' is either a list of tests to archive or a bool (archive all if True). + if not isinstance(archive_config["tests"], bool): + for test in archive_config["tests"]: + self.tests += globstar.glob(test) + elif archive_config["tests"]: + self.tests = suite.tests + + self.hooks = [] + if "hooks" in archive_config: + # 'hooks' is either a list of hooks to archive or a bool (archive all if True). + if not isinstance(archive_config["hooks"], bool): + self.hooks = archive_config["hooks"] + elif archive_config["hooks"]: + for hook in hooks: + self.hooks.append(hook["class"]) + + self._tests_repeat = {} + self._lock = threading.Lock() + + def _should_archive(self, success): + """ Return True if failed test or 'on_success' is True. """ + return not success or self.on_success + + def _archive_hook(self, logger, hook, test, success): + """ Helper to archive hooks. """ + hook_match = hook.__class__.__name__ in self.hooks + if not hook_match or not self._should_archive(success): + return + + test_name = "{}:{}".format(test.short_name(), hook.__class__.__name__) + self._archive_hook_or_test(logger, test_name, test) + + def _archive_test(self, logger, test, success): + """ Helper to archive tests. """ + test_name = test.test_name + test_match = False + for arch_test in self.tests: + # Ensure that the test_name is in the same format as the arch_test. + if os.path.normpath(test_name) == os.path.normpath(arch_test): + test_match = True + break + if not test_match or not self._should_archive(success): + return + + self._archive_hook_or_test(logger, test_name, test) + + def archive(self, logger, test, success, hook=None): + """ Archives data files for hooks or tests. """ + if not config.ARCHIVE_FILE or not self.archive_instance: + return + if hook: + self._archive_hook(logger, hook, test, success) + else: + self._archive_test(logger, test, success) + + def _archive_hook_or_test(self, logger, test_name, test): + """ Trigger archive of data files for a test or hook. """ + + with self._lock: + # Test repeat number is how many times the particular test has been archived. + if test_name not in self._tests_repeat: + self._tests_repeat[test_name] = 0 + else: + self._tests_repeat[test_name] += 1 + logger.info("Archiving data files for test %s", test_name) + # Normalize test path from a test or hook name. + test_path = \ + test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_") + file_name = "mongo-data-{}-{}-{}-{}.tgz".format( + config.EVERGREEN_TASK_ID, + test_path, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + # Retrieve root directory for all dbPaths from fixture. + input_files = test.fixture.get_dbpath() + s3_bucket = config.ARCHIVE_BUCKET + s3_path = "{}/{}/{}/datafiles/{}".format( + config.EVERGREEN_PROJECT_NAME, + config.EVERGREEN_VARIANT_NAME, + config.EVERGREEN_REVISION, + file_name) + display_name = "Data files {} - Execution {} Repetition {}".format( + test_name, + config.EVERGREEN_EXECUTION, + self._tests_repeat[test_name]) + status, message = self.archive_instance.archive_files_to_s3( + display_name, input_files, s3_bucket, s3_path) + if status: + logger.warning("Archive failed for %s: %s", test_name, message) diff --git a/buildscripts/resmokelib/testing/job.py b/buildscripts/resmokelib/testing/job.py index 4d17152b17d..c8be906dd3a 100644 --- a/buildscripts/resmokelib/testing/job.py +++ b/buildscripts/resmokelib/testing/job.py @@ -18,7 +18,7 @@ class Job(object): Runs tests from a queue. """ - def __init__(self, logger, fixture, hooks, report): + def __init__(self, logger, fixture, hooks, report, archival): """ Initializes the job with the specified fixture and custom behaviors. @@ -28,6 +28,7 @@ class Job(object): self.fixture = fixture self.hooks = hooks self.report = report + self.archival = archival def __call__(self, queue, interrupt_flag, teardown_flag=None): """ @@ -98,20 +99,38 @@ class Job(object): self._run_hooks_before_tests(test) test(self.report) - if config.FAIL_FAST and not self.report.wasSuccessful(): - self.logger.info("%s failed, so stopping..." % (test.shortDescription())) - raise errors.StopExecution("%s failed" % (test.shortDescription())) - - if not self.fixture.is_running(): - self.logger.error("%s marked as a failure because the fixture crashed during the test.", - test.shortDescription()) - self.report.setFailure(test, return_code=2) - # Always fail fast if the fixture fails. - raise errors.StopExecution("%s not running after %s" % - (self.fixture, test.shortDescription())) + try: + if config.FAIL_FAST and not self.report.wasSuccessful(): + self.logger.info("%s failed, so stopping..." % (test.shortDescription())) + raise errors.StopExecution("%s failed" % (test.shortDescription())) + + if not self.fixture.is_running(): + self.logger.error( + "%s marked as a failure because the fixture crashed during the test.", + test.shortDescription()) + self.report.setFailure(test, return_code=2) + # Always fail fast if the fixture fails. + raise errors.StopExecution("%s not running after %s" % + (self.fixture, test.shortDescription())) + + finally: + success = self.report.find_test_info(test).status == "pass" + self.archival.archive(self.logger, test, success) + if self.archival: + self.archival.archive(self.logger, test, success) self._run_hooks_after_tests(test) + def _run_hook(self, hook, hook_function, test): + """Provide helper to run hook and archival.""" + try: + success = False + hook_function(test, self.report) + success = True + finally: + if self.archival: + self.archival.archive(self.logger, test, success, hook=hook) + def _run_hooks_before_tests(self, test): """ Runs the before_test method on each of the hooks. @@ -122,7 +141,7 @@ class Job(object): try: for hook in self.hooks: - hook.before_test(test, self.report) + self._run_hook(hook, hook.before_test, test) except errors.StopExecution: raise @@ -156,7 +175,7 @@ class Job(object): """ try: for hook in self.hooks: - hook.after_test(test, self.report) + self._run_hook(hook, hook.after_test, test) except errors.StopExecution: raise diff --git a/buildscripts/resmokelib/testing/report.py b/buildscripts/resmokelib/testing/report.py index b9cccccc14f..e3c68393a4d 100644 --- a/buildscripts/resmokelib/testing/report.py +++ b/buildscripts/resmokelib/testing/report.py @@ -147,7 +147,7 @@ class TestReport(unittest.TestResult): unittest.TestResult.stopTest(self, test) with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.end_time = time.time() time_taken = test_info.end_time - test_info.start_time @@ -177,7 +177,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_errored += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "error" test_info.return_code = test.return_code @@ -187,7 +187,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -211,7 +211,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_failed += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "fail" test_info.return_code = test.return_code @@ -221,7 +221,7 @@ class TestReport(unittest.TestResult): """ with self._lock: - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) if test_info.end_time is None: raise ValueError("stopTest was not called on %s" % (test.basename())) @@ -244,7 +244,7 @@ class TestReport(unittest.TestResult): with self._lock: self.num_succeeded += 1 - test_info = self._find_test_info(test) + test_info = self.find_test_info(test) test_info.status = "pass" test_info.return_code = test.return_code @@ -352,7 +352,7 @@ class TestReport(unittest.TestResult): # protecting it with the lock. self.__original_loggers = {} - def _find_test_info(self, test): + def find_test_info(self, test): """ Returns the status and timing information associated with 'test'. diff --git a/buildscripts/resmokelib/utils/__init__.py b/buildscripts/resmokelib/utils/__init__.py index df387cc3323..bdd169af132 100644 --- a/buildscripts/resmokelib/utils/__init__.py +++ b/buildscripts/resmokelib/utils/__init__.py @@ -9,6 +9,8 @@ import os.path import pymongo import yaml +from . import archival + def default_if_none(value, default): return value if value is not None else default diff --git a/buildscripts/resmokelib/utils/archival.py b/buildscripts/resmokelib/utils/archival.py index baafe90778c..9d31e053846 100644 --- a/buildscripts/resmokelib/utils/archival.py +++ b/buildscripts/resmokelib/utils/archival.py @@ -9,11 +9,14 @@ import collections import json import math import os +import sys import tarfile import tempfile import threading import time +_IS_WINDOWS = sys.platform == "win32" or sys.platform == "cygwin" + UploadArgs = collections.namedtuple( "UploadArgs", ["archival_file", @@ -128,6 +131,10 @@ class Archival(object): Returns status and message, where message contains information if status is non-0. """ + # TODO: Support archival on Windows (SERVER-33144). + if _IS_WINDOWS: + return 1, "Archival not supported on Windows" + start_time = time.time() with self._lock: if not input_files: @@ -135,10 +142,10 @@ class Archival(object): message = "No input_files specified" elif self.limit_size_mb and self.size_mb >= self.limit_size_mb: status = 1 - message = "Files not archived, limit size {}MB reached".format(self.limit_size_mb) + message = "Files not archived, {}MB size limit reached".format(self.limit_size_mb) elif self.limit_files and self.num_files >= self.limit_files: status = 1 - message = "Files not archived, limit files {} reached".format(self.limit_files) + message = "Files not archived, {} file limit reached".format(self.limit_files) else: status, message, file_size_mb = self._archive_files( display_name, @@ -203,7 +210,10 @@ class Archival(object): logger.exception("Upload to S3 error %s", err) if upload_args.delete_file: - os.remove(upload_args.local_file) + try: + os.remove(upload_args.local_file) + except Exception as err: + logger.exception("Upload to S3 file removal error %s", err) remote_file = "https://s3.amazonaws.com/{}/{}".format( upload_args.s3_bucket, upload_args.s3_path) @@ -232,28 +242,27 @@ class Archival(object): size_mb = 0 # Tar/gzip to a temporary file. - temp_file = tempfile.NamedTemporaryFile(suffix=".tgz", delete=False) - local_file = temp_file.name + _, temp_file = tempfile.mkstemp(suffix=".tgz") # Check if there is sufficient space for the temporary tgz file. - if file_list_size(input_files) > free_space(local_file): - os.remove(local_file) + if file_list_size(input_files) > free_space(temp_file): + os.remove(temp_file) return 1, "Insufficient space for {}".format(message), 0 try: - with tarfile.open(local_file, "w:gz") as tar_handle: + with tarfile.open(temp_file, "w:gz") as tar_handle: for input_file in input_files: tar_handle.add(input_file) except (IOError, tarfile.TarError) as err: - message = str(err) - status = 1 + os.remove(temp_file) + return 1, str(err), 0 - # Round up the size of archive. - size_mb = int(math.ceil(float(file_list_size(local_file)) / (1024 * 1024))) + # Round up the size of the archive. + size_mb = int(math.ceil(float(file_list_size(temp_file)) / (1024 * 1024))) self._upload_queue.put(UploadArgs( self.archival_json_file, display_name, - local_file, + temp_file, "application/x-gzip", s3_bucket, s3_path, diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 502280c9912..83d75f410f9 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -483,16 +483,16 @@ functions: command: shell.exec type: test params: - working_dir: src script: | # exit immediately if virtualenv is not found set -o errexit set -o verbose - if [ ${build_variant} == enterprise-suse12-64 ]; then - virtualenv ./venv - else - virtualenv --system-site-packages ./venv + python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2}) + if [ "Windows_NT" = "$OS" ]; then + python_loc=$(cygpath -w $python_loc) fi + # Set up virtualenv in ${workdir} + virtualenv --python "$python_loc" --system-site-packages "${workdir}/venv" "run tests" : - command: expansions.update @@ -520,14 +520,8 @@ functions: export TMPDIR="${workdir}/tmp" mkdir -p $TMPDIR - # check if virtualenv is set up - if [ -d "venv" ]; then - if [ "Windows_NT" = "$OS" ]; then - . ./venv/Scripts/activate - else - . ./venv/bin/activate - fi - fi + ${activate_virtualenv} + pip install boto3 if [ -f /proc/self/coredump_filter ]; then # Set the shell process (and its children processes) to dump ELF headers (bit 4), @@ -608,10 +602,18 @@ functions: ${lang_environment} \ ${san_options} \ ${rlp_environment} \ - ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py \ + $python buildscripts/resmoke.py \ ${resmoke_args} \ $extra_args ${test_flags} \ --log=buildlogger \ + --taskId=${task_id} \ + --taskName=${task_name} \ + --executionNumber=${execution} \ + --projectName=${project} \ + --variantName=${build_variant} \ + --distroId=${distro_id} \ + --gitRevision=${revision} \ + --archiveFile=archive.json \ --reportFile=report.json @@ -912,9 +914,28 @@ pre: - command: shell.track - func: "kill processes" - func: "cleanup environment" + - func: "set up virtualenv" - command: expansions.update params: updates: + - key: activate_virtualenv + value: | + # check if virtualenv is set up + if [ -d "${workdir}/venv" ]; then + if [ "Windows_NT" = "$OS" ]; then + # Need to quote the path on Windows to preserve the separator. + . "${workdir}/venv/Scripts/activate" 2> /tmp/activate_error.log + else + . ${workdir}/venv/bin/activate 2> /tmp/activate_error.log + fi + if [ $? -ne 0 ]; then + echo "Failed to activate virtualenv: $(cat /tmp/activate_error.log)" + fi + python=python + else + python=${python|/opt/mongodbtoolchain/v2/bin/python2} + fi + echo "python set to $(which $python)" - key: set_sudo value: | set -o | grep errexit | grep on @@ -986,32 +1007,6 @@ post: echo "No OOM (Out of memory) killed processes detected" fi - # Gather and archive FTDC data. - - command: shell.exec - params: - working_dir: src - script: | - # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories. - # The archive.targz_pack command is not being used here because the command's glob support - # did not allow us to gather all directories. - if [ -d /data/db ]; then - file_list=$(cd /data/db && find . -type d -name diagnostic.data) - if [ -n "$file_list" ]; then - ${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list - fi - fi - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/diagnostic-data.tgz - remote_file: mongodb-mongo-v3.4/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: FTDC Diagnostic Data - Execution ${execution} - optional: true - # Process and save coverage data. - command: shell.exec params: @@ -1477,15 +1472,11 @@ tasks: enterprise: ${enterprise_rev} - command: shell.exec - func: "do setup" - - func: "set up virtualenv" - command: shell.exec params: working_dir: burn_in_tests_clonedir script: | set -o errexit - # Create a symbolic link to the venv in the src directory so activate_virtualenv can use it - # if it exists. - ln -s ../src/venv venv ${activate_virtualenv} set -o verbose # If this is a scheduled build, we check for changes against the last scheduled commit. @@ -1496,7 +1487,7 @@ tasks: # list of dbtest suites. cp ../src/dbtest${exe} . # Capture a list of new and modified tests. - ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args + $python buildscripts/burn_in_tests.py --branch=${branch_name} --buildVariant=${build_variant} --testListOutfile=jstests/new_tests.json --noExec $burn_in_args # Copy the results to the src dir. cp jstests/new_tests.json ../src/jstests/new_tests.json - func: "do multiversion setup" @@ -1937,7 +1928,6 @@ tasks: name: ese_WT commands: - func: "do setup" - - func: "set up virtualenv" - command: shell.exec type: test params: @@ -1946,11 +1936,7 @@ tasks: set -o errexit set -o verbose - if [ "Windows_NT" = "$OS" ]; then - . ./venv/Scripts/activate - else - . ./venv/bin/activate - fi + ${activate_virtualenv} # we should go back to using pip when it works on SLES easy_install --upgrade `cat src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt` - func: "run tests" |