1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
"""
Enables supports for archiving tests or hooks.
"""
from __future__ import absolute_import
import os
import threading
from .. import config
from .. import utils
from ..utils import globstar
class HookTestArchival(object):
"""
Archives hooks and tests to S3.
"""
def __init__(self, suite, hooks, archive_instance, archive_config):
self.archive_instance = archive_instance
archive_config = utils.default_if_none(archive_config, {})
self.on_success = archive_config.get("on_success", False)
self.tests = []
if "tests" in archive_config:
# 'tests' is either a list of tests to archive or a bool (archive all if True).
if not isinstance(archive_config["tests"], bool):
for test in archive_config["tests"]:
self.tests += globstar.glob(test)
elif archive_config["tests"]:
self.tests = suite.tests
self.hooks = []
if "hooks" in archive_config:
# 'hooks' is either a list of hooks to archive or a bool (archive all if True).
if not isinstance(archive_config["hooks"], bool):
self.hooks = archive_config["hooks"]
elif archive_config["hooks"]:
for hook in hooks:
self.hooks.append(hook["class"])
self._tests_repeat = {}
self._lock = threading.Lock()
def _should_archive(self, success):
""" Return True if failed test or 'on_success' is True. """
return not success or self.on_success
def _archive_hook(self, logger, hook, test, success):
""" Helper to archive hooks. """
hook_match = hook.REGISTERED_NAME in self.hooks
if not hook_match or not self._should_archive(success):
return
test_name = "{}:{}".format(test.short_name(), hook.REGISTERED_NAME)
self._archive_hook_or_test(logger, test_name, test)
def _archive_test(self, logger, test, success):
""" Helper to archive tests. """
test_name = test.test_name
test_match = False
for arch_test in self.tests:
# Ensure that the test_name is in the same format as the arch_test.
if os.path.normpath(test_name) == os.path.normpath(arch_test):
test_match = True
break
if not test_match or not self._should_archive(success):
return
self._archive_hook_or_test(logger, test_name, test)
def archive(self, logger, test, success, hook=None):
""" Archives data files for hooks or tests. """
if not config.ARCHIVE_FILE or not self.archive_instance:
return
if hook:
self._archive_hook(logger, hook, test, success)
else:
self._archive_test(logger, test, success)
def _archive_hook_or_test(self, logger, test_name, test):
""" Trigger archive of data files for a test or hook. """
with self._lock:
# Test repeat number is how many times the particular test has been archived.
if test_name not in self._tests_repeat:
self._tests_repeat[test_name] = 0
else:
self._tests_repeat[test_name] += 1
# Normalize test path from a test or hook name.
test_path = \
test_name.replace("/", "_").replace("\\", "_").replace(".", "_").replace(":", "_")
file_name = "mongo-data-{}-{}-{}-{}.tgz".format(
config.EVERGREEN_TASK_ID,
test_path,
config.EVERGREEN_EXECUTION,
self._tests_repeat[test_name])
# Retrieve root directory for all dbPaths from fixture.
input_files = test.fixture.get_dbpath_prefix()
s3_bucket = config.ARCHIVE_BUCKET
s3_path = "{}/{}/{}/datafiles/{}".format(
config.EVERGREEN_PROJECT_NAME,
config.EVERGREEN_VARIANT_NAME,
config.EVERGREEN_REVISION,
file_name)
display_name = "Data files {} - Execution {} Repetition {}".format(
test_name,
config.EVERGREEN_EXECUTION,
self._tests_repeat[test_name])
logger.info("Archiving data files for test %s from %s", test_name, input_files)
status, message = self.archive_instance.archive_files_to_s3(
display_name, input_files, s3_bucket, s3_path)
if status:
logger.warning("Archive failed for %s: %s", test_name, message)
|
