1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
|
#!/usr/bin/env python
"""
Command line utility for executing MongoDB tests in Evergreen.
"""
from __future__ import absolute_import
import collections
import os.path
import sys
# Get relative imports to work when the package is not installed on the PYTHONPATH.
if __name__ == "__main__" and __package__ is None:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from buildscripts import resmoke
from buildscripts import resmokelib
_TagInfo = collections.namedtuple("_TagInfo", ["tag_name", "evergreen_aware", "suite_options"])
class Main(resmoke.Main):
"""
A class for executing potentially multiple resmoke.py test suites in a way that handles
additional options for running unreliable tests in Evergreen.
"""
UNRELIABLE_TAG = _TagInfo(tag_name="unreliable", evergreen_aware=True,
suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace(
report_failure_status="silentfail"))
RESOURCE_INTENSIVE_TAG = _TagInfo(
tag_name="resource_intensive", evergreen_aware=False,
suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace(num_jobs=1))
RETRY_ON_FAILURE_TAG = _TagInfo(
tag_name="retry_on_failure", evergreen_aware=True,
suite_options=resmokelib.config.SuiteOptions.ALL_INHERITED._replace(
fail_fast=False, num_repeats=2, report_failure_status="silentfail"))
def _make_evergreen_aware_tags(self, tag_name):
"""
Returns a list of resmoke.py tags for task, variant, and distro combinations in Evergreen.
"""
tags_format = ["{tag_name}"]
if resmokelib.config.EVERGREEN_TASK_NAME is not None:
tags_format.append("{tag_name}|{task_name}")
if resmokelib.config.EVERGREEN_VARIANT_NAME is not None:
tags_format.append("{tag_name}|{task_name}|{variant_name}")
if resmokelib.config.EVERGREEN_DISTRO_ID is not None:
tags_format.append("{tag_name}|{task_name}|{variant_name}|{distro_id}")
return [
tag.format(tag_name=tag_name, task_name=resmokelib.config.EVERGREEN_TASK_NAME,
variant_name=resmokelib.config.EVERGREEN_VARIANT_NAME,
distro_id=resmokelib.config.EVERGREEN_DISTRO_ID) for tag in tags_format
]
@classmethod
def _make_tag_combinations(cls):
"""
Returns a list of (tag, enabled) pairs representing all possible combinations of all
possible pairings of whether the tags are enabled or disabled together.
"""
combinations = []
if resmokelib.config.EVERGREEN_PATCH_BUILD:
combinations.append(("unreliable and resource intensive",
((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, True))))
combinations.append(("unreliable and not resource intensive",
((cls.UNRELIABLE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG, False))))
combinations.append(("reliable and resource intensive",
((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG, True))))
combinations.append(("reliable and not resource intensive",
((cls.UNRELIABLE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
False))))
else:
combinations.append(("retry on failure and resource intensive",
((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
True))))
combinations.append(("retry on failure and not resource intensive",
((cls.RETRY_ON_FAILURE_TAG, True), (cls.RESOURCE_INTENSIVE_TAG,
False))))
combinations.append(("run once and resource intensive",
((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
True))))
combinations.append(("run once and not resource intensive",
((cls.RETRY_ON_FAILURE_TAG, False), (cls.RESOURCE_INTENSIVE_TAG,
False))))
return combinations
def _get_suites(self):
"""
Returns a list of resmokelib.testing.suite.Suite instances to execute.
For every resmokelib.testing.suite.Suite instance returned by resmoke.Main._get_suites(),
multiple copies of that test suite are run using different resmokelib.config.SuiteOptions()
depending on whether each tag in the combination is enabled or not.
"""
suites = []
for suite in resmoke.Main._get_suites(self):
if suite.test_kind != "js_test":
# Tags are only support for JavaScript tests, so we leave the test suite alone when
# running any other kind of test.
suites.append(suite)
continue
for (tag_desc, tag_combo) in self._make_tag_combinations():
suite_options_list = []
for (tag_info, enabled) in tag_combo:
if tag_info.evergreen_aware:
tags = self._make_evergreen_aware_tags(tag_info.tag_name)
include_tags = {"$anyOf": tags}
else:
include_tags = tag_info.tag_name
if enabled:
suite_options = tag_info.suite_options._replace(include_tags=include_tags)
else:
suite_options = resmokelib.config.SuiteOptions.ALL_INHERITED._replace(
include_tags={"$not": include_tags})
suite_options_list.append(suite_options)
suite_options = resmokelib.config.SuiteOptions.combine(*suite_options_list)
suite_options = suite_options._replace(description=tag_desc)
suites.append(suite.with_options(suite_options))
return suites
if __name__ == "__main__":
Main().run()
|
