1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
#!/usr/bin/env python3
"""Command line utility for running newly added or modified jstests under the appropriate multiversion passthrough suites."""
import os
import sys
from typing import Dict
import click
from evergreen.api import EvergreenApi
from git import Repo
from shrub.v2 import BuildVariant, ExistingTask, ShrubProject
import structlog
from structlog.stdlib import LoggerFactory
import buildscripts.evergreen_gen_multiversion_tests as gen_multiversion
import buildscripts.evergreen_generate_resmoke_tasks as gen_resmoke
from buildscripts.burn_in_tests import GenerateConfig, DEFAULT_PROJECT, CONFIG_FILE, _configure_logging, RepeatConfig, \
_get_evg_api, EVERGREEN_FILE, DEFAULT_REPO_LOCATIONS, _set_resmoke_cmd, create_tests_by_task, \
find_changed_tests, run_tests
from buildscripts.ciconfig.evergreen import parse_evergreen_file
from buildscripts.patch_builds.task_generation import validate_task_generation_limit
from buildscripts.resmokelib.suitesconfig import get_named_suites_with_root_level_key
from buildscripts.util.fileops import write_file
structlog.configure(logger_factory=LoggerFactory())
LOGGER = structlog.getLogger(__name__)
MULTIVERSION_CONFIG_KEY = gen_multiversion.MULTIVERSION_CONFIG_KEY
MULTIVERSION_PASSTHROUGH_TAG = gen_multiversion.PASSTHROUGH_TAG
RANDOM_MULTIVERSION_REPLSETS_TAG = gen_multiversion.RANDOM_REPLSETS_TAG
BURN_IN_MULTIVERSION_TASK = gen_multiversion.BURN_IN_TASK
TASK_PATH_SUFFIX = "/data/multiversion"
def create_multiversion_generate_tasks_config(tests_by_task: Dict, evg_api: EvergreenApi,
generate_config: GenerateConfig) -> BuildVariant:
"""
Create the multiversion config for the Evergreen generate.tasks file.
:param tests_by_task: Dictionary of tests to generate tasks for.
:param evg_api: Evergreen API.
:param generate_config: Configuration of what to generate.
:return: Shrub configuration with added tasks.
"""
build_variant = BuildVariant(generate_config.build_variant)
tasks = set()
if tests_by_task:
# Get the multiversion suites that will run in as part of burn_in_multiversion.
multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
for suite in multiversion_suites:
idx = 0
if suite["origin"] not in tests_by_task.keys():
# Only generate burn in multiversion tasks for suites that would run the detected
# changed tests.
continue
LOGGER.debug("Generating multiversion suite", suite=suite["multiversion_name"])
# We hardcode the number of fallback sub suites and the target resmoke time here
# since burn_in_tests cares about individual tests and not entire suites. The config
# options here are purely used to generate the proper multiversion suites to run
# tests against.
config_options = {
"suite": suite["origin"],
"fallback_num_sub_suites": 1,
"project": generate_config.project,
"build_variant": generate_config.build_variant,
"task_id": generate_config.task_id,
"task_name": suite["multiversion_name"],
"target_resmoke_time": 60,
}
config_options.update(gen_resmoke.DEFAULT_CONFIG_VALUES)
config_generator = gen_multiversion.EvergreenMultiversionConfigGenerator(
evg_api, gen_resmoke.ConfigOptions(config_options))
test_list = tests_by_task[suite["origin"]]["tests"]
for test in test_list:
# Exclude files that should be blacklisted from multiversion testing.
files_to_exclude = gen_multiversion.get_exclude_files(suite["multiversion_name"],
TASK_PATH_SUFFIX)
LOGGER.debug("Files to exclude", files_to_exclude=files_to_exclude, test=test,
suite=suite["multiversion_name"])
if test not in files_to_exclude:
# Generate the multiversion tasks for each test.
sub_tasks = config_generator.get_burn_in_tasks(test, idx)
tasks = tasks.union(sub_tasks)
idx += 1
existing_tasks = {ExistingTask(f"{BURN_IN_MULTIVERSION_TASK}_gen")}
build_variant.display_task(BURN_IN_MULTIVERSION_TASK, tasks,
execution_existing_tasks=existing_tasks)
return build_variant
@click.command()
@click.option("--no-exec", "no_exec", default=False, is_flag=True,
help="Do not execute the found tests.")
@click.option("--generate-tasks-file", "generate_tasks_file", default=None, metavar='FILE',
help="Run in 'generate.tasks' mode. Store task config to given file.")
@click.option("--build-variant", "build_variant", default=None, metavar='BUILD_VARIANT',
help="Tasks to run will be selected from this build variant.")
@click.option("--run-build-variant", "run_build_variant", default=None, metavar='BUILD_VARIANT',
help="Burn in tasks will be generated on this build variant.")
@click.option("--distro", "distro", default=None, metavar='DISTRO',
help="The distro the tasks will execute on.")
@click.option("--project", "project", default=DEFAULT_PROJECT, metavar='PROJECT',
help="The evergreen project the tasks will execute on.")
@click.option("--evg-api-config", "evg_api_config", default=CONFIG_FILE, metavar="FILE",
help="Configuration file with connection info for Evergreen API.")
@click.option("--verbose", "verbose", default=False, is_flag=True, help="Enable extra logging.")
@click.option("--task_id", "task_id", default=None, metavar='TASK_ID',
help="The evergreen task id.")
@click.argument("resmoke_args", nargs=-1, type=click.UNPROCESSED)
# pylint: disable=too-many-arguments,too-many-locals
def main(build_variant, run_build_variant, distro, project, generate_tasks_file, no_exec,
resmoke_args, evg_api_config, verbose, task_id):
"""
Run new or changed tests in repeated mode to validate their stability.
Running burn_in_tests_multiversion will run new or changed tests against the appropriate generated multiversion
suites. The purpose of these tests are to signal bugs in the generated multiversion suites as these tasks are
excluded from the required build variants and are only run in certain daily build variants. As such, we only expect
the burn-in multiversion tests to be run once for each binary version configuration, and `--repeat-*` arguments
should be None when executing this script.
There are two modes that burn_in_tests_multiversion can run in:
(1) Normal mode: by default burn_in_tests will attempt to run all detected tests the
configured number of times. This is useful if you have a test or tests you would like to
check before submitting a patch to evergreen.
(2) By specifying the `--generate-tasks-file`, burn_in_tests will run generate a configuration
file that can then be sent to the Evergreen 'generate.tasks' command to create evergreen tasks
to do all the test executions. This is the mode used to run tests in patch builds.
NOTE: There is currently a limit of the number of tasks burn_in_tests will attempt to generate
in evergreen. The limit is 1000. If you change enough tests that more than 1000 tasks would
be generated, burn_in_test will fail. This is to avoid generating more tasks than evergreen
can handle.
\f
:param build_variant: Build variant to query tasks from.
:param run_build_variant:Build variant to actually run against.
:param distro: Distro to run tests on.
:param project: Project to run tests on.
:param generate_tasks_file: Create a generate tasks configuration in this file.
:param no_exec: Just perform test discover, do not execute the tests.
:param resmoke_args: Arguments to pass through to resmoke.
:param evg_api_config: Location of configuration file to connect to evergreen.
:param verbose: Log extra debug information.
"""
_configure_logging(verbose)
evg_conf = parse_evergreen_file(EVERGREEN_FILE)
repeat_config = RepeatConfig() # yapf: disable
generate_config = GenerateConfig(build_variant=build_variant,
run_build_variant=run_build_variant,
distro=distro,
project=project,
task_id=task_id) # yapf: disable
if generate_tasks_file:
generate_config.validate(evg_conf)
evg_api = _get_evg_api(evg_api_config, False)
repos = [Repo(x) for x in DEFAULT_REPO_LOCATIONS if os.path.isdir(x)]
resmoke_cmd = _set_resmoke_cmd(repeat_config, list(resmoke_args))
changed_tests = find_changed_tests(repos)
tests_by_task = create_tests_by_task(generate_config.build_variant, evg_conf, changed_tests)
LOGGER.debug("tests and tasks found", tests_by_task=tests_by_task)
if generate_tasks_file:
multiversion_tasks = evg_conf.get_task_names_by_tag(MULTIVERSION_PASSTHROUGH_TAG)
LOGGER.debug("Multiversion tasks by tag", tasks=multiversion_tasks,
tag=MULTIVERSION_PASSTHROUGH_TAG)
# We expect the number of suites with MULTIVERSION_PASSTHROUGH_TAG to be the same as in
# multiversion_suites. Multiversion passthrough suites must include
# MULTIVERSION_CONFIG_KEY as a root level key and must be set to true.
multiversion_suites = get_named_suites_with_root_level_key(MULTIVERSION_CONFIG_KEY)
assert len(multiversion_tasks) == len(multiversion_suites)
build_variant = create_multiversion_generate_tasks_config(tests_by_task, evg_api,
generate_config)
shrub_project = ShrubProject.empty()
shrub_project.add_build_variant(build_variant)
if not validate_task_generation_limit(shrub_project):
sys.exit(1)
write_file(generate_tasks_file, shrub_project.json())
elif not no_exec:
run_tests(tests_by_task, resmoke_cmd)
else:
LOGGER.info("Not running tests due to 'no_exec' option.")
if __name__ == "__main__":
main() # pylint: disable=no-value-for-parameter
|
