blob: 99daa890f0daeeeca70a2385bdae32667dab35ed (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
. "$DIR/prelude.sh"
cd src
# Export these before verbose is set to avoid sharing sensitive info.
export CEDAR_USERNAME=${cedar_user}
export CEDAR_API_KEY=${cedar_api_key}
set -o errexit
set -o verbose
if [[ ${disable_unit_tests} = "false" && ! -f ${skip_tests} ]]; then
# activate the virtualenv if it has been set up
activate_venv
if [[ -f "patch_test_tags.tgz" ]]; then
tags_build_variant="${build_variant}"
# TODO SERVER-56382: create a more robust mapping between query builders and existing required builders.
if [[ "${build_variant}" =~ .*"-query-patch-only" ]]; then
# Use the RHEL 8 all feature flags variant for the classic engine variant. The original
# classic engine variant is not a required builder and therefore not captured in patch
# test failure history.
tags_build_variant="enterprise-rhel-80-64-bit-dynamic-all-feature-flags-required"
fi
$python buildscripts/testmatrix/getdisplaytaskname.py "${task_name}" "${build_variant}" >display_task_name.txt
display_task_name=$(cat display_task_name.txt)
tar -xzf patch_test_tags.tgz
calculated_tags_file_path="failedtesttags/${tags_build_variant}/${display_task_name}.yml"
if [[ -f $calculated_tags_file_path ]]; then
extra_args="$extra_args --tagFile=failedtesttags/${tags_build_variant}/${display_task_name}.yml --includeWithAllTags=recent_failure"
else
echo "calculated tags file does not exist: $calculated_tags_file_path"
fi
fi
# on *SAN builds, extract the debug symbols so they're available
# to the symbolizer
if [[ -n "${san_options}" ]]; then
# the debug symbols archive isn't always available (not every *SAN
# task requires compile)
if [[ -f "mongo-debugsymbols.tgz" ]]; then
tar xf mongo-debugsymbols.tgz
else
echo "mongo-debugsymbols.tgz is not available. If you're seeing this message in a task that uses mongod or mongos binaries, please ensure debug symbols have been generated, otherwise the llvm-symbolizer may not correctly symbolize the sanitizer output."
fi
fi
# Set the TMPDIR environment variable to be a directory in the task's working
# directory so that temporary files created by processes spawned by resmoke.py get
# cleaned up after the task completes. This also ensures the spawned processes
# aren't impacted by limited space in the mount point for the /tmp directory.
export TMPDIR="${workdir}/tmp"
mkdir -p $TMPDIR
if [ -f /proc/self/coredump_filter ]; then
# Set the shell process (and its children processes) to dump ELF headers (bit 4),
# anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
echo 0x13 >/proc/self/coredump_filter
if [ -f /sbin/sysctl ]; then
# Check that the core pattern is set explicitly on our distro image instead
# of being the OS's default value. This ensures that coredump names are consistent
# across distros and can be picked up by Evergreen.
core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
if [ "$core_pattern" = "dump_%e.%p.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
fi
if [ $(uname -s) == "Darwin" ]; then
core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
echo "Enabling coredumps"
ulimit -c unlimited
fi
fi
extra_args="$extra_args --jobs=${resmoke_jobs}"
if [ ${should_shuffle} = true ]; then
extra_args="$extra_args --shuffle"
fi
if [ ${continue_on_failure} = true ]; then
extra_args="$extra_args --continueOnFailure"
fi
# We reduce the storage engine's cache size to reduce the likelihood of a mongod process
# being killed by the OOM killer. The --storageEngineCacheSizeGB command line option is only
# filled in with a default value here if one hasn't already been specified in the task's
# definition or build variant's definition.
set +o errexit
echo "${resmoke_args} ${test_flags}" | grep -q storageEngineCacheSizeGB
if [ $? -eq 1 ]; then
echo "${resmoke_args} ${test_flags}" | grep -q "\-\-storageEngine=inMemory"
if [ $? -eq 0 ]; then
# We use a default of 4GB for the InMemory storage engine.
extra_args="$extra_args --storageEngineCacheSizeGB=4"
else
# We use a default of 1GB for all other storage engines.
extra_args="$extra_args --storageEngineCacheSizeGB=1"
fi
fi
set -o errexit
# Reduce the JSHeapLimit for the serial_run task task on Code Coverage builder variant.
if [[ "${build_variant}" = "enterprise-rhel-80-64-bit-coverage" && "${task_name}" = "serial_run" ]]; then
extra_args="$extra_args --mongodSetParameter \"{'jsHeapLimitMB':10}\""
fi
path_value="$PATH:/data/multiversion"
# The "resmoke_wrapper" expansion is used by the 'burn_in_tests' task to wrap the resmoke.py
# invocation. It doesn't set any environment variables and should therefore come last in
# this list of expansions.
set +o errexit
PATH="$path_value" \
AWS_PROFILE=${aws_profile_remote} \
eval \
${gcov_environment} \
${lang_environment} \
${san_options} \
${snmp_config_path} \
${resmoke_wrapper} \
$python buildscripts/resmoke.py run \
${record_with} \
${resmoke_args} \
$extra_args \
${test_flags} \
--log=buildlogger \
--staggerJobs=on \
--installDir=${install_dir} \
--buildId=${build_id} \
--distroId=${distro_id} \
--executionNumber=${execution} \
--projectName=${project} \
--gitRevision=${revision} \
--revisionOrderId=${revision_order_id} \
--taskId=${task_id} \
--taskName=${task_name} \
--variantName=${build_variant} \
--versionId=${version_id} \
--reportFile=report.json \
--perfReportFile=perf.json
resmoke_exit_code=$?
set -o errexit
if [[ -n "${record_with}" ]]; then
recording_size=$( (du -ch ./*.undo ./*.undo.tokeep || true) | grep total)
echo "UndoDB produced recordings that were $recording_size (uncompressed) on disk"
# Unittests recordings are renamed so there's never a need to store any .undo files.
if [[ $resmoke_exit_code = 0 || "${task_name}" == "run_unittests_with_recording" ]]; then
echo "Removing UndoDB recordings of successful tests."
rm *.undo || true
fi
fi
# 74 is exit code for IOError on POSIX systems, which is raised when the machine is
# shutting down.
#
# 75 is exit code resmoke.py uses when the log output would be incomplete due to failing
# to communicate with logkeeper.
if [[ $resmoke_exit_code = 74 || $resmoke_exit_code = 75 ]]; then
echo $resmoke_exit_code >run_tests_infrastructure_failure
exit 0
elif [ $resmoke_exit_code != 0 ]; then
# On failure save the resmoke exit code.
echo $resmoke_exit_code >resmoke_error_code
elif [ $resmoke_exit_code = 0 ]; then
# On success delete core files.
core_files=$(/usr/bin/find -H .. \( -name "*.core" -o -name "*.mdmp" \) 2>/dev/null)
rm -rf $core_files
fi
exit $resmoke_exit_code
fi # end if [[ ${disable_unit_tests} && ! -f ${skip_tests|/dev/null} ]]
|
