diff options
| author | Jonathan Abrahams <jonathan@mongodb.com> | 2016-06-09 15:02:01 -0400 |
|---|---|---|
| committer | Jonathan Abrahams <jonathan@mongodb.com> | 2016-06-21 13:23:29 -0400 |
| commit | 9756d828821e0fdeb9d58e9b993639148d07ba1a (patch) | |
| tree | 3cfec0816ee8cb043936c337da257a66134a7b1c | |
| parent | bf6e2ae440efa10dbc28026d384e6c1241641599 (diff) | |
| download | mongo-9756d828821e0fdeb9d58e9b993639148d07ba1a.tar.gz | |
SERVER-24211 Add OOM killed process messages to system log
(cherry picked from commit 5e26352a654193b3d323b8cb1bac54d41ea03765)
| -rw-r--r-- | etc/evergreen.yml | 51 |
1 files changed, 51 insertions, 0 deletions
diff --git a/etc/evergreen.yml b/etc/evergreen.yml index b65aa6b2681..e0836076d98 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -400,6 +400,25 @@ pre: set -o verbose rm -rf src /data/db/* mongo-coredumps.tgz + - command: expansions.update + params: + updates: + - key: set_sudo + value: | + set -o | grep errexit | grep on + errexit_on=$? + # Set errexit "off". + set +o errexit + sudo= + # Use sudo, if it is supported. + sudo date > /dev/null 2>&1 + if [ $? -eq 0 ]; then + sudo=sudo + fi + # Set errexit "on", if previously enabled. + if [ $errexit_on -eq 0 ]; then + set -o errexit + fi - command: shell.exec params: system_log: true @@ -408,6 +427,14 @@ pre: set -o verbose ulimit -a + # Clear the dmesg ring buffer. The "post" phase will check dmesg for OOM messages. + ${set_sudo} + $sudo dmesg --clear > /dev/null 2>&1 + if [ $? -eq 0 ]; then + echo "Cleared the dmesg ring buffer" + else + echo "Could not clear the dmesg ring buffer" + fi post: - command: attach.results @@ -436,6 +463,30 @@ post: content_type: ${content_type|application/x-gzip} display_name: Core Dumps - Execution ${execution} optional: true + - command: shell.exec + params: + system_log: true + working_dir: src # Temporary files created in src will be cleaned up in "pre". + script: | + ${set_sudo} + # Use dmesg -T option, if supported, to display timestamps. + dmesg=dmesg + dmesg -T > /dev/null 2>&1 + if [ $? -eq 0 ]; then + dmesg="dmesg -T" + fi + $sudo $dmesg 2> /dev/null > dmesg.txt + if [ $? -ne 0 ]; then + echo "Cannot check for OOM (Out of memory) killed processes on this platform" + exit 0 + fi + egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt + if [ -s oom.txt ]; then + echo "OOM (Out of memory) killed processes detected" + cat oom.txt + else + echo "No OOM (Out of memory) killed processes detected" + fi - command: shell.cleanup timeout: |