summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--etc/evergreen.yml222
1 files changed, 127 insertions, 95 deletions
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 4d0539187dc..1c74905f743 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -457,8 +457,6 @@ functions:
start_time=$(date +%s)
lein run --tarball "file:///root/mongo-binaries.tgz" ${jepsen_key_time_limit} ${jepsen_protocol_version} ${jepsen_read_concern} ${jepsen_read_with_find_and_modify} ${jepsen_storage_engine} ${jepsen_time_limit} ${jepsen_write_concern}
exit_code=$?
- # Create exit_code file, so we can signal an error, after the archive/put steps have run.
- echo $exit_code > ../../jepsen-exit_code-${task_id}-${execution}.txt
end_time=$(date +%s)
elapsed_secs=$((end_time-start_time))
if [ $exit_code -eq 0 ]; then
@@ -470,51 +468,7 @@ functions:
fi
# Create report.json
echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../../report.json
- - command: shell.exec
- params:
- working_dir: src/jepsen/mongodb
- script: |
- # Copy mongod.log from each LXC host for archive purposes
- numVms=5
- for i in $(seq 1 $numVms)
- do
- hostn=n$i
- if [ ! -d store/latest ]; then
- echo "Creating the store/latest directory, which was not created by Jepsen"
- mkdir -p store/latest
- fi
- mongod_log=store/latest/mongod-$hostn.log
- scp root@$hostn:mongod.log $mongod_log
- if [ -f $mongod_log ]; then
- echo "Copied mongod.log from $hostn to $mongod_log"
- else
- echo "Failed to copy mongod.log from $hostn to $mongod_log"
- fi
- done
- # Always exit without an error so the archive steps will run.
- exit 0
- - command: archive.targz_pack
- params:
- target: "src/jepsen-results.tgz"
- source_dir: "src/jepsen/mongodb/store"
- include:
- - "./**"
- - command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/jepsen-results.tgz
- remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-results-${task_id}-${execution}.tgz
- bucket: mciuploads
- permissions: public-read
- content_type: ${content_type|application/x-gzip}
- display_name: Jepsen Test Results - ${execution}
- - command: shell.exec
- type: test
- params:
- working_dir: src
- script: |
- exit $(cat jepsen-exit_code-${task_id}-${execution}.txt)
+ exit $exit_code
"run jstestfuzz":
- command: shell.exec
@@ -697,36 +651,127 @@ post:
params:
file_location: src/report.json
- func: "kill processes"
- - command: archive.targz_pack
+
+ # Print out any Out of Memory killed process messages.
+ - command: shell.exec
params:
- target: "mongo-coredumps.tgz"
- source_dir: "./"
- include:
- - "./**.mdmp"
- # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories. The
- # archive.targz_pack command is not being used here because the command's glob support did not
- # allow us to gather all directories.
+ system_log: true
+ working_dir: src # Temporary files created in src will be cleaned up in "pre".
+ script: |
+ ${set_sudo}
+ # Use dmesg -T option, if supported, to display timestamps.
+ dmesg=dmesg
+ dmesg -T > /dev/null 2>&1
+ if [ $? -eq 0 ]; then
+ dmesg="dmesg -T"
+ fi
+ $sudo $dmesg 2> /dev/null > dmesg.txt
+ if [ $? -ne 0 ]; then
+ echo "Cannot check for OOM (Out of memory) killed processes on this platform"
+ exit 0
+ fi
+ egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt
+ if [ -s oom.txt ]; then
+ echo "OOM (Out of memory) killed processes detected"
+ cat oom.txt
+ else
+ echo "No OOM (Out of memory) killed processes detected"
+ fi
+
+ # Gather and archive FTDC data.
- command: shell.exec
params:
working_dir: src
script: |
+ # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories.
+ # The archive.targz_pack command is not being used here because the command's glob support
+ # did not allow us to gather all directories.
if [ -d /data/db ]; then
file_list=$(cd /data/db && find . -type d -name diagnostic.data)
if [ -n "$file_list" ]; then
${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list
fi
fi
+ - command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/diagnostic-data.tgz
+ remote_file: mongodb-mongo-master/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: FTDC Diagnostic Data - Execution ${execution}
+ optional: true
+
+ # Gather and archive the Jepsen mongod logs.
- command: shell.exec
params:
working_dir: src
script: |
- # removes files from the (local) scons cache when it's over a
- # threshold, to the $prune_ratio percentage. Ideally override
- # these default values in the distro config in evergreen.
-
- if [ -d "${scons_cache_path}" ]; then
- ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
+ # For Jepsen tasks, archive the mongod log files.
+ if [ -d jepsen ]; then
+ # Copy mongod.log from each LXC host for archive purposes. The log can reside in
+ # 1 of 2 places:
+ # If the task completed: /root
+ # If the task failed to complete: /opt/mongodb
+ # There may be a mongod.log in both places, but the one in /opt/mongodb should override
+ # the one in /root, since the log in /root could be from a previous successful run.
+ numVms=5
+ for i in $(seq 1 $numVms)
+ do
+ hostn=n$i
+ mongod_log=mongod-$hostn.log
+ for mongod_log_remote in /root/mongod.log /opt/mongodb/mongod.log
+ do
+ scp -q root@$hostn:$mongod_log_remote $mongod_log 2> /dev/null
+ if [ -f $mongod_log ]; then
+ echo "Copied $mongod_log_remote from $hostn to $mongod_log"
+ fi
+ done
+ if [ ! -f $mongod_log ]; then
+ echo "No mongod.log from $hostn copied to $mongod_log"
+ fi
+ done
fi
+ - command: archive.targz_pack
+ params:
+ target: "src/jepsen-mongod-logs.tgz"
+ source_dir: "src"
+ include:
+ - "mongod*.log"
+ - command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/jepsen-mongod-logs.tgz
+ remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-mongod-logs-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Jepsen mongod Logs - ${execution}
+ optional: true
+
+ # Gather and archive the Jepsen results.
+ - command: archive.targz_pack
+ params:
+ target: "src/jepsen-results.tgz"
+ source_dir: "src/jepsen/mongodb/store"
+ include:
+ - "./**"
+ - command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/jepsen-results.tgz
+ remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-results-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Jepsen Test Results - ${execution}
+ optional: true
+
+ # Gather and archive mongo coredumps.
- command: shell.exec
params:
working_dir: src
@@ -737,11 +782,10 @@ post:
fi
- command: archive.targz_pack
params:
- target: "diskstats.tgz"
+ target: "mongo-coredumps.tgz"
source_dir: "./"
include:
- - "./mongo-diskstats*"
- - "./mongo-diskstats*.csv"
+ - "./**.mdmp"
- command: s3.put
params:
aws_key: ${aws_key}
@@ -753,16 +797,15 @@ post:
content_type: ${content_type|application/x-gzip}
display_name: Core Dumps - Execution ${execution}
optional: true
- - command: s3.put
+
+ # Gather and archive disk statistics.
+ - command: archive.targz_pack
params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/diagnostic-data.tgz
- remote_file: mongodb-mongo-master/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz
- bucket: mciuploads
- permissions: public-read
- content_type: ${content_type|application/x-gzip}
- display_name: FTDC Diagnostic Data - Execution ${execution}
+ target: "diskstats.tgz"
+ source_dir: "./"
+ include:
+ - "./mongo-diskstats*"
+ - "./mongo-diskstats*.csv"
- command: s3.put
params:
aws_key: ${aws_key}
@@ -774,34 +817,23 @@ post:
content_type: ${content_type|application/x-gzip}
display_name: Disk Stats - Execution ${execution}
optional: true
+
+ # Cleanup steps.
- command: shell.exec
params:
+ working_dir: src
script: |
- rm -rf ~/.aws
+ # removes files from the (local) scons cache when it's over a
+ # threshold, to the $prune_ratio percentage. Ideally override
+ # these default values in the distro config in evergreen.
+
+ if [ -d "${scons_cache_path}" ]; then
+ ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
+ fi
- command: shell.exec
params:
- system_log: true
- working_dir: src # Temporary files created in src will be cleaned up in "pre".
script: |
- ${set_sudo}
- # Use dmesg -T option, if supported, to display timestamps.
- dmesg=dmesg
- dmesg -T > /dev/null 2>&1
- if [ $? -eq 0 ]; then
- dmesg="dmesg -T"
- fi
- $sudo $dmesg 2> /dev/null > dmesg.txt
- if [ $? -ne 0 ]; then
- echo "Cannot check for OOM (Out of memory) killed processes on this platform"
- exit 0
- fi
- egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt
- if [ -s oom.txt ]; then
- echo "OOM (Out of memory) killed processes detected"
- cat oom.txt
- else
- echo "No OOM (Out of memory) killed processes detected"
- fi
+ rm -rf ~/.aws
- command: shell.cleanup
timeout: