SERVER-25321 Archive mongod logs for Jepsen tasks, even on failure.

Reorganize post processing step

1 files changed, 127 insertions, 95 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 4d0539187dc..1c74905f743 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -457,8 +457,6 @@ functions:
           start_time=$(date +%s)
           lein run --tarball "file:///root/mongo-binaries.tgz" ${jepsen_key_time_limit} ${jepsen_protocol_version} ${jepsen_read_concern} ${jepsen_read_with_find_and_modify} ${jepsen_storage_engine} ${jepsen_time_limit} ${jepsen_write_concern}
           exit_code=$?
-          # Create exit_code file, so we can signal an error, after the archive/put steps have run.
-          echo $exit_code > ../../jepsen-exit_code-${task_id}-${execution}.txt
           end_time=$(date +%s)
           elapsed_secs=$((end_time-start_time))
           if [ $exit_code -eq 0 ]; then
@@ -470,51 +468,7 @@ functions:
           fi
           # Create report.json
           echo "{\"failures\": $failures, \"results\": [{\"status\": $status, \"exit_code\": $exit_code, \"test_file\": \"${task_name}\", \"start\": $start_time, \"end\": $end_time, \"elapsed\": $elapsed_secs}]}" > ../../report.json
-    - command: shell.exec
-      params:
-        working_dir: src/jepsen/mongodb
-        script: |
-          # Copy mongod.log from each LXC host for archive purposes
-          numVms=5
-          for i in $(seq 1 $numVms)
-          do
-            hostn=n$i
-            if [ ! -d store/latest ]; then
-              echo "Creating the store/latest directory, which was not created by Jepsen"
-              mkdir -p store/latest
-            fi
-            mongod_log=store/latest/mongod-$hostn.log
-            scp root@$hostn:mongod.log $mongod_log
-            if [ -f $mongod_log ]; then
-              echo "Copied mongod.log from $hostn to $mongod_log"
-            else
-              echo "Failed to copy mongod.log from $hostn to $mongod_log"
-            fi
-          done
-          # Always exit without an error so the archive steps will run.
-          exit 0
-    - command: archive.targz_pack
-      params:
-        target: "src/jepsen-results.tgz"
-        source_dir: "src/jepsen/mongodb/store"
-        include:
-          - "./**"
-    - command: s3.put
-      params:
-        aws_key: ${aws_key}
-        aws_secret: ${aws_secret}
-        local_file: src/jepsen-results.tgz
-        remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-results-${task_id}-${execution}.tgz
-        bucket: mciuploads
-        permissions: public-read
-        content_type: ${content_type|application/x-gzip}
-        display_name: Jepsen Test Results - ${execution}
-    - command: shell.exec
-      type: test
-      params:
-        working_dir: src
-        script: |
-          exit $(cat jepsen-exit_code-${task_id}-${execution}.txt)
+          exit $exit_code
 
   "run jstestfuzz":
     - command: shell.exec
@@ -697,36 +651,127 @@ post:
     params:
       file_location: src/report.json
   - func: "kill processes"
-  - command: archive.targz_pack
+
+  # Print out any Out of Memory killed process messages.
+  - command: shell.exec
     params:
-      target: "mongo-coredumps.tgz"
-      source_dir: "./"
-      include:
-        - "./**.mdmp"
-  # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories.  The
-  # archive.targz_pack command is not being used here because the command's glob support did not
-  # allow us to gather all directories.
+      system_log: true
+      working_dir: src # Temporary files created in src will be cleaned up in "pre".
+      script: |
+        ${set_sudo}
+        # Use dmesg -T option, if supported, to display timestamps.
+        dmesg=dmesg
+        dmesg -T > /dev/null 2>&1
+        if [ $? -eq 0 ]; then
+          dmesg="dmesg -T"
+        fi
+        $sudo $dmesg 2> /dev/null > dmesg.txt
+        if [ $? -ne 0 ]; then
+          echo "Cannot check for OOM (Out of memory) killed processes on this platform"
+          exit 0
+        fi
+        egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt
+        if [ -s oom.txt ]; then
+          echo "OOM (Out of memory) killed processes detected"
+          cat oom.txt
+        else
+          echo "No OOM (Out of memory) killed processes detected"
+        fi
+
+  # Gather and archive FTDC data.
   - command: shell.exec
     params:
       working_dir: src
       script: |
+        # Using shell and tar to recurse properly to all possible diagnostic.data subdirectories.
+        # The archive.targz_pack command is not being used here because the command's glob support
+        # did not allow us to gather all directories.
         if [ -d /data/db ]; then
             file_list=$(cd /data/db && find . -type d -name diagnostic.data)
             if [ -n "$file_list" ]; then
                 ${tar|tar} cvzf diagnostic-data.tgz -C /data/db $file_list
             fi
         fi
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/diagnostic-data.tgz
+      remote_file: mongodb-mongo-master/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/x-gzip}
+      display_name: FTDC Diagnostic Data - Execution ${execution}
+      optional: true
+
+  # Gather and archive the Jepsen mongod logs.
   - command: shell.exec
     params:
       working_dir: src
       script: |
-        # removes files from the (local) scons cache when it's over a
-        # threshold, to the $prune_ratio percentage. Ideally override
-        # these default values in the distro config in evergreen.
-
-        if [ -d "${scons_cache_path}" ]; then
-            ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
+        # For Jepsen tasks, archive the mongod log files.
+        if [ -d jepsen ]; then
+          # Copy mongod.log from each LXC host for archive purposes. The log can reside in
+          # 1 of 2 places:
+          #   If the task completed: /root
+          #   If the task failed to complete: /opt/mongodb
+          # There may be a mongod.log in both places, but the one in /opt/mongodb should override
+          # the one in /root, since the log in /root could be from a previous successful run.
+          numVms=5
+          for i in $(seq 1 $numVms)
+          do
+            hostn=n$i
+            mongod_log=mongod-$hostn.log
+            for mongod_log_remote in /root/mongod.log /opt/mongodb/mongod.log
+            do
+              scp -q root@$hostn:$mongod_log_remote $mongod_log 2> /dev/null
+              if [ -f $mongod_log ]; then
+                echo "Copied $mongod_log_remote from $hostn to $mongod_log"
+              fi
+            done
+            if [ ! -f $mongod_log ]; then
+              echo "No mongod.log from $hostn copied to $mongod_log"
+            fi
+          done
         fi
+  - command: archive.targz_pack
+    params:
+      target: "src/jepsen-mongod-logs.tgz"
+      source_dir: "src"
+      include:
+        - "mongod*.log"
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/jepsen-mongod-logs.tgz
+      remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-mongod-logs-${task_id}-${execution}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/x-gzip}
+      display_name: Jepsen mongod Logs - ${execution}
+      optional: true
+
+  # Gather and archive the Jepsen results.
+  - command: archive.targz_pack
+    params:
+      target: "src/jepsen-results.tgz"
+      source_dir: "src/jepsen/mongodb/store"
+      include:
+        - "./**"
+  - command: s3.put
+    params:
+      aws_key: ${aws_key}
+      aws_secret: ${aws_secret}
+      local_file: src/jepsen-results.tgz
+      remote_file: mongodb-mongo-master/${build_variant}/${revision}/jepsen/jepsen-results-${task_id}-${execution}.tgz
+      bucket: mciuploads
+      permissions: public-read
+      content_type: ${content_type|application/x-gzip}
+      display_name: Jepsen Test Results - ${execution}
+      optional: true
+
+  # Gather and archive mongo coredumps.
   - command: shell.exec
     params:
       working_dir: src
@@ -737,11 +782,10 @@ post:
         fi
   - command: archive.targz_pack
     params:
-      target: "diskstats.tgz"
+      target: "mongo-coredumps.tgz"
       source_dir: "./"
       include:
-        - "./mongo-diskstats*"
-        - "./mongo-diskstats*.csv"
+        - "./**.mdmp"
   - command: s3.put
     params:
       aws_key: ${aws_key}
@@ -753,16 +797,15 @@ post:
       content_type: ${content_type|application/x-gzip}
       display_name: Core Dumps - Execution ${execution}
       optional: true
-  - command: s3.put
+
+  # Gather and archive disk statistics.
+  - command: archive.targz_pack
     params:
-      aws_key: ${aws_key}
-      aws_secret: ${aws_secret}
-      local_file: src/diagnostic-data.tgz
-      remote_file: mongodb-mongo-master/${build_variant}/${revision}/ftdc/mongo-diagnostic-data-${task_id}-${execution}.tgz
-      bucket: mciuploads
-      permissions: public-read
-      content_type: ${content_type|application/x-gzip}
-      display_name: FTDC Diagnostic Data - Execution ${execution}
+      target: "diskstats.tgz"
+      source_dir: "./"
+      include:
+        - "./mongo-diskstats*"
+        - "./mongo-diskstats*.csv"
   - command: s3.put
     params:
       aws_key: ${aws_key}
@@ -774,34 +817,23 @@ post:
       content_type: ${content_type|application/x-gzip}
       display_name: Disk Stats - Execution ${execution}
       optional: true
+
+  # Cleanup steps.
   - command: shell.exec
     params:
+      working_dir: src
       script: |
-        rm -rf ~/.aws
+        # removes files from the (local) scons cache when it's over a
+        # threshold, to the $prune_ratio percentage. Ideally override
+        # these default values in the distro config in evergreen.
+
+        if [ -d "${scons_cache_path}" ]; then
+            ${python|python} buildscripts/scons_cache_prune.py --cache-dir '${scons_cache_path}' --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8}
+        fi
   - command: shell.exec
     params:
-      system_log: true
-      working_dir: src # Temporary files created in src will be cleaned up in "pre".
       script: |
-        ${set_sudo}
-        # Use dmesg -T option, if supported, to display timestamps.
-        dmesg=dmesg
-        dmesg -T > /dev/null 2>&1
-        if [ $? -eq 0 ]; then
-          dmesg="dmesg -T"
-        fi
-        $sudo $dmesg 2> /dev/null > dmesg.txt
-        if [ $? -ne 0 ]; then
-          echo "Cannot check for OOM (Out of memory) killed processes on this platform"
-          exit 0
-        fi
-        egrep -i '(Out of memory|OOM[- ]killer|Killed process)' dmesg.txt > oom.txt
-        if [ -s oom.txt ]; then
-          echo "OOM (Out of memory) killed processes detected"
-          cat oom.txt
-        else
-          echo "No OOM (Out of memory) killed processes detected"
-        fi
+        rm -rf ~/.aws
   - command: shell.cleanup
 
 timeout: