SERVER-33009 AWS credential profiles in evergreen.yml

1 files changed, 160 insertions, 116 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 23d455b391a..502280c9912 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -342,7 +342,7 @@ functions:
           sleep 60
         done
 
-  "setup credentials" : &setup_credentials
+  "set up credentials" : &set_up_credentials
     command: shell.exec
     params:
       working_dir: src
@@ -356,6 +356,53 @@ functions:
         build_phase='${task_name}_${execution}'
         END_OF_CREDS
 
+  "set up remote credentials": &set_up_remote_credentials
+    command: shell.exec
+    params:
+      silent: true
+      script: |
+        set -o errexit
+
+        # Since the macros 'private_key_remote' and 'private_key_file' are not always defined
+        # we default to /dev/null to avoid syntax errors of an empty expansion.
+        if [ ! -z "${private_key_remote}" ] && [ ! -z "${private_key_file}" ] ; then
+          mkdir -p ~/.ssh
+          echo -n "${private_key_remote}" > ${private_key_file|/dev/null}
+          chmod 0600 ${private_key_file|/dev/null}
+        fi
+
+        if [ ! -d ~.aws ]; then
+          mkdir -p ~/.aws
+        fi
+
+        # If ${aws_profile_remote} is not specified then the config & credentials are
+        # stored in the 'default' profile.
+        aws_profile="${aws_profile_remote|default}"
+
+        # The profile in the config file is specified as [profile <profile>], except
+        # for [default], see http://boto3.readthedocs.io/en/latest/guide/configuration.html
+        if [ $aws_profile = "default" ] ; then
+          aws_profile_config="[default]"
+        else
+          aws_profile_config="[profile $aws_profile]"
+        fi
+        cat <<EOF >> ~/.aws/config
+        $aws_profile_config
+        region = us-east-1
+        EOF
+
+        # The profile in the credentials file is specified as [<profile>].
+        cat <<EOF >> ~/.aws/credentials
+        [$aws_profile]
+        aws_access_key_id = ${aws_key_remote}
+        aws_secret_access_key = ${aws_secret_remote}
+        EOF
+
+        cat <<EOF > ~/.boto
+        [Boto]
+        https_validate_certificates = False
+        EOF
+
   "upload debugsymbols" : &upload_debugsymbols
     command: s3.put
     params:
@@ -428,7 +475,7 @@ functions:
   - *extract_binaries
   - *check_binary_version
   - *get_buildnumber
-  - *setup_credentials
+  - *set_up_credentials
   - *run_diskstats
   - *monitor_process_threads
 
@@ -448,103 +495,124 @@ functions:
           fi
 
   "run tests" :
-    command: shell.exec
-    type: test
-    params:
-      working_dir: src
-      script: |
-        set -o errexit
-        set -o verbose
+    - command: expansions.update
+      params:
+        updates:
+        - key: aws_key_remote
+          value: ${mongodatafiles_aws_key}
+        - key: aws_profile_remote
+          value: mongodata_aws
+        - key: aws_secret_remote
+          value: ${mongodatafiles_aws_secret}
+    - *set_up_remote_credentials
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        script: |
+          set -o errexit
+          set -o verbose
 
-        # Set the TMPDIR environment variable to be a directory in the task's working
-        # directory so that temporary files created by processes spawned by resmoke.py get
-        # cleaned up after the task completes. This also ensures the spawned processes
-        # aren't impacted by limited space in the mount point for the /tmp directory.
-        export TMPDIR="${workdir}/tmp"
-        mkdir -p $TMPDIR
+          # Set the TMPDIR environment variable to be a directory in the task's working
+          # directory so that temporary files created by processes spawned by resmoke.py get
+          # cleaned up after the task completes. This also ensures the spawned processes
+          # aren't impacted by limited space in the mount point for the /tmp directory.
+          export TMPDIR="${workdir}/tmp"
+          mkdir -p $TMPDIR
 
-        # check if virtualenv is set up
-        if [ -d "venv" ]; then
-          if [ "Windows_NT" = "$OS" ]; then
-            . ./venv/Scripts/activate
-          else
-            . ./venv/bin/activate
+          # check if virtualenv is set up
+          if [ -d "venv" ]; then
+            if [ "Windows_NT" = "$OS" ]; then
+              . ./venv/Scripts/activate
+            else
+              . ./venv/bin/activate
+            fi
           fi
-        fi
 
-        if [ -f /proc/self/coredump_filter ]; then
-          # Set the shell process (and its children processes) to dump ELF headers (bit 4),
-          # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
-          echo 0x13 > /proc/self/coredump_filter
-
-          if [ -f /sbin/sysctl ]; then
-            # Check that the core pattern is set explicitly on our distro image instead
-            # of being the OS's default value. This ensures that coredump names are consistent
-            # across distros and can be picked up by Evergreen.
-            core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
-            if [ "$core_pattern" = "dump_%e.%p.core" ]; then
-              echo "Enabling coredumps"
-              ulimit -c unlimited
+          if [ -f /proc/self/coredump_filter ]; then
+            # Set the shell process (and its children processes) to dump ELF headers (bit 4),
+            # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
+            echo 0x13 > /proc/self/coredump_filter
+
+            if [ -f /sbin/sysctl ]; then
+              # Check that the core pattern is set explicitly on our distro image instead
+              # of being the OS's default value. This ensures that coredump names are consistent
+              # across distros and can be picked up by Evergreen.
+              core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
+              if [ "$core_pattern" = "dump_%e.%p.core" ]; then
+                echo "Enabling coredumps"
+                ulimit -c unlimited
+              fi
             fi
           fi
-        fi
 
-        extra_args=""
-        if [ ${run_multiple_jobs|false} = true ]; then
-          processor_architecture=$(uname -m)
-          num_jobs_available=${num_jobs_available|1}
-          # Reduce the number of available jobs by half when running any sharding*, replica_sets*
-          # and select jsCore passthrough tasks on Windows and ARM to avoid overwhelming test hosts.
-          if [ "Windows_NT" = "$OS" ] || [ "aarch64" = $processor_architecture ]; then
-            case "${task_name}" in
-            replica_sets_initsync_jscore_passthrough          \
-            |replica_sets_initsync_static_jscore_passthrough  \
-            |replica_sets_resync_static_jscore_passthrough    \
-            |replica_sets*                           \
-            |sharding*                              \
-            )
-              num_jobs_available=$((${num_jobs_available|2} / 2))
-              echo "Reducing jobs from ${num_jobs_available|1} to $num_jobs_available"
-              ;;
-            esac
-          fi
+          extra_args=""
+          if [ ${run_multiple_jobs|false} = true ]; then
+            processor_architecture=$(uname -m)
+            num_jobs_available=${num_jobs_available|1}
+            # Reduce the number of available jobs by half when running any sharding*, replica_sets*
+            # and select jsCore passthrough tasks on Windows and ARM to avoid overwhelming test hosts.
+            if [ "Windows_NT" = "$OS" ] || [ "aarch64" = $processor_architecture ]; then
+              case "${task_name}" in
+              replica_sets_initsync_jscore_passthrough          \
+              |replica_sets_initsync_static_jscore_passthrough  \
+              |replica_sets_resync_static_jscore_passthrough    \
+              |replica_sets*                           \
+              |sharding*                              \
+              )
+                num_jobs_available=$((${num_jobs_available|2} / 2))
+                echo "Reducing jobs from ${num_jobs_available|1} to $num_jobs_available"
+                ;;
+              esac
+            fi
 
-          if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then
-            extra_args="$extra_args --jobs=${max_jobs}"
-          else
-            extra_args="$extra_args --jobs=$num_jobs_available"
+            if [ ${max_jobs|0} -gt 0 ] && [ ${max_jobs|0} -lt $num_jobs_available ]; then
+              extra_args="$extra_args --jobs=${max_jobs}"
+            else
+              extra_args="$extra_args --jobs=$num_jobs_available"
+            fi
           fi
-        fi
 
-        if [ ${should_shuffle|true} = true ]; then
-          extra_args="$extra_args --shuffle"
-        fi
+          if [ ${should_shuffle|true} = true ]; then
+            extra_args="$extra_args --shuffle"
+          fi
 
-        if [ ${continue_on_failure|true} = true ]; then
-          extra_args="$extra_args --continueOnFailure"
-        fi
+          if [ ${continue_on_failure|true} = true ]; then
+            extra_args="$extra_args --continueOnFailure"
+          fi
 
-        # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed.
-        # resmoke will assign to the appropriate parameter on storage engines that support it.
-        set +o errexit
-        echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB
-        if [ $? -eq 1 ]; then
-          extra_args="$extra_args --storageEngineCacheSizeGB=1"
-        fi
-        set -o errexit
+          # Default storageEngineCacheSizeGB to 1. Override on individual test config if needed.
+          # resmoke will assign to the appropriate parameter on storage engines that support it.
+          set +o errexit
+          echo "${resmoke_args}" | grep -q storageEngineCacheSizeGB
+          if [ $? -eq 1 ]; then
+            extra_args="$extra_args --storageEngineCacheSizeGB=1"
+          fi
+          set -o errexit
 
-        # Combine the exclusion tags from both the variant and the task.
-        extra_args="$extra_args --excludeWithAnyTags=${task_excluded_tags},${variant_excluded_flags}"
+          # Combine the exclusion tags from both the variant and the task.
+          extra_args="$extra_args --excludeWithAnyTags=${task_excluded_tags},${variant_excluded_flags}"
 
-        path_value="$PATH"
-        if [ ${variant_path_suffix} ]; then
-          path_value="$path_value:${variant_path_suffix}"
-        fi
-        if [ ${task_path_suffix} ]; then
-          path_value="$path_value:${task_path_suffix}"
-        fi
+          path_value="$PATH"
+          if [ ${variant_path_suffix} ]; then
+            path_value="$path_value:${variant_path_suffix}"
+          fi
+          if [ ${task_path_suffix} ]; then
+            path_value="$path_value:${task_path_suffix}"
+          fi
 
-        ${resmoke_wrapper} PATH="$path_value" ${san_symbolizer} ${lang_environment} ${san_options} ${rlp_environment} ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py ${resmoke_args} $extra_args ${test_flags} --log=buildlogger --reportFile=report.json
+          ${resmoke_wrapper}                 \
+          PATH="$path_value"                 \
+          AWS_PROFILE=${aws_profile_remote}  \
+          ${san_symbolizer}                  \
+          ${lang_environment}                \
+          ${san_options}                     \
+          ${rlp_environment}                 \
+          ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/resmoke.py  \
+            ${resmoke_args}                  \
+            $extra_args ${test_flags}        \
+            --log=buildlogger                \
+            --reportFile=report.json
 
 
   "do jepsen setup" :
@@ -810,28 +878,6 @@ functions:
           kill_process "$process"
         done
 
-  "set up kitchen credentials":
-    command: shell.exec
-    params:
-      silent: true
-      script: |
-        set -o errexit
-
-        mkdir -p ~/.ssh ~/.aws
-        echo -n "${kitchen_private_key}" > ~/.ssh/kitchen.pem
-        chmod 0600 ~/.ssh/kitchen.pem
-
-        cat <<EOF > ~/.aws/config
-        [default]
-        region = us-east-1
-        EOF
-
-        cat <<EOF > ~/.aws/credentials
-        [default]
-        aws_access_key_id = ${kitchen_aws_key}
-        aws_secret_access_key = ${kitchen_aws_secret}
-        EOF
-
   "run kitchen":
     command: shell.exec
     type: test
@@ -841,13 +887,6 @@ functions:
       script: |
         set -o errexit
 
-        # Override the aws credentials with the kitchen specific credentials
-        cat <<EOF > ~/.aws/credentials
-        [default]
-        aws_access_key_id = ${kitchen_aws_key}
-        aws_secret_access_key = ${kitchen_aws_secret}
-        EOF
-
         export KITCHEN_ARTIFACTS_URL="https://s3.amazonaws.com/mciuploads/${project}/${build_variant}/${revision}/artifacts/${build_id}.tgz"
         export KITCHEN_SECURITY_GROUP="${kitchen_security_group}"
         export KITCHEN_SSH_KEY_ID="${kitchen_ssh_key_id}"
@@ -1228,7 +1267,7 @@ tasks:
       params:
         directory: src
     - func: "get buildnumber"
-    - func: "setup credentials"
+    - func: "set up credentials"
     - func: "build new tools" # noop if ${newtools} is not "true"
     - command: shell.exec
       params:
@@ -1379,7 +1418,7 @@ tasks:
       params:
         directory: src
     - func: "get buildnumber"
-    - func: "setup credentials"
+    - func: "set up credentials"
     - func: "build new tools" # noop if ${newtools} is not "true"
     - *generate_compile_expansions
     # Then we load the generated version data into the agent so we can use it in task definitions.
@@ -3036,7 +3075,12 @@ tasks:
   - name: compile
   commands:
     - func: "fetch artifacts"
-    - func: "set up kitchen credentials"
+    - func: "set up remote credentials"
+      vars:
+        private_key_file: ~/.ssh/kitchen.pem
+        private_key_remote: ${kitchen_private_key}
+        aws_key_remote: ${kitchen_aws_key}
+        aws_secret_remote: ${kitchen_aws_secret}
     - func: "run kitchen"
 
 - name: push