SERVER-33813 launch_evergreen_ec2_instance.sh should immediately fail if the call to aws_ec2.py is unsuccessful

1 files changed, 33 insertions, 32 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 9326c8b1089..32caeaeb591 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -165,7 +165,7 @@ variables:
 - &powercycle_test
   ec2_artifacts: ${log_path} ${db_path} ${backup_path_after} ${backup_path_before}
   program_options: --logLevel=info --backupPathBefore=${backup_path_before} --backupPathAfter=${backup_path_after}
-  connection_options: --sshUserHost=${ip_address} --sshConnection=\"${ssh_identity} ${ssh_connection_options}\"
+  connection_options: --sshUserHost=${private_ip_address} --sshConnection=\"${ssh_identity} ${ssh_connection_options}\"
   test_options: --testLoops=15 --seedDocNum=10000 --rsync --rsyncExcludeFiles=diagnostic.data/metrics.interim* --validate=local --canary=local
   crash_options: --crashMethod=internal --crashOption=${windows_crash_cmd} --crashWaitTime=45 --jitterForCrashWaitTime=5 --instanceId=${instance_id}
   client_options: --numCrudClients=20 --numFsmClients=20
@@ -1454,25 +1454,30 @@ functions:
           if [ -z "${ec2_monitor_files}" ] || [ -z "${instance_id}" ]; then
             exit 0
           fi
-          # Ensure we use the latest ip_address, as it could change if the EC2 instance
+          # Ensure we use the latest private_ip_address, as it could change if the EC2 instance
           # has been stopped and started.
           ${activate_virtualenv}
           # Specify '--mode start' to ensure the remote instance is running.
-          aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start)
+          monitor_ec2_yml=monitor_ec2.yml
+          aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $monitor_ec2_yml)
           echo "AMI EC2 instance ${instance_id} status: $aws_ec2"
-          ip_address=$(echo $aws_ec2 | sed -e "s/[^[:print:]\t]//g; s/.*private_ip_address: //; s/ .*//")
+          private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $monitor_ec2_yml --yamlKey private_ip_address)
+          if [ -z "$private_ip_address" ]; then
+            echo "Cannot determine the IP address for the remote monitor."
+            continue
+          fi
           cmd="${tar|tar} czf ec2_monitor_files.tgz ${ec2_monitor_files}"
           ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
           ssh_connection_options="$ssh_connection_options -o ConnectionAttempts=3"
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@$ip_address                     \
+            --userHost $USER@$private_ip_address             \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmd"
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@$ip_address                     \
+            --userHost $USER@$private_ip_address             \
             --operation "copy_from"                          \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
@@ -1535,7 +1540,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --operation "copy_to"                            \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
@@ -1573,7 +1578,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmds"
@@ -1600,7 +1605,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmds"
@@ -1622,7 +1627,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --operation "copy_to"                            \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
@@ -1649,7 +1654,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmds"
@@ -1691,7 +1696,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmds"
@@ -1762,7 +1767,7 @@ functions:
             ${activate_virtualenv}
             $python buildscripts/remote_operations.py          \
               --verbose                                        \
-              --userHost $USER@${ip_address}                   \
+              --userHost $USER@${private_ip_address}           \
               --sshConnectionOptions "$ssh_connection_options" \
               --retries ${ssh_retries|0}                       \
               --commands "$cmds"
@@ -1785,7 +1790,7 @@ functions:
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries|0}                       \
             --commands "$cmds"
@@ -1802,20 +1807,19 @@ functions:
         if [ -z "${instance_id}" ]; then
           exit 0
         fi
-        # Ensure we use the latest ip_address, as it could change if the EC2 instance
+        # Ensure we use the latest private_ip_address, as it could change if the EC2 instance
         # has been stopped and started.
         ${activate_virtualenv}
         # Specify '--mode start' to ensure the remote instance is running.
-        aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start)
-        echo "AMI EC2 instance ${instance_id} status: $aws_ec2"
-        ip_address=$(echo $aws_ec2 | sed -e "s/[^[:print:]\t]//g; s/.*private_ip_address: //; s/ .*//")
-        sed -i -e "s/${ip_address}/$ip_address/" ${aws_ec2_yml}
-        echo "Instance status: $aws_ec2"
+        aws_ec2_status_yml=aws_ec2_status.yml
+        aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $aws_ec2_status_yml)
+        private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $aws_ec2_status_yml --yamlKey private_ip_address)
+        echo "private_ip_address: $private_ip_address" > private_ip_address.yml
 
   "update EC2 address": &update_ec2_address
     command: expansions.update
     params:
-      file: src/${aws_ec2_yml}
+      file: src/private_ip_address.yml
 
   ### Process & archive remote EC2 artifacts ###
   "tar EC2 artifacts": &tar_ec2_artifacts
@@ -1833,7 +1837,7 @@ functions:
         ${activate_virtualenv}
         $python buildscripts/remote_operations.py          \
           --verbose                                        \
-          --userHost $USER@${ip_address}                   \
+          --userHost $USER@${private_ip_address}           \
           --sshConnectionOptions "$ssh_connection_options" \
           --retries ${ssh_retries|0}                       \
           --commands "$cmd"
@@ -1852,7 +1856,7 @@ functions:
         ${activate_virtualenv}
         $python buildscripts/remote_operations.py          \
           --verbose                                        \
-          --userHost $USER@${ip_address}                   \
+          --userHost $USER@${private_ip_address}           \
           --operation "copy_from"                          \
           --sshConnectionOptions "$ssh_connection_options" \
           --retries ${ssh_retries|0}                       \
@@ -1892,7 +1896,7 @@ functions:
         ${activate_virtualenv}
         $python buildscripts/remote_operations.py          \
           --verbose                                        \
-          --userHost $USER@${ip_address}                   \
+          --userHost $USER@${private_ip_address}           \
           --sshConnectionOptions "$ssh_connection_options" \
           --retries ${ssh_retries}                         \
           --commands "$cmds"                               \
@@ -1911,7 +1915,7 @@ functions:
         ${activate_virtualenv}
         $python buildscripts/remote_operations.py          \
           --verbose                                        \
-          --userHost $USER@${ip_address}                   \
+          --userHost $USER@${private_ip_address}           \
           --operation "copy_from"                          \
           --sshConnectionOptions "$ssh_connection_options" \
           --retries ${ssh_retries}                         \
@@ -2384,10 +2388,7 @@ functions:
         PATH="/opt/mongodbtoolchain/gdb/bin:$PATH" $sudo $python buildscripts/hang_analyzer.py $hang_analyzer_option
 
         # Call hang_analyzer.py script for tasks that are running remote mongo processes
-        # The file ${aws_ec2_yml} will define the remote's ${ip_address}.
-        # Any task which uses remote processes will have previously loaded this file to set these
-        # expansion macros.
-        if [ -f ${aws_ec2_yml|""} ]; then
+        if [ -n "${private_ip_address}" ]; then
           core_ext=core
           if [ "Windows_NT" = "$OS" ]; then
             core_ext=mdmp
@@ -2405,7 +2406,7 @@ functions:
           if [ ! -z "$file_param" ]; then
             $python buildscripts/remote_operations.py          \
               --verbose                                        \
-              --userHost $USER@${ip_address}                   \
+              --userHost $USER@${private_ip_address}           \
               --operation "copy_to"                            \
               --sshConnectionOptions "$ssh_connection_options" \
               --retries ${ssh_retries}                         \
@@ -2424,14 +2425,14 @@ functions:
           cmds="$cmds; PATH=\"/opt/mongodbtoolchain/gdb/bin:\$PATH\" $sudo python buildscripts/hang_analyzer.py $hang_analyzer_option"
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries}                         \
             --commands "$cmds"
 
           $python buildscripts/remote_operations.py          \
             --verbose                                        \
-            --userHost $USER@${ip_address}                   \
+            --userHost $USER@${private_ip_address}           \
             --operation "copy_from"                          \
             --sshConnectionOptions "$ssh_connection_options" \
             --retries ${ssh_retries}                         \