summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJonathan Abrahams <jonathan@mongodb.com>2018-07-02 09:16:44 -0400
committerJonathan Abrahams <jonathan@mongodb.com>2018-07-02 11:43:01 -0400
commitf0158ae28bc36522be55143866c51d0893c916de (patch)
tree36c5608ebcb4e23263230b1f3a24c4f39176c400
parent4f09fb49ad01fb90e7065d11efd353f8fe164f49 (diff)
downloadmongo-f0158ae28bc36522be55143866c51d0893c916de.tar.gz
SERVER-34996 Save console_log & console_screenshot in aws_ec2.py
SERVER-34996 Fix lint (cherry picked from commit d464916c1aedfe9d254219bcb478def6ab3c6395)
-rwxr-xr-xbuildscripts/aws_ec2.py114
-rw-r--r--etc/evergreen.yml377
2 files changed, 293 insertions, 198 deletions
diff --git a/buildscripts/aws_ec2.py b/buildscripts/aws_ec2.py
index 7fe45abf2d8..bc467d6e388 100755
--- a/buildscripts/aws_ec2.py
+++ b/buildscripts/aws_ec2.py
@@ -4,6 +4,7 @@
from __future__ import print_function
+import base64
import collections
import datetime
import optparse
@@ -17,6 +18,24 @@ import yaml
_MODES = ("status", "create", "start", "stop", "force-stop", "reboot", "terminate")
+def write_binary_file(path, string_buffer):
+ """Write string_buffer to path in binary format."""
+ with open(path, "wb") as fh:
+ fh.write(string_buffer)
+
+
+def write_utf8_file(path, string_buffer):
+ """Write string_buffer to path in utf-8 format."""
+ with open(path, "w") as fh:
+ fh.write(string_buffer.encode("utf-8"))
+
+
+def write_yaml_file(path, dictionary):
+ """Write dictionary to path in YML format."""
+ with open(path, "w") as ystream:
+ yaml.safe_dump(dictionary, ystream)
+
+
class AwsEc2(object):
"""Class to support controlling AWS EC2 istances."""
@@ -80,8 +99,10 @@ class AwsEc2(object):
sys.stdout.flush()
return 0 if reached_state else 1
- def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False):
- """Controls an AMI instance. Returns 0 & status information, if successful."""
+ def control_instance( #pylint: disable=too-many-arguments,too-many-branches
+ self, mode, image_id, wait_time_secs=0, show_progress=False, console_output_file=None,
+ console_screenshot_file=None):
+ """Control an AMI instance. Returns 0 & status information, if successful."""
if mode not in _MODES:
raise ValueError(
"Invalid mode '{}' specified, choose from {}.".format(mode, _MODES))
@@ -127,8 +148,30 @@ class AwsEc2(object):
getattr(instance, "private_ip_address", None),
getattr(instance, "public_ip_address", None),
getattr(instance, "private_dns_name", None),
- getattr(instance, "public_dns_name", None),
- getattr(instance, "tags", None))
+ getattr(instance, "public_dns_name", None), getattr(instance, "tags", None))
+
+ if console_output_file:
+ try:
+ console_ouput = instance.console_output()
+ if console_ouput and "Output" in console_ouput:
+ write_utf8_file(console_output_file, console_ouput["Output"])
+ else:
+ print("Unable to generate console_ouptut file, data not available")
+ except botocore.exceptions.ClientError as err:
+ print("Unable to generate console_ouptut file: {}".format(err.message))
+
+ if console_screenshot_file:
+ client = boto3.client("ec2")
+ try:
+ console_screenshot = client.get_console_screenshot(InstanceId=image_id)
+ if console_screenshot and "ImageData" in console_screenshot:
+ write_binary_file(console_screenshot_file,
+ base64.decodestring(console_screenshot["ImageData"]))
+ else:
+ print("Unable to generate console_screenshot file, data not available")
+ except botocore.exceptions.ClientError as err:
+ print("Unable to generate console_screenshot file: {}".format(err.message))
+
except botocore.exceptions.ClientError as err:
return 1, err.message
@@ -151,21 +194,14 @@ class AwsEc2(object):
time.sleep(i + 1)
instance.create_tags(Tags=tags)
- def launch_instance(self,
- ami,
- instance_type,
- block_devices=None,
- key_name=None,
- security_group_ids=None,
- security_groups=None,
- subnet_id=None,
- tags=None,
- wait_time_secs=0,
- show_progress=False,
- **kwargs):
- """Launches and tags an AMI instance.
-
- Returns the tuple (0, status_information), if successful."""
+ def launch_instance( # pylint: disable=too-many-arguments,too-many-locals
+ self, ami, instance_type, block_devices=None, key_name=None, security_group_ids=None,
+ security_groups=None, subnet_id=None, tags=None, wait_time_secs=0, show_progress=False,
+ console_output_file=None, console_screenshot_file=None, **kwargs):
+ """Launch and tag an AMI instance.
+
+ Return the tuple (0, status_information), if successful.
+ """
bdms = []
if block_devices is None:
@@ -206,7 +242,9 @@ class AwsEc2(object):
self.tag_instance(instance.instance_id, tags)
- return self.control_instance("status", instance.instance_id)
+ return self.control_instance("status", instance.instance_id,
+ console_output_file=console_output_file,
+ console_screenshot_file=console_screenshot_file)
def main():
@@ -312,6 +350,15 @@ def main():
default=None,
help="Save the status into the specified YAML file.")
+ status_options.add_option("--consoleOutputFile", dest="console_output_file", default=None,
+ help="Save the console output into the specified file, if"
+ " available.")
+
+ status_options.add_option("--consoleScreenshotFile", dest="console_screenshot_file",
+ default=None,
+ help="Save the console screenshot (JPG format) into the specified"
+ " file, if available.")
+
parser.add_option_group(control_options)
parser.add_option_group(create_options)
parser.add_option_group(status_options)
@@ -346,27 +393,21 @@ def main():
my_kwargs = yaml.safe_load(options.extra_args)
(ret_code, instance_status) = aws_ec2.launch_instance(
- ami=options.ami,
- instance_type=options.instance_type,
- block_devices=block_devices,
- key_name=options.key_name,
- security_group_ids=options.security_group_ids,
- security_groups=options.security_groups,
- subnet_id=options.subnet_id,
- tags=tags,
- wait_time_secs=options.wait_time_secs,
- show_progress=True,
- **my_kwargs)
+ ami=options.ami, instance_type=options.instance_type, block_devices=block_devices,
+ key_name=options.key_name, security_group_ids=options.security_group_ids,
+ security_groups=options.security_groups, subnet_id=options.subnet_id, tags=tags,
+ wait_time_secs=options.wait_time_secs, show_progress=True,
+ console_output_file=options.console_output_file,
+ console_screenshot_file=options.console_screenshot_file, **my_kwargs)
else:
if not getattr(options, "image_id", None):
parser.print_help()
parser.error("Missing required control option")
(ret_code, instance_status) = aws_ec2.control_instance(
- mode=options.mode,
- image_id=options.image_id,
- wait_time_secs=options.wait_time_secs,
- show_progress=True)
+ mode=options.mode, image_id=options.image_id, wait_time_secs=options.wait_time_secs,
+ show_progress=True, console_output_file=options.console_output_file,
+ console_screenshot_file=options.console_screenshot_file)
if ret_code:
print("Return code: {}, {}".format(ret_code, instance_status))
@@ -378,8 +419,7 @@ def main():
if options.yaml_file:
print("Saving status to {}".format(options.yaml_file))
- with open(options.yaml_file, "w") as ystream:
- yaml.safe_dump(status_dict, ystream)
+ write_yaml_file(options.yaml_file, status_dict)
print(yaml.safe_dump(status_dict))
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 6b71b59d5f0..5311d8297a9 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -1238,7 +1238,8 @@ functions:
${activate_virtualenv}
# Specify '--mode start' to ensure the remote instance is running.
monitor_ec2_yml=monitor_ec2.yml
- aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $monitor_ec2_yml)
+ $python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $monitor_ec2_yml
+ echo "AMI EC2 instance ${instance_id} status: $(cat $monitor_ec2_yml)"
private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $monitor_ec2_yml --yamlKey private_ip_address)
if [ -z "$private_ip_address" ]; then
echo "Cannot determine the IP address for the remote monitor."
@@ -1574,69 +1575,78 @@ functions:
- *copy_ec2_monitor_files
+ ### Determine & set remote EC2 IP address ###
+ "get EC2 address": &get_ec2_address
+ command: shell.exec
+ params:
+ shell: bash
+ working_dir: src
+ script: |
+ if [ -z "${instance_id}" ]; then
+ exit 0
+ fi
+ # Ensure we use the latest private_ip_address, as it could change if the EC2 instance
+ # has been stopped and started.
+ ${activate_virtualenv}
+ # Specify '--mode start' to ensure the remote instance is running.
+ now=$(date +'%Y%m%d%H%M%S')
+ aws_ec2_status_yml=aws_ec2_status.yml
+ $python buildscripts/aws_ec2.py \
+ --imageId ${instance_id} \
+ --mode start \
+ --yamlFile $aws_ec2_status_yml \
+ --consoleOutputFile ec2_console_$now.log \
+ --consoleScreenshotFile ec2_console_screen_shot_$now.jpg
+ private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $aws_ec2_status_yml --yamlKey private_ip_address)
+ echo "private_ip_address: $private_ip_address" > private_ip_address.yml
+
+ "update EC2 address": &update_ec2_address
+ command: expansions.update
+ params:
+ file: src/private_ip_address.yml
- "set EC2 address":
- - command: shell.exec
- params:
- shell: bash
- working_dir: src
- script: |
- if [ -z "${instance_id}" ]; then
- exit 0
- fi
- # Ensure we use the latest private_ip_address, as it could change if the EC2 instance
- # has been stopped and started.
- ${activate_virtualenv}
- # Specify '--mode start' to ensure the remote instance is running.
- aws_ec2_status_yml=aws_ec2_status.yml
- aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $aws_ec2_status_yml)
- private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $aws_ec2_status_yml --yamlKey private_ip_address)
- echo "private_ip_address: $private_ip_address" > private_ip_address.yml
-
- - command: expansions.update
- params:
- file: src/private_ip_address.yml
-
- "copy EC2 artifacts":
- - command: shell.exec
- params:
- shell: bash
- working_dir: src
- script: |
- ${activate_virtualenv}
- # Tar/zip artifacts on remote host.
- if [ -z "${ec2_artifacts}" ]; then
- exit 0
- fi
- cmd="${tar|tar} czf ec2_artifacts.tgz ${ec2_artifacts}"
- ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
- $python buildscripts/remote_operations.py \
- --verbose \
- --userHost $USER@${private_ip_address} \
- --sshConnectionOptions "$ssh_connection_options" \
- --retries ${ssh_retries|0} \
- --commands "$cmd"
+ ### Process & archive remote EC2 artifacts ###
+ "tar EC2 artifacts": &tar_ec2_artifacts
+ command: shell.exec
+ params:
+ shell: bash
+ working_dir: src
+ script: |
+ ${activate_virtualenv}
+ # Tar/zip artifacts on remote host.
+ if [ -z "${ec2_artifacts}" ]; then
+ exit 0
+ fi
+ cmd="${tar|tar} czf ec2_artifacts.tgz ${ec2_artifacts}"
+ ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
+ $python buildscripts/remote_operations.py \
+ --verbose \
+ --userHost $USER@${private_ip_address} \
+ --sshConnectionOptions "$ssh_connection_options" \
+ --retries ${ssh_retries|0} \
+ --commands "$cmd"
- - command: shell.exec
- params:
- shell: bash
- working_dir: src
- script: |
- ${activate_virtualenv}
- # Copy remote artifacts.
- if [ -z "${ec2_artifacts}" ]; then
- exit 0
- fi
- ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
- $python buildscripts/remote_operations.py \
- --verbose \
- --userHost $USER@${private_ip_address} \
- --operation "copy_from" \
- --sshConnectionOptions "$ssh_connection_options" \
- --retries ${ssh_retries|0} \
- --file ec2_artifacts.tgz
+ "copy EC2 artifacts": &copy_ec2_artifacts
+ command: shell.exec
+ params:
+ shell: bash
+ working_dir: src
+ script: |
+ ${activate_virtualenv}
+ # Copy remote artifacts.
+ if [ -z "${ec2_artifacts}" ]; then
+ exit 0
+ fi
+ ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
+ $python buildscripts/remote_operations.py \
+ --verbose \
+ --userHost $USER@${private_ip_address} \
+ --operation "copy_from" \
+ --sshConnectionOptions "$ssh_connection_options" \
+ --retries ${ssh_retries|0} \
+ --file ec2_artifacts.tgz
- "cleanup EC2 instance":
+ "cleanup EC2 instance": &cleanup_ec2_instance
command: shell.exec
params:
working_dir: src
@@ -1649,6 +1659,147 @@ functions:
aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode terminate)
echo "Terminated AMI EC2 instance: $aws_ec2"
+ "gather remote mongo coredumps": &gather_remote_mongo_coredumps
+ command: shell.exec
+ params:
+ working_dir: "src"
+ script: |
+ if [ ! -f ${aws_ec2_yml|""} ]; then
+ exit 0
+ fi
+ ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
+ remote_dir=${remote_dir|.}
+ # Find all core files and move to $remote_dir
+ cmds="core_files=\$(/usr/bin/find -H . \( -name '*.core' -o -name '*.mdmp' \) 2> /dev/null)"
+ cmds="$cmds; if [ -z \"\$core_files\" ]; then exit 0; fi"
+ cmds="$cmds; echo Found remote core files \$core_files, moving to \$(pwd)"
+ cmds="$cmds; for core_file in \$core_files"
+ cmds="$cmds; do base_name=\$(echo \$core_file | sed 's/.*\///')"
+ cmds="$cmds; if [ ! -f \$base_name ]; then mv \$core_file .; fi"
+ cmds="$cmds; done"
+ ${activate_virtualenv}
+ $python buildscripts/remote_operations.py \
+ --verbose \
+ --userHost $USER@${private_ip_address} \
+ --sshConnectionOptions "$ssh_connection_options" \
+ --retries ${ssh_retries} \
+ --commands "$cmds" \
+ --commandDir $remote_dir
+
+ "copy remote mongo coredumps": &copy_remote_mongo_coredumps
+ command: shell.exec
+ params:
+ working_dir: "src"
+ script: |
+ if [ ! -f ${aws_ec2_yml|""} ]; then
+ exit 0
+ fi
+ ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
+ remote_dir=${remote_dir|.}
+ ${activate_virtualenv}
+ $python buildscripts/remote_operations.py \
+ --verbose \
+ --userHost $USER@${private_ip_address} \
+ --operation "copy_from" \
+ --sshConnectionOptions "$ssh_connection_options" \
+ --retries ${ssh_retries} \
+ --file "$remote_dir/*.core" \
+ --file "$remote_dir/*.mdmp"
+ # Since both type of core files do not exist on the same host, this command
+ # will always return non-zero. As the core file retrieval is optional, we
+ # always exit successfully.
+ exit 0
+
+ "archive remote EC2 artifacts": &archive_remote_ec2_artifacts
+ command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/ec2_artifacts.tgz
+ remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_artifacts-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Remote EC2 Artifacts - Execution ${execution}
+ optional: true
+
+ "archive remote EC2 monitor files": &archive_remote_ec2_monitor_files
+ command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/ec2_monitor_files.tgz
+ remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_monitor-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Remote EC2 Monitor - Execution ${execution}
+ optional: true
+
+ "gather EC2 console artifacts": &gather_ec2_console_artifacts
+ command: shell.exec
+ params:
+ working_dir: src
+ script: |
+ ec2_console_files=$(ls ec2_console* 2> /dev/null)
+ if [ -n "$ec2_console_files" ]; then
+ ${tar|tar} czf ec2_console_files.tgz $ec2_console_files
+ fi
+
+ "archive EC2 console artifacts": &archive_ec2_console_artifacts
+ command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/ec2_console_files.tgz
+ remote_file: ${project}/${build_variant}/${revision}/ec2/ec2_console-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: EC2 Console files - Execution ${execution}
+ optional: true
+
+ "save ec2 task artifacts":
+ - *get_ec2_address
+ - *update_ec2_address
+ - *tar_ec2_artifacts
+ - *copy_ec2_artifacts
+ - *gather_remote_mongo_coredumps
+ - *copy_remote_mongo_coredumps
+ - *gather_ec2_console_artifacts
+ - *archive_ec2_console_artifacts
+ - *cleanup_ec2_instance
+ - *archive_remote_ec2_artifacts
+ - *archive_remote_ec2_monitor_files
+
+ ### Process & archive local client logs ###
+ "tar local client logs": &tar_local_client_logs
+ command: shell.exec
+ params:
+ working_dir: src
+ script: |
+ client_logs=$(ls crud*.log fsm*.log 2> /dev/null)
+ if [ ! -z "$client_logs" ]; then
+ ${tar|tar} czf client-logs.tgz $client_logs
+ fi
+
+ "archive local client logs": &archive_local_client_logs
+ command: s3.put
+ params:
+ aws_key: ${aws_key}
+ aws_secret: ${aws_secret}
+ local_file: src/client-logs.tgz
+ remote_file: ${project}/${build_variant}/${revision}/client_logs/mongo-client-logs-${task_id}-${execution}.tgz
+ bucket: mciuploads
+ permissions: public-read
+ content_type: ${content_type|application/x-gzip}
+ display_name: Client logs - Execution ${execution}
+ optional: true
+
+ "save local client logs":
+ - *tar_local_client_logs
+ - *archive_local_client_logs
+
pre:
- func: "kill processes"
- func: "cleanup environment"
@@ -1726,59 +1877,9 @@ post:
files:
- src/archive.json
- - func: "set EC2 address"
- - func: "copy EC2 artifacts"
- # Gather remote mongo coredumps.
- - command: shell.exec
- params:
- working_dir: "src"
- script: |
- if [ ! -f ${aws_ec2_yml|""} ]; then
- exit 0
- fi
- ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
- remote_dir=${remote_dir|.}
- # Find all core files and move to $remote_dir
- cmds="core_files=\$(/usr/bin/find -H . \( -name '*.core' -o -name '*.mdmp' \) 2> /dev/null)"
- cmds="$cmds; if [ -z \"\$core_files\" ]; then exit 0; fi"
- cmds="$cmds; echo Found remote core files \$core_files, moving to \$(pwd)"
- cmds="$cmds; for core_file in \$core_files"
- cmds="$cmds; do base_name=\$(echo \$core_file | sed 's/.*\///')"
- cmds="$cmds; if [ ! -f \$base_name ]; then mv \$core_file .; fi"
- cmds="$cmds; done"
- ${activate_virtualenv}
- $python buildscripts/remote_operations.py \
- --verbose \
- --userHost $USER@${private_ip_address} \
- --sshConnectionOptions "$ssh_connection_options" \
- --retries ${ssh_retries} \
- --commands "$cmds" \
- --commandDir $remote_dir
- - command: shell.exec
- params:
- working_dir: "src"
- script: |
- if [ ! -f ${aws_ec2_yml|""} ]; then
- exit 0
- fi
- ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
- remote_dir=${remote_dir|.}
- ${activate_virtualenv}
- $python buildscripts/remote_operations.py \
- --verbose \
- --userHost $USER@${private_ip_address} \
- --operation "copy_from" \
- --sshConnectionOptions "$ssh_connection_options" \
- --retries ${ssh_retries} \
- --file "$remote_dir/*.core" \
- --file "$remote_dir/*.mdmp"
- # Since both type of core files do not exist on the same host, this command
- # will always return non-zero. As the core file retrieval is optional, we
- # always exit successfully.
- exit 0
- - func: "cleanup EC2 instance"
-
+ - func: "save ec2 task artifacts"
- func: "kill processes"
+ - func: "save local client logs"
# Print out any Out of Memory killed process messages.
- command: shell.exec
params:
@@ -1818,32 +1919,6 @@ post:
rm -rf /data/charybdefs
fi
- # Archive remote EC2 monitor files.
- - command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/ec2_monitor_files.tgz
- remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_monitor-${task_id}-${execution}.tgz
- bucket: mciuploads
- permissions: public-read
- content_type: ${content_type|application/x-gzip}
- display_name: Remote EC2 Monitor - Execution ${execution}
- optional: true
-
- # Archive remote EC2 artifacts.
- - command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/ec2_artifacts.tgz
- remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_artifacts-${task_id}-${execution}.tgz
- bucket: mciuploads
- permissions: public-read
- content_type: ${content_type|application/x-gzip}
- display_name: Remote EC2 Artifacts - Execution ${execution}
- optional: true
-
# Attach scons config log
- command: s3.put
params:
@@ -1857,27 +1932,6 @@ post:
content_type: text/plain
display_name: config.log
- # Gather and archive the local client logs.
- - command: shell.exec
- params:
- working_dir: src
- script: |
- client_logs=$(ls crud*.log fsm*.log 2> /dev/null)
- if [ ! -z "$client_logs" ]; then
- ${tar|tar} czf client-logs.tgz $client_logs
- fi
- - command: s3.put
- params:
- aws_key: ${aws_key}
- aws_secret: ${aws_secret}
- local_file: src/client-logs.tgz
- remote_file: ${project}/${build_variant}/${revision}/client_logs/mongo-client-logs-${task_id}-${execution}.tgz
- bucket: mciuploads
- permissions: public-read
- content_type: ${content_type|application/x-gzip}
- display_name: Client logs - Execution ${execution}
- optional: true
-
# Process and save coverage data.
- command: shell.exec
params:
@@ -2136,7 +2190,8 @@ post:
timeout:
- func: "fetch debugsymbols archive"
- func: "extract debugsymbols"
- - func: "set EC2 address"
+ - func: "get EC2 address"
+ - func: "update EC2 address"
- command: shell.exec
params:
working_dir: src