diff options
author | Jonathan Abrahams <jonathan@mongodb.com> | 2018-07-02 09:16:44 -0400 |
---|---|---|
committer | Jonathan Abrahams <jonathan@mongodb.com> | 2018-07-02 11:43:01 -0400 |
commit | f0158ae28bc36522be55143866c51d0893c916de (patch) | |
tree | 36c5608ebcb4e23263230b1f3a24c4f39176c400 | |
parent | 4f09fb49ad01fb90e7065d11efd353f8fe164f49 (diff) | |
download | mongo-f0158ae28bc36522be55143866c51d0893c916de.tar.gz |
SERVER-34996 Save console_log & console_screenshot in aws_ec2.py
SERVER-34996 Fix lint
(cherry picked from commit d464916c1aedfe9d254219bcb478def6ab3c6395)
-rwxr-xr-x | buildscripts/aws_ec2.py | 114 | ||||
-rw-r--r-- | etc/evergreen.yml | 377 |
2 files changed, 293 insertions, 198 deletions
diff --git a/buildscripts/aws_ec2.py b/buildscripts/aws_ec2.py index 7fe45abf2d8..bc467d6e388 100755 --- a/buildscripts/aws_ec2.py +++ b/buildscripts/aws_ec2.py @@ -4,6 +4,7 @@ from __future__ import print_function +import base64 import collections import datetime import optparse @@ -17,6 +18,24 @@ import yaml _MODES = ("status", "create", "start", "stop", "force-stop", "reboot", "terminate") +def write_binary_file(path, string_buffer): + """Write string_buffer to path in binary format.""" + with open(path, "wb") as fh: + fh.write(string_buffer) + + +def write_utf8_file(path, string_buffer): + """Write string_buffer to path in utf-8 format.""" + with open(path, "w") as fh: + fh.write(string_buffer.encode("utf-8")) + + +def write_yaml_file(path, dictionary): + """Write dictionary to path in YML format.""" + with open(path, "w") as ystream: + yaml.safe_dump(dictionary, ystream) + + class AwsEc2(object): """Class to support controlling AWS EC2 istances.""" @@ -80,8 +99,10 @@ class AwsEc2(object): sys.stdout.flush() return 0 if reached_state else 1 - def control_instance(self, mode, image_id, wait_time_secs=0, show_progress=False): - """Controls an AMI instance. Returns 0 & status information, if successful.""" + def control_instance( #pylint: disable=too-many-arguments,too-many-branches + self, mode, image_id, wait_time_secs=0, show_progress=False, console_output_file=None, + console_screenshot_file=None): + """Control an AMI instance. Returns 0 & status information, if successful.""" if mode not in _MODES: raise ValueError( "Invalid mode '{}' specified, choose from {}.".format(mode, _MODES)) @@ -127,8 +148,30 @@ class AwsEc2(object): getattr(instance, "private_ip_address", None), getattr(instance, "public_ip_address", None), getattr(instance, "private_dns_name", None), - getattr(instance, "public_dns_name", None), - getattr(instance, "tags", None)) + getattr(instance, "public_dns_name", None), getattr(instance, "tags", None)) + + if console_output_file: + try: + console_ouput = instance.console_output() + if console_ouput and "Output" in console_ouput: + write_utf8_file(console_output_file, console_ouput["Output"]) + else: + print("Unable to generate console_ouptut file, data not available") + except botocore.exceptions.ClientError as err: + print("Unable to generate console_ouptut file: {}".format(err.message)) + + if console_screenshot_file: + client = boto3.client("ec2") + try: + console_screenshot = client.get_console_screenshot(InstanceId=image_id) + if console_screenshot and "ImageData" in console_screenshot: + write_binary_file(console_screenshot_file, + base64.decodestring(console_screenshot["ImageData"])) + else: + print("Unable to generate console_screenshot file, data not available") + except botocore.exceptions.ClientError as err: + print("Unable to generate console_screenshot file: {}".format(err.message)) + except botocore.exceptions.ClientError as err: return 1, err.message @@ -151,21 +194,14 @@ class AwsEc2(object): time.sleep(i + 1) instance.create_tags(Tags=tags) - def launch_instance(self, - ami, - instance_type, - block_devices=None, - key_name=None, - security_group_ids=None, - security_groups=None, - subnet_id=None, - tags=None, - wait_time_secs=0, - show_progress=False, - **kwargs): - """Launches and tags an AMI instance. - - Returns the tuple (0, status_information), if successful.""" + def launch_instance( # pylint: disable=too-many-arguments,too-many-locals + self, ami, instance_type, block_devices=None, key_name=None, security_group_ids=None, + security_groups=None, subnet_id=None, tags=None, wait_time_secs=0, show_progress=False, + console_output_file=None, console_screenshot_file=None, **kwargs): + """Launch and tag an AMI instance. + + Return the tuple (0, status_information), if successful. + """ bdms = [] if block_devices is None: @@ -206,7 +242,9 @@ class AwsEc2(object): self.tag_instance(instance.instance_id, tags) - return self.control_instance("status", instance.instance_id) + return self.control_instance("status", instance.instance_id, + console_output_file=console_output_file, + console_screenshot_file=console_screenshot_file) def main(): @@ -312,6 +350,15 @@ def main(): default=None, help="Save the status into the specified YAML file.") + status_options.add_option("--consoleOutputFile", dest="console_output_file", default=None, + help="Save the console output into the specified file, if" + " available.") + + status_options.add_option("--consoleScreenshotFile", dest="console_screenshot_file", + default=None, + help="Save the console screenshot (JPG format) into the specified" + " file, if available.") + parser.add_option_group(control_options) parser.add_option_group(create_options) parser.add_option_group(status_options) @@ -346,27 +393,21 @@ def main(): my_kwargs = yaml.safe_load(options.extra_args) (ret_code, instance_status) = aws_ec2.launch_instance( - ami=options.ami, - instance_type=options.instance_type, - block_devices=block_devices, - key_name=options.key_name, - security_group_ids=options.security_group_ids, - security_groups=options.security_groups, - subnet_id=options.subnet_id, - tags=tags, - wait_time_secs=options.wait_time_secs, - show_progress=True, - **my_kwargs) + ami=options.ami, instance_type=options.instance_type, block_devices=block_devices, + key_name=options.key_name, security_group_ids=options.security_group_ids, + security_groups=options.security_groups, subnet_id=options.subnet_id, tags=tags, + wait_time_secs=options.wait_time_secs, show_progress=True, + console_output_file=options.console_output_file, + console_screenshot_file=options.console_screenshot_file, **my_kwargs) else: if not getattr(options, "image_id", None): parser.print_help() parser.error("Missing required control option") (ret_code, instance_status) = aws_ec2.control_instance( - mode=options.mode, - image_id=options.image_id, - wait_time_secs=options.wait_time_secs, - show_progress=True) + mode=options.mode, image_id=options.image_id, wait_time_secs=options.wait_time_secs, + show_progress=True, console_output_file=options.console_output_file, + console_screenshot_file=options.console_screenshot_file) if ret_code: print("Return code: {}, {}".format(ret_code, instance_status)) @@ -378,8 +419,7 @@ def main(): if options.yaml_file: print("Saving status to {}".format(options.yaml_file)) - with open(options.yaml_file, "w") as ystream: - yaml.safe_dump(status_dict, ystream) + write_yaml_file(options.yaml_file, status_dict) print(yaml.safe_dump(status_dict)) diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 6b71b59d5f0..5311d8297a9 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -1238,7 +1238,8 @@ functions: ${activate_virtualenv} # Specify '--mode start' to ensure the remote instance is running. monitor_ec2_yml=monitor_ec2.yml - aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $monitor_ec2_yml) + $python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $monitor_ec2_yml + echo "AMI EC2 instance ${instance_id} status: $(cat $monitor_ec2_yml)" private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $monitor_ec2_yml --yamlKey private_ip_address) if [ -z "$private_ip_address" ]; then echo "Cannot determine the IP address for the remote monitor." @@ -1574,69 +1575,78 @@ functions: - *copy_ec2_monitor_files + ### Determine & set remote EC2 IP address ### + "get EC2 address": &get_ec2_address + command: shell.exec + params: + shell: bash + working_dir: src + script: | + if [ -z "${instance_id}" ]; then + exit 0 + fi + # Ensure we use the latest private_ip_address, as it could change if the EC2 instance + # has been stopped and started. + ${activate_virtualenv} + # Specify '--mode start' to ensure the remote instance is running. + now=$(date +'%Y%m%d%H%M%S') + aws_ec2_status_yml=aws_ec2_status.yml + $python buildscripts/aws_ec2.py \ + --imageId ${instance_id} \ + --mode start \ + --yamlFile $aws_ec2_status_yml \ + --consoleOutputFile ec2_console_$now.log \ + --consoleScreenshotFile ec2_console_screen_shot_$now.jpg + private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $aws_ec2_status_yml --yamlKey private_ip_address) + echo "private_ip_address: $private_ip_address" > private_ip_address.yml + + "update EC2 address": &update_ec2_address + command: expansions.update + params: + file: src/private_ip_address.yml - "set EC2 address": - - command: shell.exec - params: - shell: bash - working_dir: src - script: | - if [ -z "${instance_id}" ]; then - exit 0 - fi - # Ensure we use the latest private_ip_address, as it could change if the EC2 instance - # has been stopped and started. - ${activate_virtualenv} - # Specify '--mode start' to ensure the remote instance is running. - aws_ec2_status_yml=aws_ec2_status.yml - aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode start --yamlFile $aws_ec2_status_yml) - private_ip_address=$($python buildscripts/yaml_key_value.py --yamlFile $aws_ec2_status_yml --yamlKey private_ip_address) - echo "private_ip_address: $private_ip_address" > private_ip_address.yml - - - command: expansions.update - params: - file: src/private_ip_address.yml - - "copy EC2 artifacts": - - command: shell.exec - params: - shell: bash - working_dir: src - script: | - ${activate_virtualenv} - # Tar/zip artifacts on remote host. - if [ -z "${ec2_artifacts}" ]; then - exit 0 - fi - cmd="${tar|tar} czf ec2_artifacts.tgz ${ec2_artifacts}" - ssh_connection_options="${ssh_identity} ${ssh_connection_options}" - $python buildscripts/remote_operations.py \ - --verbose \ - --userHost $USER@${private_ip_address} \ - --sshConnectionOptions "$ssh_connection_options" \ - --retries ${ssh_retries|0} \ - --commands "$cmd" + ### Process & archive remote EC2 artifacts ### + "tar EC2 artifacts": &tar_ec2_artifacts + command: shell.exec + params: + shell: bash + working_dir: src + script: | + ${activate_virtualenv} + # Tar/zip artifacts on remote host. + if [ -z "${ec2_artifacts}" ]; then + exit 0 + fi + cmd="${tar|tar} czf ec2_artifacts.tgz ${ec2_artifacts}" + ssh_connection_options="${ssh_identity} ${ssh_connection_options}" + $python buildscripts/remote_operations.py \ + --verbose \ + --userHost $USER@${private_ip_address} \ + --sshConnectionOptions "$ssh_connection_options" \ + --retries ${ssh_retries|0} \ + --commands "$cmd" - - command: shell.exec - params: - shell: bash - working_dir: src - script: | - ${activate_virtualenv} - # Copy remote artifacts. - if [ -z "${ec2_artifacts}" ]; then - exit 0 - fi - ssh_connection_options="${ssh_identity} ${ssh_connection_options}" - $python buildscripts/remote_operations.py \ - --verbose \ - --userHost $USER@${private_ip_address} \ - --operation "copy_from" \ - --sshConnectionOptions "$ssh_connection_options" \ - --retries ${ssh_retries|0} \ - --file ec2_artifacts.tgz + "copy EC2 artifacts": ©_ec2_artifacts + command: shell.exec + params: + shell: bash + working_dir: src + script: | + ${activate_virtualenv} + # Copy remote artifacts. + if [ -z "${ec2_artifacts}" ]; then + exit 0 + fi + ssh_connection_options="${ssh_identity} ${ssh_connection_options}" + $python buildscripts/remote_operations.py \ + --verbose \ + --userHost $USER@${private_ip_address} \ + --operation "copy_from" \ + --sshConnectionOptions "$ssh_connection_options" \ + --retries ${ssh_retries|0} \ + --file ec2_artifacts.tgz - "cleanup EC2 instance": + "cleanup EC2 instance": &cleanup_ec2_instance command: shell.exec params: working_dir: src @@ -1649,6 +1659,147 @@ functions: aws_ec2=$($python buildscripts/aws_ec2.py --imageId ${instance_id} --mode terminate) echo "Terminated AMI EC2 instance: $aws_ec2" + "gather remote mongo coredumps": &gather_remote_mongo_coredumps + command: shell.exec + params: + working_dir: "src" + script: | + if [ ! -f ${aws_ec2_yml|""} ]; then + exit 0 + fi + ssh_connection_options="${ssh_identity} ${ssh_connection_options}" + remote_dir=${remote_dir|.} + # Find all core files and move to $remote_dir + cmds="core_files=\$(/usr/bin/find -H . \( -name '*.core' -o -name '*.mdmp' \) 2> /dev/null)" + cmds="$cmds; if [ -z \"\$core_files\" ]; then exit 0; fi" + cmds="$cmds; echo Found remote core files \$core_files, moving to \$(pwd)" + cmds="$cmds; for core_file in \$core_files" + cmds="$cmds; do base_name=\$(echo \$core_file | sed 's/.*\///')" + cmds="$cmds; if [ ! -f \$base_name ]; then mv \$core_file .; fi" + cmds="$cmds; done" + ${activate_virtualenv} + $python buildscripts/remote_operations.py \ + --verbose \ + --userHost $USER@${private_ip_address} \ + --sshConnectionOptions "$ssh_connection_options" \ + --retries ${ssh_retries} \ + --commands "$cmds" \ + --commandDir $remote_dir + + "copy remote mongo coredumps": ©_remote_mongo_coredumps + command: shell.exec + params: + working_dir: "src" + script: | + if [ ! -f ${aws_ec2_yml|""} ]; then + exit 0 + fi + ssh_connection_options="${ssh_identity} ${ssh_connection_options}" + remote_dir=${remote_dir|.} + ${activate_virtualenv} + $python buildscripts/remote_operations.py \ + --verbose \ + --userHost $USER@${private_ip_address} \ + --operation "copy_from" \ + --sshConnectionOptions "$ssh_connection_options" \ + --retries ${ssh_retries} \ + --file "$remote_dir/*.core" \ + --file "$remote_dir/*.mdmp" + # Since both type of core files do not exist on the same host, this command + # will always return non-zero. As the core file retrieval is optional, we + # always exit successfully. + exit 0 + + "archive remote EC2 artifacts": &archive_remote_ec2_artifacts + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/ec2_artifacts.tgz + remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_artifacts-${task_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Remote EC2 Artifacts - Execution ${execution} + optional: true + + "archive remote EC2 monitor files": &archive_remote_ec2_monitor_files + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/ec2_monitor_files.tgz + remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_monitor-${task_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Remote EC2 Monitor - Execution ${execution} + optional: true + + "gather EC2 console artifacts": &gather_ec2_console_artifacts + command: shell.exec + params: + working_dir: src + script: | + ec2_console_files=$(ls ec2_console* 2> /dev/null) + if [ -n "$ec2_console_files" ]; then + ${tar|tar} czf ec2_console_files.tgz $ec2_console_files + fi + + "archive EC2 console artifacts": &archive_ec2_console_artifacts + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/ec2_console_files.tgz + remote_file: ${project}/${build_variant}/${revision}/ec2/ec2_console-${task_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: EC2 Console files - Execution ${execution} + optional: true + + "save ec2 task artifacts": + - *get_ec2_address + - *update_ec2_address + - *tar_ec2_artifacts + - *copy_ec2_artifacts + - *gather_remote_mongo_coredumps + - *copy_remote_mongo_coredumps + - *gather_ec2_console_artifacts + - *archive_ec2_console_artifacts + - *cleanup_ec2_instance + - *archive_remote_ec2_artifacts + - *archive_remote_ec2_monitor_files + + ### Process & archive local client logs ### + "tar local client logs": &tar_local_client_logs + command: shell.exec + params: + working_dir: src + script: | + client_logs=$(ls crud*.log fsm*.log 2> /dev/null) + if [ ! -z "$client_logs" ]; then + ${tar|tar} czf client-logs.tgz $client_logs + fi + + "archive local client logs": &archive_local_client_logs + command: s3.put + params: + aws_key: ${aws_key} + aws_secret: ${aws_secret} + local_file: src/client-logs.tgz + remote_file: ${project}/${build_variant}/${revision}/client_logs/mongo-client-logs-${task_id}-${execution}.tgz + bucket: mciuploads + permissions: public-read + content_type: ${content_type|application/x-gzip} + display_name: Client logs - Execution ${execution} + optional: true + + "save local client logs": + - *tar_local_client_logs + - *archive_local_client_logs + pre: - func: "kill processes" - func: "cleanup environment" @@ -1726,59 +1877,9 @@ post: files: - src/archive.json - - func: "set EC2 address" - - func: "copy EC2 artifacts" - # Gather remote mongo coredumps. - - command: shell.exec - params: - working_dir: "src" - script: | - if [ ! -f ${aws_ec2_yml|""} ]; then - exit 0 - fi - ssh_connection_options="${ssh_identity} ${ssh_connection_options}" - remote_dir=${remote_dir|.} - # Find all core files and move to $remote_dir - cmds="core_files=\$(/usr/bin/find -H . \( -name '*.core' -o -name '*.mdmp' \) 2> /dev/null)" - cmds="$cmds; if [ -z \"\$core_files\" ]; then exit 0; fi" - cmds="$cmds; echo Found remote core files \$core_files, moving to \$(pwd)" - cmds="$cmds; for core_file in \$core_files" - cmds="$cmds; do base_name=\$(echo \$core_file | sed 's/.*\///')" - cmds="$cmds; if [ ! -f \$base_name ]; then mv \$core_file .; fi" - cmds="$cmds; done" - ${activate_virtualenv} - $python buildscripts/remote_operations.py \ - --verbose \ - --userHost $USER@${private_ip_address} \ - --sshConnectionOptions "$ssh_connection_options" \ - --retries ${ssh_retries} \ - --commands "$cmds" \ - --commandDir $remote_dir - - command: shell.exec - params: - working_dir: "src" - script: | - if [ ! -f ${aws_ec2_yml|""} ]; then - exit 0 - fi - ssh_connection_options="${ssh_identity} ${ssh_connection_options}" - remote_dir=${remote_dir|.} - ${activate_virtualenv} - $python buildscripts/remote_operations.py \ - --verbose \ - --userHost $USER@${private_ip_address} \ - --operation "copy_from" \ - --sshConnectionOptions "$ssh_connection_options" \ - --retries ${ssh_retries} \ - --file "$remote_dir/*.core" \ - --file "$remote_dir/*.mdmp" - # Since both type of core files do not exist on the same host, this command - # will always return non-zero. As the core file retrieval is optional, we - # always exit successfully. - exit 0 - - func: "cleanup EC2 instance" - + - func: "save ec2 task artifacts" - func: "kill processes" + - func: "save local client logs" # Print out any Out of Memory killed process messages. - command: shell.exec params: @@ -1818,32 +1919,6 @@ post: rm -rf /data/charybdefs fi - # Archive remote EC2 monitor files. - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/ec2_monitor_files.tgz - remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_monitor-${task_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: Remote EC2 Monitor - Execution ${execution} - optional: true - - # Archive remote EC2 artifacts. - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/ec2_artifacts.tgz - remote_file: ${project}/${build_variant}/${revision}/remote_ec2/remote_ec2_artifacts-${task_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: Remote EC2 Artifacts - Execution ${execution} - optional: true - # Attach scons config log - command: s3.put params: @@ -1857,27 +1932,6 @@ post: content_type: text/plain display_name: config.log - # Gather and archive the local client logs. - - command: shell.exec - params: - working_dir: src - script: | - client_logs=$(ls crud*.log fsm*.log 2> /dev/null) - if [ ! -z "$client_logs" ]; then - ${tar|tar} czf client-logs.tgz $client_logs - fi - - command: s3.put - params: - aws_key: ${aws_key} - aws_secret: ${aws_secret} - local_file: src/client-logs.tgz - remote_file: ${project}/${build_variant}/${revision}/client_logs/mongo-client-logs-${task_id}-${execution}.tgz - bucket: mciuploads - permissions: public-read - content_type: ${content_type|application/x-gzip} - display_name: Client logs - Execution ${execution} - optional: true - # Process and save coverage data. - command: shell.exec params: @@ -2136,7 +2190,8 @@ post: timeout: - func: "fetch debugsymbols archive" - func: "extract debugsymbols" - - func: "set EC2 address" + - func: "get EC2 address" + - func: "update EC2 address" - command: shell.exec params: working_dir: src |