From 7da565b3aea86ef7b1ac304a423ca20b2c0e6c12 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Mon, 18 Mar 2019 08:29:03 -0500 Subject: parse botocore.endpoint logs into a list of AWS actions (#49312) * Add an option to parse botocore.endpoint logs for the AWS actions performed during a task Add a callback to consolidate all AWS actions used by modules Added some documentation to the AWS guidelines * Enable aws_resource_actions callback only for AWS tests * Add script to help generate policies * Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests Ensure AWS tests inherit environment (also remove AWS CLI in aws_rds inventory tests and use the module) --- hacking/aws_config/build_iam_policy_framework.py | 327 +++++++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 hacking/aws_config/build_iam_policy_framework.py (limited to 'hacking') diff --git a/hacking/aws_config/build_iam_policy_framework.py b/hacking/aws_config/build_iam_policy_framework.py new file mode 100644 index 0000000000..39edae75fa --- /dev/null +++ b/hacking/aws_config/build_iam_policy_framework.py @@ -0,0 +1,327 @@ +# Requires pandas, bs4, html5lib, and lxml +# +# Call script with the output from aws_resource_actions callback, e.g. +# python build_iam_policy_framework.py ['ec2:AuthorizeSecurityGroupEgress', 'ec2:AuthorizeSecurityGroupIngress', 'sts:GetCallerIdentity'] +# +# The sample output: +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Sid": "AnsibleEditor0", +# "Effect": "Allow", +# "Action": [ +# "ec2:AuthorizeSecurityGroupEgress", +# "ec2:AuthorizeSecurityGroupIngress" +# ], +# "Resource": "arn:aws:ec2:${Region}:${Account}:security-group/${SecurityGroupId}" +# }, +# { +# "Sid": "AnsibleEditor1", +# "Effect": "Allow", +# "Action": [ +# "sts:GetCallerIdentity" +# ], +# "Resource": "*" +# } +# ] +# } +# +# Policy troubleshooting: +# - If there are more actions in the policy than you provided, AWS has documented dependencies for some of your actions and +# those have been added to the policy. +# - If there are fewer actions in the policy than you provided, some of your actions are not in the IAM table of actions for +# that service. For example, the API call s3:DeleteObjects does not actually correlate to the permission needed in a policy. +# In this case s3:DeleteObject is the permission required to allow both the s3:DeleteObjects action and the s3:DeleteObject action. +# - The policies output are only as accurate as the AWS documentation. If the policy does not permit the +# necessary actions, look for undocumented dependencies. For example, redshift:CreateCluster requires ec2:DescribeVpcs, +# ec2:DescribeSubnets, ec2:DescribeSecurityGroups, and ec2:DescribeInternetGateways, but AWS does not document this. +# + +import json +import requests +import sys + +missing_dependencies = [] +try: + import pandas as pd +except ImportError: + missing_dependencies.append('pandas') +try: + import bs4 +except ImportError: + missing_dependencies.append('bs4') +try: + import html5lib +except ImportError: + missing_dependencies.append('html5lib') +try: + import lxml +except ImportError: + missing_dependencies.append('lxml') + + +irregular_service_names = { + 'a4b': 'alexaforbusiness', + 'appstream': 'appstream2.0', + 'acm': 'certificatemanager', + 'acm-pca': 'certificatemanagerprivatecertificateauthority', + 'aws-marketplace-management': 'marketplacemanagementportal', + 'ce': 'costexplorerservice', + 'cognito-identity': 'cognitoidentity', + 'cognito-sync': 'cognitosync', + 'cognito-idp': 'cognitouserpools', + 'cur': 'costandusagereport', + 'dax': 'dynamodbacceleratordax', + 'dlm': 'datalifecyclemanager', + 'dms': 'databasemigrationservice', + 'ds': 'directoryservice', + 'ec2messages': 'messagedeliveryservice', + 'ecr': 'ec2containerregistry', + 'ecs': 'elasticcontainerservice', + 'eks': 'elasticcontainerserviceforkubernetes', + 'efs': 'elasticfilesystem', + 'es': 'elasticsearchservice', + 'events': 'cloudwatchevents', + 'firehose': 'kinesisfirehose', + 'fms': 'firewallmanager', + 'health': 'healthapisandnotifications', + 'importexport': 'importexportdiskservice', + 'iot1click': 'iot1-click', + 'kafka': 'managedstreamingforkafka', + 'kinesisvideo': 'kinesisvideostreams', + 'kms': 'keymanagementservice', + 'license-manager': 'licensemanager', + 'logs': 'cloudwatchlogs', + 'opsworks-cm': 'opsworksconfigurationmanagement', + 'mediaconnect': 'elementalmediaconnect', + 'mediaconvert': 'elementalmediaconvert', + 'medialive': 'elementalmedialive', + 'mediapackage': 'elementalmediapackage', + 'mediastore': 'elementalmediastore', + 'mgh': 'migrationhub', + 'mobiletargeting': 'pinpoint', + 'pi': 'performanceinsights', + 'pricing': 'pricelist', + 'ram': 'resourceaccessmanager', + 'resource-groups': 'resourcegroups', + 'sdb': 'simpledb', + 'servicediscovery': 'cloudmap', + 'serverlessrepo': 'serverlessapplicationrepository', + 'sms': 'servermigrationservice', + 'sms-voice': 'pinpointsmsandvoiceservice', + 'sso-directory': 'ssodirectory', + 'ssm': 'systemsmanager', + 'ssmmessages': 'sessionmanagermessagegatewayservice', + 'states': 'stepfunctions', + 'sts': 'securitytokenservice', + 'swf': 'simpleworkflowservice', + 'tag': 'resourcegrouptaggingapi', + 'transfer': 'transferforsftp', + 'waf-regional': 'wafregional', + 'wam': 'workspacesapplicationmanager', + 'xray': 'x-ray' +} + +irregular_service_links = { + 'apigateway': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_manageamazonapigateway.html' + ], + 'aws-marketplace': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplace.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplacemeteringservice.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsprivatemarketplace.html' + ], + 'discovery': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_applicationdiscovery.html' + ], + 'elasticloadbalancing': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancing.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancingv2.html' + ], + 'globalaccelerator': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_globalaccelerator.html' + ] +} + + +def get_docs_by_prefix(prefix): + amazon_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazon{0}.html' + aws_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_aws{0}.html' + + if prefix in irregular_service_links: + links = irregular_service_links[prefix] + else: + if prefix in irregular_service_names: + prefix = irregular_service_names[prefix] + links = [amazon_link_form.format(prefix), aws_link_form.format(prefix)] + + return links + + +def get_html(links): + html_list = [] + for link in links: + html = requests.get(link).content + try: + parsed_html = pd.read_html(html) + html_list.append(parsed_html) + except ValueError as e: + if 'No tables found' in str(e): + pass + else: + raise e + + return html_list + + +def get_tables(service): + links = get_docs_by_prefix(service) + html_list = get_html(links) + action_tables = [] + arn_tables = [] + for df_list in html_list: + for df in df_list: + table = json.loads(df.to_json(orient='split')) + table_data = table['data'][0] + if 'Actions' in table_data and 'Resource Types (*required)' in table_data: + action_tables.append(table['data'][1::]) + elif 'Resource Types' in table_data and 'ARN' in table_data: + arn_tables.append(table['data'][1::]) + + # Action table indices: + # 0: Action, 1: Description, 2: Access level, 3: Resource type, 4: Condition keys, 5: Dependent actions + # ARN tables indices: + # 0: Resource type, 1: ARN template, 2: Condition keys + return action_tables, arn_tables + + +def add_dependent_action(resources, dependency): + resource, action = dependency.split(':') + if resource in resources: + resources[resource].append(action) + else: + resources[resource] = [action] + return resources + + +def get_dependent_actions(resources): + for service in dict(resources): + action_tables, arn_tables = get_tables(service) + for found_action_table in action_tables: + for action_stuff in found_action_table: + if action_stuff is None: + continue + if action_stuff[0] in resources[service] and action_stuff[5]: + dependencies = action_stuff[5].split() + if isinstance(dependencies, list): + for dependency in dependencies: + resources = add_dependent_action(resources, dependency) + else: + resources = add_dependent_action(resources, dependencies) + return resources + + +def get_actions_by_service(resources): + service_action_dict = {} + dependencies = {} + for service in resources: + action_tables, arn_tables = get_tables(service) + + # Create dict of the resource type to the corresponding ARN + arn_dict = {} + for found_arn_table in arn_tables: + for arn_stuff in found_arn_table: + arn_dict["{0}*".format(arn_stuff[0])] = arn_stuff[1] + + # Create dict of the action to the corresponding ARN + action_dict = {} + for found_action_table in action_tables: + for action_stuff in found_action_table: + if action_stuff[0] is None: + continue + if arn_dict.get(action_stuff[3]): + action_dict[action_stuff[0]] = arn_dict[action_stuff[3]] + else: + action_dict[action_stuff[0]] = None + service_action_dict[service] = action_dict + return service_action_dict + + +def get_resource_arns(aws_actions, action_dict): + resource_arns = {} + for resource_action in aws_actions: + resource, action = resource_action.split(':') + if action not in action_dict: + continue + if action_dict[action] is None: + resource = "*" + else: + resource = action_dict[action].replace("${Partition}", "aws") + if resource not in resource_arns: + resource_arns[resource] = [] + resource_arns[resource].append(resource_action) + return resource_arns + + +def get_resources(actions): + resources = {} + for action in actions: + resource, action = action.split(':') + if resource not in resources: + resources[resource] = [] + resources[resource].append(action) + return resources + + +def combine_arn_actions(resources, service_action_arn_dict): + arn_actions = {} + for service in service_action_arn_dict: + service_arn_actions = get_resource_arns(aws_actions, service_action_arn_dict[service]) + for resource in service_arn_actions: + if resource in arn_actions: + arn_actions[resource].extend(service_arn_actions[resource]) + else: + arn_actions[resource] = service_arn_actions[resource] + return arn_actions + + +def combine_actions_and_dependent_actions(resources): + aws_actions = [] + for resource in resources: + for action in resources[resource]: + aws_actions.append('{0}:{1}'.format(resource, action)) + return set(aws_actions) + + +def get_actions_restricted_by_arn(aws_actions): + resources = get_resources(aws_actions) + resources = get_dependent_actions(resources) + service_action_arn_dict = get_actions_by_service(resources) + aws_actions = combine_actions_and_dependent_actions(resources) + return combine_arn_actions(aws_actions, service_action_arn_dict) + + +def main(aws_actions): + arn_actions = get_actions_restricted_by_arn(aws_actions) + statement = [] + for resource_restriction in arn_actions: + statement.append({ + "Sid": "AnsibleEditor{0}".format(len(statement)), + "Effect": "Allow", + "Action": arn_actions[resource_restriction], + "Resource": resource_restriction + }) + + policy = {"Version": "2012-10-17", "Statement": statement} + print(json.dumps(policy, indent=4)) + + +if __name__ == '__main__': + if missing_dependencies: + sys.exit('Missing Python libraries: {0}'.format(', '.join(missing_dependencies))) + actions = sys.argv[1:] + if len(actions) == 1: + actions = sys.argv[1].split(',') + aws_actions = [action.strip('[], "\'') for action in actions] + main(aws_actions) -- cgit v1.2.1