Implement AND'd filters in ec2.py/ini (#30272)

* Implement AND'd filters in ec2.py/ini

remove debug print

* Adjusting code to changed filters' data structure

2 files changed, 67 insertions, 49 deletions

diff --git a/contrib/inventory/ec2.ini b/contrib/inventory/ec2.ini
index ab1ad6222d..8637d0ff59 100644
--- a/contrib/inventory/ec2.ini
+++ b/contrib/inventory/ec2.ini
@@ -159,7 +159,9 @@ group_by_elasticache_replication_group = True
 # inventory. For the full list of possible filters, please read the EC2 API
 # docs: http://docs.aws.amazon.com/AWSEC2/latest/APIReference/ApiReference-query-DescribeInstances.html#query-DescribeInstances-filters
 # Filters are key/value pairs separated by '=', to list multiple filters use
-# a list separated by commas. See examples below.
+# a list separated by commas. To "AND" criteria together, use "&". Note that
+# the "AND" is not useful along with stack_filters and so such usage is not allowed.
+# See examples below.
 
 # If you want to apply multiple filters simultaneously, set stack_filters to
 # True. Default behaviour is to combine the results of all filters. Stacking
@@ -181,6 +183,13 @@ stack_filters = False
 # (ex. webservers15, webservers1a, webservers123 etc)
 # instance_filters = tag:Name=webservers1*
 
+# Retrieve only instances of type t1.micro that also have tag env=stage
+# instance_filters = instance-type=t1.micro&tag:env=stage
+
+# Retrieve instances of type t1.micro AND tag env=stage, as well as any instance
+# that are of type m3.large, regardless of env tag
+# instance_filters = instance-type=t1.micro&tag:env=stage,instance-type=m3.large
+
 # An IAM role can be assumed, so all requests are run as that role.
 # This can be useful for connecting across different accounts, or to limit user
 # access
diff --git a/contrib/inventory/ec2.py b/contrib/inventory/ec2.py
index f8bccac2f6..b0ce35c9c7 100755
--- a/contrib/inventory/ec2.py
+++ b/contrib/inventory/ec2.py
@@ -488,19 +488,27 @@ class Ec2Inventory(object):
             self.stack_filters = False
 
         # Instance filters (see boto and EC2 API docs). Ignore invalid filters.
-        self.ec2_instance_filters = defaultdict(list)
+        self.ec2_instance_filters = []
         if config.has_option('ec2', 'instance_filters'):
-
-            filters = [f for f in config.get('ec2', 'instance_filters').split(',') if f]
-
-            for instance_filter in filters:
-                instance_filter = instance_filter.strip()
-                if not instance_filter or '=' not in instance_filter:
-                    continue
-                filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
-                if not filter_key:
-                    continue
-                self.ec2_instance_filters[filter_key].append(filter_value)
+            filters = config.get('ec2', 'instance_filters')
+
+            if self.stack_filters and '&' in filters:
+                self.fail_with_error("AND filters along with stack_filter enabled is not supported.\n")
+
+            filter_sets = [f for f in filters.split(',') if f]
+
+            for filter_set in filter_sets:
+                filters = {}
+                filter_set = filter_set.strip()
+                for instance_filter in filter_set.split("&"):
+                    instance_filter = instance_filter.strip()
+                    if not instance_filter or '=' not in instance_filter:
+                        continue
+                    filter_key, filter_value = [x.strip() for x in instance_filter.split('=', 1)]
+                    if not filter_key:
+                        continue
+                    filters[filter_key] = filter_value
+                self.ec2_instance_filters.append(filters.copy())
 
     def parse_cli_args(self):
         ''' Command line argument processing '''
@@ -582,12 +590,12 @@ class Ec2Inventory(object):
             if self.ec2_instance_filters:
                 if self.stack_filters:
                     filters_dict = {}
-                    for filter_key, filter_values in self.ec2_instance_filters.items():
-                        filters_dict[filter_key] = filter_values
+                    for filters in self.ec2_instance_filters:
+                        filters_dict.update(filters)
                     reservations.extend(conn.get_all_instances(filters=filters_dict))
                 else:
-                    for filter_key, filter_values in self.ec2_instance_filters.items():
-                        reservations.extend(conn.get_all_instances(filters={filter_key: filter_values}))
+                    for filters in self.ec2_instance_filters:
+                        reservations.extend(conn.get_all_instances(filters=filters))
             else:
                 reservations = conn.get_all_instances()
 
@@ -627,31 +635,28 @@ class Ec2Inventory(object):
         ''' return True if given tags match configured filters '''
         if not self.ec2_instance_filters:
             return True
-        match = self.stack_filters
-        for filter_name, filter_value in self.ec2_instance_filters.items():
-            if filter_name[:4] != 'tag:':
-                continue
-            filter_name = filter_name[4:]
-            if filter_name not in tags:
-                if self.stack_filters:
-                    match = False
-                    break
-                continue
-            if isinstance(filter_value, list):
-                if self.stack_filters and tags[filter_name] not in filter_value:
-                    match = False
-                    break
-                if not self.stack_filters and tags[filter_name] in filter_value:
-                    match = True
-                    break
-            if isinstance(filter_value, six.string_types):
-                if self.stack_filters and tags[filter_name] != filter_value:
-                    match = False
-                    break
-                if not self.stack_filters and tags[filter_name] == filter_value:
-                    match = True
-                    break
-        return match
+
+        for filters in self.ec2_instance_filters:
+            for filter_name, filter_value in filters.items():
+                if filter_name[:4] != 'tag:':
+                    continue
+                filter_name = filter_name[4:]
+                if filter_name not in tags:
+                    if self.stack_filters:
+                        return False
+                    continue
+                if isinstance(filter_value, list):
+                    if self.stack_filters and tags[filter_name] not in filter_value:
+                        return False
+                    if not self.stack_filters and tags[filter_name] in filter_value:
+                        return True
+                if isinstance(filter_value, six.string_types):
+                    if self.stack_filters and tags[filter_name] != filter_value:
+                        return False
+                    if not self.stack_filters and tags[filter_name] == filter_value:
+                        return True
+
+        return self.stack_filters
 
     def get_rds_instances_by_region(self, region):
         ''' Makes an AWS API call to the list of RDS instances in a particular
@@ -718,7 +723,7 @@ class Ec2Inventory(object):
             if 'LatestRestorableTime' in c:
                 del c['LatestRestorableTime']
 
-            if self.ec2_instance_filters == {}:
+            if not self.ec2_instance_filters:
                 matches_filter = True
             else:
                 matches_filter = False
@@ -730,14 +735,18 @@ class Ec2Inventory(object):
                 c['Tags'] = tags['TagList']
 
                 if self.ec2_instance_filters:
-                    for filter_key, filter_values in self.ec2_instance_filters.items():
-                        # get AWS tag key e.g. tag:env will be 'env'
-                        tag_name = filter_key.split(":", 1)[1]
-                        # Filter values is a list (if you put multiple values for the same tag name)
-                        matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])
+                    for filters in self.ec2_instance_filters:
+                        for filter_key, filter_values in filters.items():
+                            # get AWS tag key e.g. tag:env will be 'env'
+                            tag_name = filter_key.split(":", 1)[1]
+                            # Filter values is a list (if you put multiple values for the same tag name)
+                            matches_filter = any(d['Key'] == tag_name and d['Value'] in filter_values for d in c['Tags'])
+
+                            if matches_filter:
+                                # it matches a filter, so stop looking for further matches
+                                break
 
                         if matches_filter:
-                            # it matches a filter, so stop looking for further matches
                             break
 
             except Exception as e: