[Inventory] Cache the result of enumerating groups and host names

for `VariableManager._get_magic_variables()`.

This saves a lot of time re-iterating the nearly always constant global
list of groups and their members.

Generate once and cache, and invalidate cache in case `add_host:` or
`group_by:` are used.

4 files changed, 38 insertions, 11 deletions

diff --git a/lib/ansible/inventory/__init__.py b/lib/ansible/inventory/__init__.py
index f76702133d..f8a98929f4 100644
--- a/lib/ansible/inventory/__init__.py
+++ b/lib/ansible/inventory/__init__.py
@@ -25,7 +25,7 @@ import sys
 import re
 import itertools
 
-from ansible.compat.six import string_types
+from ansible.compat.six import string_types, iteritems
 
 from ansible import constants as C
 from ansible.errors import AnsibleError
@@ -63,11 +63,12 @@ class Inventory(object):
         # caching to avoid repeated calculations, particularly with
         # external inventory scripts.
 
-        self._vars_per_host  = {}
-        self._vars_per_group = {}
-        self._hosts_cache    = {}
-        self._pattern_cache  = {}
-        self._vars_plugins   = []
+        self._vars_per_host    = {}
+        self._vars_per_group   = {}
+        self._hosts_cache      = {}
+        self._pattern_cache    = {}
+        self._group_dict_cache = {}
+        self._vars_plugins     = []
 
         self._basedir = self.basedir()
 
@@ -88,6 +89,7 @@ class Inventory(object):
         # clear the cache here, which is only useful if more than
         # one Inventory objects are created when using the API directly
         self.clear_pattern_cache()
+        self.clear_group_dict_cache()
 
         self.parse_inventory(host_list)
 
@@ -220,7 +222,7 @@ class Inventory(object):
                     hosts = [ h for h in hosts if h in subset ]
 
                 # exclude hosts mentioned in any restriction (ex: failed hosts)
-                if self._restriction is not None:
+                if self._restriction:
                     hosts = [ h for h in hosts if h.name in self._restriction ]
 
             seen = set()
@@ -502,6 +504,10 @@ class Inventory(object):
         HOSTS_PATTERNS_CACHE = {}
         self._pattern_cache = {}
 
+    def clear_group_dict_cache(self):
+        ''' called exclusively by the add_host and group_by plugins '''
+        self._group_dict_cache = {}
+
     def groups_for_host(self, host):
         if host in self._hosts_cache:
             return self._hosts_cache[host].get_groups()
@@ -568,6 +574,20 @@ class Inventory(object):
 
         return vars
 
+    def get_group_dict(self):
+        """
+        In get_vars() we merge a 'magic' dictionary 'groups' with group name
+        keys and hostname list values into every host variable set.
+
+        Cache the creation of this structure here
+        """
+
+        if not self._group_dict_cache:
+            for (group_name, group) in iteritems(self.groups):
+                self._group_dict_cache[group_name] = [h.name for h in group.get_hosts()]
+
+        return self._group_dict_cache
+
     def get_vars(self, hostname, update_cached=False, vault_password=None):
 
         host = self.get_host(hostname)
@@ -829,6 +849,7 @@ class Inventory(object):
     def refresh_inventory(self):
 
         self.clear_pattern_cache()
+        self.clear_group_dict_cache()
 
         self._hosts_cache    = {}
         self._vars_per_host  = {}
diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py
index 11cab2727a..e4b9d1e971 100644
--- a/lib/ansible/plugins/strategy/__init__.py
+++ b/lib/ansible/plugins/strategy/__init__.py
@@ -475,6 +475,9 @@ class StrategyBase:
         # patterns may have referenced the group
         self._inventory.clear_pattern_cache()
 
+        # clear cache of group dict, which is used in magic host variables
+        self._inventory.clear_group_dict_cache()
+
         # also clear the hostvar cache entry for the given play, so that
         # the new hosts are available if hostvars are referenced
         self._variable_manager.invalidate_hostvars_cache(play=iterator._play)
@@ -495,6 +498,9 @@ class StrategyBase:
         group_name = result_item.get('add_group')
         new_group = self._inventory.get_group(group_name)
         if not new_group:
+            # clear cache of group dict, which is used in magic host variables
+            self._inventory.clear_group_dict_cache()
+
             # create the new group and add it to inventory
             new_group = Group(name=group_name)
             self._inventory.add_group(new_group)
diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py
index 5f714ce116..2c5a4b4f60 100644
--- a/lib/ansible/vars/__init__.py
+++ b/lib/ansible/vars/__init__.py
@@ -393,10 +393,9 @@ class VariableManager:
         if host:
             variables['group_names'] = sorted([group.name for group in host.get_groups() if group.name != 'all'])
 
-            if self._inventory is not None:
-                variables['groups']  = dict()
-                for (group_name, group) in iteritems(self._inventory.groups):
-                    variables['groups'][group_name] = [h.name for h in group.get_hosts()]
+            if self._inventory:
+                variables['groups']  = self._inventory.get_group_dict()
+
         if play:
             variables['role_names'] = [r._role_name for r in play.roles]
 
diff --git a/test/units/plugins/strategies/test_strategy_base.py b/test/units/plugins/strategies/test_strategy_base.py
index 231897d0ed..2f7a13f1cc 100644
--- a/test/units/plugins/strategies/test_strategy_base.py
+++ b/test/units/plugins/strategies/test_strategy_base.py
@@ -237,6 +237,7 @@ class TestStrategyBase(unittest.TestCase):
         mock_inventory.get_host.side_effect = _get_host
         mock_inventory.get_group.side_effect = _get_group
         mock_inventory.clear_pattern_cache.return_value = None
+        mock_inventory.clear_group_dict_cache.return_value = None
         mock_inventory.get_host_vars.return_value = {}
 
         mock_var_mgr = MagicMock()