Several var fixes

* Fixes hostvar serialization issue (#12005)
* Fixes regression in include_vars from within a role (#9498), where
  we had the precedence order for vars_cache (include_vars, set_fact)
  incorrectly before role vars.
* Fixes another bug in which vars loaded from files in the format of
  a list instead of dictionary would cause a failure.

Fixes #9498
Fixes #12005

2 files changed, 83 insertions, 25 deletions

diff --git a/lib/ansible/vars/__init__.py b/lib/ansible/vars/__init__.py
index 86992f4d7e..fbae5cf2e4 100644
--- a/lib/ansible/vars/__init__.py
+++ b/lib/ansible/vars/__init__.py
@@ -84,6 +84,26 @@ class VariableManager:
     def set_inventory(self, inventory):
         self._inventory = inventory
 
+    def _preprocess_vars(self, a):
+        '''
+        Ensures that vars contained in the parameter passed in are
+        returned as a list of dictionaries, to ensure for instance
+        that vars loaded from a file conform to an expected state.
+        '''
+
+        if a is None:
+            return None
+        elif not isinstance(a, list):
+            data = [ a ]
+        else:
+            data = a
+
+        for item in data:
+            if not isinstance(item, MutableMapping):
+                raise AnsibleError("variable files must contain either a dictionary of variables, or a list of dictionaries. Got: %s (%s)" % (a, type(a)))
+
+        return data
+
     def _validate_both_dicts(self, a, b):
         '''
         Validates that both arguments are dictionaries, or an error is raised.
@@ -127,7 +147,7 @@ class VariableManager:
 
         return result
 
-    def get_vars(self, loader, play=None, host=None, task=None, use_cache=True):
+    def get_vars(self, loader, play=None, host=None, task=None, include_hostvars=True, use_cache=True):
         '''
         Returns the variables, with optional "context" given via the parameters
         for the play, host, and task (which could possibly result in different
@@ -168,16 +188,22 @@ class VariableManager:
 
             # we merge in the special 'all' group_vars first, if they exist
             if 'all' in self._group_vars_files:
-                all_vars = self._combine_vars(all_vars, self._group_vars_files['all'])
+                data = self._preprocess_vars(self._group_vars_files['all'])
+                for item in data:
+                    all_vars = self._combine_vars(all_vars, item)
 
             for group in host.get_groups():
                 all_vars = self._combine_vars(all_vars, group.get_vars())
                 if group.name in self._group_vars_files and group.name != 'all':
-                    all_vars = self._combine_vars(all_vars, self._group_vars_files[group.name])
+                    data = self._preprocess_vars(self._group_vars_files[group.name])
+                    for item in data:
+                        all_vars = self._combine_vars(all_vars, item)
 
             host_name = host.get_name()
             if host_name in self._host_vars_files:
-                all_vars = self._combine_vars(all_vars, self._host_vars_files[host_name])
+                data = self._preprocess_vars(self._host_vars_files[host_name])
+                for item in data:
+                    all_vars = self._combine_vars(all_vars, self._host_vars_files[host_name])
 
             # then we merge in vars specified for this host
             all_vars = self._combine_vars(all_vars, host.get_vars())
@@ -209,9 +235,10 @@ class VariableManager:
                     # as soon as we read one from the list. If none are found, we
                     # raise an error, which is silently ignored at this point.
                     for vars_file in vars_file_list:
-                        data = loader.load_from_file(vars_file)
+                        data = self._preprocess_vars(loader.load_from_file(vars_file))
                         if data is not None:
-                            all_vars = self._combine_vars(all_vars, data)
+                            for item in data:
+                                all_vars = self._combine_vars(all_vars, item)
                             break
                     else:
                         raise AnsibleError("vars file %s was not found" % vars_file_item)
@@ -222,14 +249,14 @@ class VariableManager:
                 for role in play.get_roles():
                     all_vars = self._combine_vars(all_vars, role.get_vars())
 
-        if host:
-            all_vars = self._combine_vars(all_vars, self._vars_cache.get(host.get_name(), dict()))
-
         if task:
             if task._role:
                 all_vars = self._combine_vars(all_vars, task._role.get_vars())
             all_vars = self._combine_vars(all_vars, task.get_vars())
 
+        if host:
+            all_vars = self._combine_vars(all_vars, self._vars_cache.get(host.get_name(), dict()))
+
         all_vars = self._combine_vars(all_vars, self._extra_vars)
 
         # FIXME: make sure all special vars are here
@@ -241,9 +268,10 @@ class VariableManager:
             all_vars['groups'] = [group.name for group in host.get_groups()]
 
             if self._inventory is not None:
-                hostvars = HostVars(vars_manager=self, play=play, inventory=self._inventory, loader=loader)
-                all_vars['hostvars'] = hostvars
                 all_vars['groups']   = self._inventory.groups_list()
+                if include_hostvars:
+                    hostvars = HostVars(vars_manager=self, play=play, inventory=self._inventory, loader=loader)
+                    all_vars['hostvars'] = hostvars
 
         if task:
             if task._role:
diff --git a/lib/ansible/vars/hostvars.py b/lib/ansible/vars/hostvars.py
index af3d086ae8..39c6dfa26a 100644
--- a/lib/ansible/vars/hostvars.py
+++ b/lib/ansible/vars/hostvars.py
@@ -20,9 +20,12 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import collections
+import sys
 
 from jinja2 import Undefined as j2undefined
 
+from ansible import constants as C
+from ansible.inventory.host import Host
 from ansible.template import Templar
 
 __all__ = ['HostVars']
@@ -32,22 +35,47 @@ class HostVars(collections.Mapping):
     ''' A special view of vars_cache that adds values from the inventory when needed. '''
 
     def __init__(self, vars_manager, play, inventory, loader):
-        self._vars_manager = vars_manager
-        self._play         = play
-        self._inventory    = inventory
-        self._loader       = loader
-        self._lookup       = {}
+        self._lookup = {}
+        self._loader = loader
+
+        # temporarily remove the inventory filter restriction
+        # so we can compile the variables for all of the hosts
+        # in inventory
+        restriction = inventory._restriction
+        inventory.remove_restriction()
+        hosts = inventory.get_hosts()
+        inventory.restrict_to_hosts(restriction)
+
+        # check to see if localhost is in the hosts list, as we
+        # may have it referenced via hostvars but if created implicitly
+        # it doesn't sow up in the hosts list
+        has_localhost = False
+        for host in hosts:
+            if host.name in C.LOCALHOST:
+                has_localhost = True
+                break
+
+        # we don't use the method in inventory to create the implicit host,
+        # because it also adds it to the 'ungrouped' group, and we want to
+        # avoid any side-effects
+        if not has_localhost:
+            new_host =  Host(name='localhost')
+            new_host.set_variable("ansible_python_interpreter", sys.executable)
+            new_host.set_variable("ansible_connection", "local")
+            new_host.ipv4_address = '127.0.0.1'
+            hosts.append(new_host)
+
+        for host in hosts:
+            self._lookup[host.name] = vars_manager.get_vars(loader=loader, play=play, host=host, include_hostvars=False)
 
     def __getitem__(self, host_name):
 
         if host_name not in self._lookup:
-            host = self._inventory.get_host(host_name)
-            if not host:
-                return j2undefined
-            result = self._vars_manager.get_vars(loader=self._loader, play=self._play, host=host)
-            templar = Templar(variables=result, loader=self._loader)
-            self._lookup[host_name] = templar.template(result, fail_on_undefined=False)
-        return self._lookup[host_name]
+            return j2undefined
+
+        data = self._lookup.get(host_name)
+        templar = Templar(variables=data, loader=self._loader)
+        return templar.template(data, fail_on_undefined=False)
 
     def __contains__(self, host_name):
         item = self.get(host_name)
@@ -62,7 +90,9 @@ class HostVars(collections.Mapping):
         raise NotImplementedError('HostVars does not support len.  hosts entries are discovered dynamically as needed')
 
     def __getstate__(self):
-        return self._lookup
+        data = self._lookup.copy()
+        return dict(loader=self._loader, data=data)
 
     def __setstate__(self, data):
-        self._lookup = data
+        self._lookup = data.get('data')
+        self._loader = data.get('loader')