fixed lookup search path (#16630)

* fixed lookup search path

added ansible_search_path var that contains the proper list and in order
removed roledir var which was only used by first_found, rest used role_path
added needle function for lookups that mirrors the action plugin one, now
both types of plugins use same pathing.

* added missing os import

* renamed as per feedback

* fixed missing rename in first_found

* also fixed first_found

* fixed import to match new error class

* fixed getattr ref

8 files changed, 50 insertions, 66 deletions

diff --git a/lib/ansible/plugins/lookup/__init__.py b/lib/ansible/plugins/lookup/__init__.py
index 490ea959a4..2472f66577 100644
--- a/lib/ansible/plugins/lookup/__init__.py
+++ b/lib/ansible/plugins/lookup/__init__.py
@@ -22,6 +22,7 @@ __metaclass__ = type
 from abc import ABCMeta, abstractmethod
 
 from ansible.compat.six import with_metaclass
+from ansible.errors import AnsibleFileNotFound
 
 try:
     from __main__ import display
@@ -101,3 +102,19 @@ class LookupBase(with_metaclass(ABCMeta, object)):
             result_string = to_unicode(result_string)
         """
         pass
+
+    def find_file_in_search_path(self, myvars, subdir, needle):
+        '''
+        Return a file (needle) in the task's expected search path.
+        '''
+
+        if 'ansible_search_path' in myvars:
+            paths = myvars['ansible_search_path']
+        else:
+            paths = self.get_basedir(myvars)
+
+        result = self._loader.path_dwim_relative_stack(paths, subdir, needle)
+        if result is None:
+            raise AnsibleFileNotFound("Unable to find '%s' in expected paths." % needle)
+
+        return result
diff --git a/lib/ansible/plugins/lookup/csvfile.py b/lib/ansible/plugins/lookup/csvfile.py
index 42f0d44fbe..936288e26f 100644
--- a/lib/ansible/plugins/lookup/csvfile.py
+++ b/lib/ansible/plugins/lookup/csvfile.py
@@ -72,8 +72,6 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
 
-        basedir = self.get_basedir(variables)
-
         ret = []
 
         for term in terms:
@@ -100,7 +98,7 @@ class LookupModule(LookupBase):
             if paramvals['delimiter'] == 'TAB':
                 paramvals['delimiter'] = "\t"
 
-            lookupfile = self._loader.path_dwim_relative(basedir, 'files', paramvals['file'])
+            lookupfile = self.find_file_in_search_path(variables, 'files', paramvals['file'])
             var = self.read_csv(lookupfile, key, paramvals['delimiter'], paramvals['encoding'], paramvals['default'], paramvals['col'])
             if var is not None:
                 if type(var) is list:
diff --git a/lib/ansible/plugins/lookup/file.py b/lib/ansible/plugins/lookup/file.py
index 5dd31d9c88..4df100a02e 100644
--- a/lib/ansible/plugins/lookup/file.py
+++ b/lib/ansible/plugins/lookup/file.py
@@ -33,18 +33,11 @@ class LookupModule(LookupBase):
 
         ret = []
 
-        basedir = self.get_basedir(variables)
-
         for term in terms:
             display.debug("File lookup term: %s" % term)
 
-            # Special handling of the file lookup, used primarily when the
-            # lookup is done from a role. If the file isn't found in the
-            # basedir of the current file, use dwim_relative to look in the
-            # role/files/ directory, and finally the playbook directory
-            # itself (which will be relative to the current working dir)
-
-            lookupfile = self._loader.path_dwim_relative(basedir, 'files', term)
+            # Find the file in the expected search path
+            lookupfile = self.find_file_in_search_path(variables, 'files', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
             try:
                 if lookupfile:
diff --git a/lib/ansible/plugins/lookup/fileglob.py b/lib/ansible/plugins/lookup/fileglob.py
index 7889e6e5bc..2f4ece02da 100644
--- a/lib/ansible/plugins/lookup/fileglob.py
+++ b/lib/ansible/plugins/lookup/fileglob.py
@@ -26,12 +26,10 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables=None, **kwargs):
 
-        basedir = self.get_basedir(variables)
-
         ret = []
         for term in terms:
             term_file = os.path.basename(term)
-            dwimmed_path = self._loader.path_dwim_relative(basedir, 'files', os.path.dirname(term))
+            dwimmed_path = self.find_file_in_search_path(variables, 'files', os.path.dirname(term))
             globbed = glob.glob(os.path.join(dwimmed_path, term_file))
             ret.extend(g for g in globbed if os.path.isfile(g))
         return ret
diff --git a/lib/ansible/plugins/lookup/first_found.py b/lib/ansible/plugins/lookup/first_found.py
index 8145c72b02..2762a07d43 100644
--- a/lib/ansible/plugins/lookup/first_found.py
+++ b/lib/ansible/plugins/lookup/first_found.py
@@ -118,12 +118,11 @@ __metaclass__ = type
 #     - ../files/baz
 #    ignore_errors: true
 
-
 import os
 
 from jinja2.exceptions import UndefinedError
 
-from ansible.errors import AnsibleLookupError, AnsibleUndefinedVariable
+from ansible.errors import AnsibleFileNotFound, AnsibleLookupError, AnsibleUndefinedVariable
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.boolean import boolean
 
@@ -131,7 +130,6 @@ class LookupModule(LookupBase):
 
     def run(self, terms, variables, **kwargs):
 
-        result = None
         anydict = False
         skip = False
 
@@ -173,28 +171,20 @@ class LookupModule(LookupBase):
         else:
             total_search = self._flatten(terms)
 
-        roledir = variables.get('roledir')
         for fn in total_search:
             try:
                 fn = self._templar.template(fn)
-            except (AnsibleUndefinedVariable, UndefinedError) as e:
+            except (AnsibleUndefinedVariable, UndefinedError):
                 continue
 
-            if os.path.isabs(fn) and os.path.exists(fn):
-                return [fn]
-            else:
-                if roledir is not None:
-                    # check the templates and vars directories too,if they exist
-                    for subdir in ('templates', 'vars', 'files'):
-                        path = self._loader.path_dwim_relative(roledir, subdir, fn)
-                        if os.path.exists(path):
-                            return [path]
-
-                # if none of the above were found, just check the
-                # current filename against the current dir
-                path = self._loader.path_dwim(fn)
-                if os.path.exists(path):
-                    return [path]
+             # get subdir if set by task executor, default to files otherwise
+            subdir = getattr(self, '_subdir', 'files')
+            path = None
+            try:
+                path = self.find_file_in_search_path(variables, subdir, fn)
+                return [path]
+            except AnsibleFileNotFound:
+                continue
         else:
             if skip:
                 return []
diff --git a/lib/ansible/plugins/lookup/ini.py b/lib/ansible/plugins/lookup/ini.py
index 5881930e9f..5f215aedc0 100644
--- a/lib/ansible/plugins/lookup/ini.py
+++ b/lib/ansible/plugins/lookup/ini.py
@@ -107,7 +107,7 @@ class LookupModule(LookupBase):
             except (ValueError, AssertionError) as e:
                 raise AnsibleError(e)
 
-            path = self._loader.path_dwim_relative(basedir, 'files', paramvals['file'])
+            path = self.find_file_in_search_path(variables, 'files', paramvals['file'])
             if paramvals['type'] == "properties":
                 var = self.read_properties(path, key, paramvals['default'], paramvals['re'])
             else:
diff --git a/lib/ansible/plugins/lookup/shelvefile.py b/lib/ansible/plugins/lookup/shelvefile.py
index 8883dc06b9..a994614899 100644
--- a/lib/ansible/plugins/lookup/shelvefile.py
+++ b/lib/ansible/plugins/lookup/shelvefile.py
@@ -18,7 +18,6 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import shelve
-import os
 
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
@@ -43,9 +42,6 @@ class LookupModule(LookupBase):
         ret = []
 
         for term in terms:
-            playbook_path = None
-            relative_path = None
-
             paramvals = {"file": None, "key": None}
             params = term.split()
 
@@ -59,24 +55,18 @@ class LookupModule(LookupBase):
                 # In case "file" or "key" are not present
                 raise AnsibleError(e)
 
-            file = paramvals['file']
             key = paramvals['key']
-            basedir_path  = self._loader.path_dwim(file)
 
             # Search also in the role/files directory and in the playbook directory
-            if 'role_path' in variables:
-                relative_path = self._loader.path_dwim_relative(variables['role_path'], 'files', file)
-            if 'playbook_dir' in variables:
-                playbook_path = self._loader.path_dwim_relative(variables['playbook_dir'],'files', file)
-
-            for path in (basedir_path, relative_path, playbook_path):
-                if path and os.path.exists(path):
-                    res = self.read_shelve(path, key)
-                    if res is None:
-                        raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
-                    # Convert the value read to string
-                    ret.append(str(res))
-                    break
+            shelvefile = self.find_file_in_search_path(variables, 'files', paramvals['file'])
+
+            if shelvefile:
+                res = self.read_shelve(shelvefile, key)
+                if res is None:
+                    raise AnsibleError("Key %s not found in shelve file %s" % (key, file))
+                # Convert the value read to string
+                ret.append(str(res))
+                break
             else:
                 raise AnsibleError("Could not locate shelve file in lookup: %s" % file)
 
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index 30385f9de0..b485fd0746 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -19,7 +19,6 @@ __metaclass__ = type
 
 import os
 
-from ansible import constants as C
 from ansible.errors import AnsibleError
 from ansible.plugins.lookup import LookupBase
 from ansible.utils.unicode import to_unicode
@@ -36,26 +35,25 @@ class LookupModule(LookupBase):
     def run(self, terms, variables, **kwargs):
 
         convert_data_p = kwargs.get('convert_data', True)
-        basedir = self.get_basedir(variables)
-
         ret = []
 
         for term in terms:
             display.debug("File lookup term: %s" % term)
 
-            lookupfile = self._loader.path_dwim_relative(basedir, 'templates', term)
+            lookupfile = self.find_file_in_search_path(variables, 'templates', term)
             display.vvvv("File lookup using %s as file" % lookupfile)
-            if lookupfile and os.path.exists(lookupfile):
+            if lookupfile:
                 with open(lookupfile, 'r') as f:
                     template_data = to_unicode(f.read())
 
-                    searchpath = [self._loader._basedir, os.path.dirname(lookupfile)]
-                    if 'role_path' in variables:
-                        if C.DEFAULT_ROLES_PATH:
-                            searchpath[:0] = C.DEFAULT_ROLES_PATH
-                        searchpath.insert(1, variables['role_path'])
-
+                    # set jinja2 internal search path for includes
+                    if 'ansible_search_path' in variables:
+                        searchpath = variables['ansible_search_path']
+                    else:
+                        searchpath = [self._loader._basedir, os.path.dirname(lookupfile)]
                     self._templar.environment.loader.searchpath = searchpath
+
+                    # do the templating
                     res = self._templar.template(template_data, preserve_trailing_newlines=True,convert_data=convert_data_p)
                     ret.append(res)
             else: