From 7f9ac0f364b6faef0c57cd08761de3c2f9c7d99a Mon Sep 17 00:00:00 2001 From: Martin Krizek Date: Thu, 21 Jan 2021 11:22:33 +0100 Subject: Consolidate filters/tests handling into JinjaPluginIntercept (#71463) * Consolidate filters/tests handling into JinjaPluginIntercept ci_complete * Postpone loading all ansible plugins * Do we need to create an overlay? ci_complete * Typo ci_complete * Add FIXME * conditional.py: use public Environment.parse() method * Remove remaining occurrences of shared_loader_obj being passed to Templar * __UNROLLED__ not needed with this change anymore * Incorrect rebase at some point? --- lib/ansible/executor/task_executor.py | 6 +- lib/ansible/playbook/conditional.py | 8 +-- lib/ansible/plugins/strategy/__init__.py | 2 +- lib/ansible/template/__init__.py | 105 ++++++++++--------------------- 4 files changed, 40 insertions(+), 81 deletions(-) diff --git a/lib/ansible/executor/task_executor.py b/lib/ansible/executor/task_executor.py index 22e9526f75..3d8dcb306c 100644 --- a/lib/ansible/executor/task_executor.py +++ b/lib/ansible/executor/task_executor.py @@ -214,7 +214,7 @@ class TaskExecutor: if self._loader.get_basedir() not in self._job_vars['ansible_search_path']: self._job_vars['ansible_search_path'].append(self._loader.get_basedir()) - templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars) + templar = Templar(loader=self._loader, variables=self._job_vars) items = None loop_cache = self._job_vars.get('_ansible_loop_cache') if loop_cache is not None: @@ -277,7 +277,7 @@ class TaskExecutor: label = None loop_pause = 0 extended = False - templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=self._job_vars) + templar = Templar(loader=self._loader, variables=self._job_vars) # FIXME: move this to the object itself to allow post_validate to take care of templating (loop_control.post_validate) if self._task.loop_control: @@ -419,7 +419,7 @@ class TaskExecutor: if variables is None: variables = self._job_vars - templar = Templar(loader=self._loader, shared_loader_obj=self._shared_loader_obj, variables=variables) + templar = Templar(loader=self._loader, variables=variables) context_validation_error = None try: diff --git a/lib/ansible/playbook/conditional.py b/lib/ansible/playbook/conditional.py index fe5e353d64..a34b43e132 100644 --- a/lib/ansible/playbook/conditional.py +++ b/lib/ansible/playbook/conditional.py @@ -182,12 +182,8 @@ class Conditional: inside_yield=inside_yield ) try: - e = templar.environment.overlay() - e.filters.update(templar.environment.filters) - e.tests.update(templar.environment.tests) - - res = e._parse(conditional, None, None) - res = generate(res, e, None, None) + res = templar.environment.parse(conditional, None, None) + res = generate(res, templar.environment, None, None) parsed = ast.parse(res, mode='exec') cnv = CleansingNodeVisitor() diff --git a/lib/ansible/plugins/strategy/__init__.py b/lib/ansible/plugins/strategy/__init__.py index 8e58eb3e30..eaa7c0cb20 100644 --- a/lib/ansible/plugins/strategy/__init__.py +++ b/lib/ansible/plugins/strategy/__init__.py @@ -1340,7 +1340,7 @@ class Debugger(cmd.Cmd): def do_update_task(self, args): """Recreate the task from ``task._ds``, and template with updated ``task_vars``""" - templar = Templar(None, shared_loader_obj=None, variables=self.scope['task_vars']) + templar = Templar(None, variables=self.scope['task_vars']) task = self.scope['task'] task = task.load_data(task._ds) task.post_validate(templar) diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py index 884673cc69..1b7ea43eea 100644 --- a/lib/ansible/template/__init__.py +++ b/lib/ansible/template/__init__.py @@ -257,7 +257,6 @@ def _unroll_iterator(func): return list(ret) return ret - wrapper.__UNROLLED__ = True return _update_wrapper(wrapper, func) @@ -414,9 +413,30 @@ class JinjaPluginIntercept(MutableMapping): self._collection_jinja_func_cache = {} + self._ansible_plugins_loaded = False + + def _load_ansible_plugins(self): + if self._ansible_plugins_loaded: + return + + for plugin in self._pluginloader.all(): + method_map = getattr(plugin, self._method_map_name) + self._delegatee.update(method_map()) + + if self._pluginloader.class_name == 'FilterModule': + for plugin_name, plugin in self._delegatee.items(): + if self._jinja2_native and plugin_name in C.STRING_TYPE_FILTERS: + self._delegatee[plugin_name] = _wrap_native_text(plugin) + else: + self._delegatee[plugin_name] = _unroll_iterator(plugin) + + self._ansible_plugins_loaded = True + # FUTURE: we can cache FQ filter/test calls for the entire duration of a run, since a given collection's impl's # aren't supposed to change during a run def __getitem__(self, key): + self._load_ansible_plugins() + try: if not isinstance(key, string_types): raise ValueError('key must be a string') @@ -511,11 +531,14 @@ class JinjaPluginIntercept(MutableMapping): for func_name, func in iteritems(method_map()): fq_name = '.'.join((parent_prefix, func_name)) # FIXME: detect/warn on intra-collection function name collisions - if self._jinja2_native and fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \ - func_name in C.STRING_TYPE_FILTERS: - self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) + if self._pluginloader.class_name == 'FilterModule': + if self._jinja2_native and fq_name.startswith(('ansible.builtin.', 'ansible.legacy.')) and \ + func_name in C.STRING_TYPE_FILTERS: + self._collection_jinja_func_cache[fq_name] = _wrap_native_text(func) + else: + self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) else: - self._collection_jinja_func_cache[fq_name] = _unroll_iterator(func) + self._collection_jinja_func_cache[fq_name] = func function_impl = self._collection_jinja_func_cache[key] return function_impl @@ -586,27 +609,14 @@ class Templar: ''' def __init__(self, loader, shared_loader_obj=None, variables=None): - variables = {} if variables is None else variables - + # NOTE shared_loader_obj is deprecated, ansible.plugins.loader is used + # directly. Keeping the arg for now in case 3rd party code "uses" it. self._loader = loader self._filters = None self._tests = None - self._available_variables = variables + self._available_variables = {} if variables is None else variables self._cached_result = {} - - if loader: - self._basedir = loader.get_basedir() - else: - self._basedir = './' - - if shared_loader_obj: - self._filter_loader = getattr(shared_loader_obj, 'filter_loader') - self._test_loader = getattr(shared_loader_obj, 'test_loader') - self._lookup_loader = getattr(shared_loader_obj, 'lookup_loader') - else: - self._filter_loader = filter_loader - self._test_loader = test_loader - self._lookup_loader = lookup_loader + self._basedir = loader.get_basedir() if loader else './' # flags to determine whether certain failures during templating # should result in fatal errors being raised @@ -680,46 +690,6 @@ class Templar: return new_templar - def _get_filters(self): - ''' - Returns filter plugins, after loading and caching them if need be - ''' - - if self._filters is not None: - return self._filters.copy() - - self._filters = dict() - - for fp in self._filter_loader.all(): - self._filters.update(fp.filters()) - - if self.jinja2_native: - for string_filter in C.STRING_TYPE_FILTERS: - try: - orig_filter = self._filters[string_filter] - except KeyError: - try: - orig_filter = self.environment.filters[string_filter] - except KeyError: - continue - self._filters[string_filter] = _wrap_native_text(orig_filter) - - return self._filters.copy() - - def _get_tests(self): - ''' - Returns tests plugins, after loading and caching them if need be - ''' - - if self._tests is not None: - return self._tests.copy() - - self._tests = dict() - for fp in self._test_loader.all(): - self._tests.update(fp.tests()) - - return self._tests.copy() - def _get_extensions(self): ''' Return jinja2 extensions to load. @@ -1002,7 +972,7 @@ class Templar: return self._lookup(name, *args, **kwargs) def _lookup(self, name, *args, **kwargs): - instance = self._lookup_loader.get(name, loader=self._loader, templar=self) + instance = lookup_loader.get(name, loader=self._loader, templar=self) if instance is not None: wantlist = kwargs.pop('wantlist', False) @@ -1071,7 +1041,7 @@ class Templar: try: # allows template header overrides to change jinja2 options. if overrides is None: - myenv = self.environment.overlay() + myenv = self.environment else: myenv = self.environment.overlay(overrides) @@ -1085,13 +1055,6 @@ class Templar: key = key.strip() setattr(myenv, key, ast.literal_eval(val.strip())) - # Adds Ansible custom filters and tests - myenv.filters.update(self._get_filters()) - for k in myenv.filters: - if not getattr(myenv.filters[k], '__UNROLLED__', False): - myenv.filters[k] = _unroll_iterator(myenv.filters[k]) - myenv.tests.update(self._get_tests()) - if escape_backslashes: # Allow users to specify backslashes in playbooks as "\\" instead of as "\\\\". data = _escape_backslashes(data, myenv) -- cgit v1.2.1