Auto unroll generators produced by jinja filters (#68014)

* Auto unroll generators produced by jinja filters

* Unroll for native in finalize

* Fix indentation

Co-authored-by: Sam Doran <sdoran@redhat.com>

* Add changelog fragment

* ci_complete

* Always unroll regardless of jinja2

* ci_complete

Co-authored-by: Sam Doran <sdoran@redhat.com>

1 files changed, 55 insertions, 2 deletions

diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index bf2dbc6cad..6b5e778626 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -44,8 +44,9 @@ from jinja2.runtime import Context, StrictUndefined
 from ansible import constants as C
 from ansible.errors import AnsibleError, AnsibleFilterError, AnsiblePluginRemovedError, AnsibleUndefinedVariable, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, string_types, text_type
+from ansible.module_utils.six.moves import range
 from ansible.module_utils._text import to_native, to_text, to_bytes
-from ansible.module_utils.common._collections_compat import Sequence, Mapping, MutableMapping
+from ansible.module_utils.common._collections_compat import Iterator, Sequence, Mapping, MappingView, MutableMapping
 from ansible.module_utils.common.collections import is_sequence
 from ansible.module_utils.compat.importlib import import_module
 from ansible.plugins.loader import filter_loader, lookup_loader, test_loader
@@ -94,6 +95,9 @@ JINJA2_BEGIN_TOKENS = frozenset(('variable_begin', 'block_begin', 'comment_begin
 JINJA2_END_TOKENS = frozenset(('variable_end', 'block_end', 'comment_end', 'raw_end'))
 
 
+RANGE_TYPE = type(range(0))
+
+
 def generate_ansible_template_vars(path, dest_path=None):
     b_path = to_bytes(path)
     try:
@@ -230,6 +234,43 @@ def recursive_check_defined(item):
             raise AnsibleFilterError("{0} is undefined".format(item))
 
 
+def _is_rolled(value):
+    """Helper method to determine if something is an unrolled generator,
+    iterator, or similar object
+    """
+    return (
+        isinstance(value, Iterator) or
+        isinstance(value, MappingView) or
+        isinstance(value, RANGE_TYPE)
+    )
+
+
+def _unroll_iterator(func):
+    """Wrapper function, that intercepts the result of a filter
+    and auto unrolls a generator, so that users are not required to
+    explicitly use ``|list`` to unroll.
+    """
+    def wrapper(*args, **kwargs):
+        ret = func(*args, **kwargs)
+        if _is_rolled(ret):
+            return list(ret)
+        return ret
+
+    # This code is duplicated from ``functools.update_wrapper`` from Py3.7.
+    # ``functools.update_wrapper`` was failing when the func was ``functools.partial``
+    for attr in ('__module__', '__name__', '__qualname__', '__doc__', '__annotations__'):
+        try:
+            value = getattr(func, attr)
+        except AttributeError:
+            pass
+        else:
+            setattr(wrapper, attr, value)
+    for attr in ('__dict__',):
+        getattr(wrapper, attr).update(getattr(func, attr, {}))
+    wrapper.__wrapped__ = func
+    return wrapper
+
+
 class AnsibleUndefined(StrictUndefined):
     '''
     A custom Undefined class, which returns further Undefined objects on access,
@@ -446,7 +487,7 @@ class JinjaPluginIntercept(MutableMapping):
                 for f in iteritems(method_map()):
                     fq_name = '.'.join((parent_prefix, f[0]))
                     # FIXME: detect/warn on intra-collection function name collisions
-                    self._collection_jinja_func_cache[fq_name] = f[1]
+                    self._collection_jinja_func_cache[fq_name] = _unroll_iterator(f[1])
 
             function_impl = self._collection_jinja_func_cache[key]
             return function_impl
@@ -821,8 +862,18 @@ class Templar:
 
         If using ANSIBLE_JINJA2_NATIVE we bypass this and return the actual value always
         '''
+        if _is_rolled(thing):
+            # Auto unroll a generator, so that users are not required to
+            # explicitly use ``|list`` to unroll
+            # This only affects the scenario where the final result of templating
+            # is a generator, and not where a filter creates a generator in the middle
+            # of a template. See ``_unroll_iterator`` for the other case. This is probably
+            # unncessary
+            return list(thing)
+
         if USE_JINJA2_NATIVE:
             return thing
+
         return thing if thing is not None else ''
 
     def _fail_lookup(self, name, *args, **kwargs):
@@ -928,6 +979,8 @@ class Templar:
 
             # Adds Ansible custom filters and tests
             myenv.filters.update(self._get_filters())
+            for k in myenv.filters:
+                myenv.filters[k] = _unroll_iterator(myenv.filters[k])
             myenv.tests.update(self._get_tests())
 
             if escape_backslashes: