2.13: template module/lookup: fix convert_data for macros (#78259) (#78269)

* template module/lookup: fix convert_data for macros (#78259)

Fixes #78141

(cherry picked from commit 9afdb7fec199c16b33d356ef8c6ab2a1ef812323)

* Fix templating nested vars with convert_data=False (#78273)

Regression introduced in #78259.

(cherry picked from commit d070b03ad89d6384f26314c53098bf006f703a3f)

6 files changed, 36 insertions, 6 deletions

diff --git a/changelogs/fragments/78141-template-fix-convert_data.yml b/changelogs/fragments/78141-template-fix-convert_data.yml
new file mode 100644
index 0000000000..6623f972bf
--- /dev/null
+++ b/changelogs/fragments/78141-template-fix-convert_data.yml
@@ -0,0 +1,2 @@
+bugfixes:
+  - template module/lookup - fix ``convert_data`` option that was effectively always set to True for Jinja macros (https://github.com/ansible/ansible/issues/78141)
diff --git a/lib/ansible/plugins/lookup/template.py b/lib/ansible/plugins/lookup/template.py
index 6e98d28281..9c575b53ac 100644
--- a/lib/ansible/plugins/lookup/template.py
+++ b/lib/ansible/plugins/lookup/template.py
@@ -153,7 +153,7 @@ class LookupModule(LookupBase):
                     res = templar.template(template_data, preserve_trailing_newlines=True,
                                            convert_data=convert_data_p, escape_backslashes=False)
 
-                if C.DEFAULT_JINJA2_NATIVE and not jinja2_native:
+                if (C.DEFAULT_JINJA2_NATIVE and not jinja2_native) or not convert_data_p:
                     # jinja2_native is true globally but off for the lookup, we need this text
                     # not to be processed by literal_eval anywhere in Ansible
                     res = NativeJinjaText(res)
diff --git a/lib/ansible/template/__init__.py b/lib/ansible/template/__init__.py
index f5cecc8f73..02fda94364 100644
--- a/lib/ansible/template/__init__.py
+++ b/lib/ansible/template/__init__.py
@@ -1108,15 +1108,24 @@ class Templar:
             # save/restore cur_context to prevent overriding __UNSAFE__.
             cached_context = self.cur_context
 
+            # In case this is a recursive call and we set different concat
+            # function up the stack, reset it in case the value of convert_data
+            # changed in this call
+            self.environment.concat = self.environment.__class__.concat
+            # the concat function is set for each Ansible environment,
+            # however for convert_data=False we need to use the concat
+            # function that avoids any evaluation and set it temporarily
+            # on the environment so it is used correctly even when
+            # the concat function is called internally in Jinja,
+            # most notably for macro execution
+            if not self.jinja2_native and not convert_data:
+                self.environment.concat = ansible_concat
+
             self.cur_context = t.new_context(jvars, shared=True)
             rf = t.root_render_func(self.cur_context)
 
             try:
-                if not self.jinja2_native and not convert_data:
-                    res = ansible_concat(rf)
-                else:
-                    res = self.environment.concat(rf)
-
+                res = self.environment.concat(rf)
                 unsafe = getattr(self.cur_context, 'unsafe', False)
                 if unsafe:
                     res = wrap_var(res)
diff --git a/test/integration/targets/template/tasks/main.yml b/test/integration/targets/template/tasks/main.yml
index e8a2b9a82a..04919f1830 100644
--- a/test/integration/targets/template/tasks/main.yml
+++ b/test/integration/targets/template/tasks/main.yml
@@ -761,3 +761,19 @@
       - test
   vars:
     test: "{{ lookup('file', '{{ output_dir }}/empty_template.templated')|length == 0 }}"
+
+- assert:
+    that:
+      - data_not_converted | type_debug == 'NativeJinjaUnsafeText'
+      - data_converted | type_debug == 'dict'
+  vars:
+    data_not_converted: "{{ lookup('template', 'json_macro.j2', convert_data=False) }}"
+    data_converted: "{{ lookup('template', 'json_macro.j2') }}"
+
+- name: Test convert_data is correctly set to True for nested vars evaluation
+  debug:
+    msg: "{{ lookup('template', 'indirect_dict.j2', convert_data=False) }}"
+  vars:
+    d:
+      foo: bar
+    v: "{{ d }}"
diff --git a/test/integration/targets/template/templates/indirect_dict.j2 b/test/integration/targets/template/templates/indirect_dict.j2
new file mode 100644
index 0000000000..3124371f48
--- /dev/null
+++ b/test/integration/targets/template/templates/indirect_dict.j2
@@ -0,0 +1 @@
+{{ v.foo }}
diff --git a/test/integration/targets/template/templates/json_macro.j2 b/test/integration/targets/template/templates/json_macro.j2
new file mode 100644
index 0000000000..080f164876
--- /dev/null
+++ b/test/integration/targets/template/templates/json_macro.j2
@@ -0,0 +1,2 @@
+{% macro m() %}{{ {"foo":"bar"} }}{% endmacro %}
+{{ m() }}