Only template values in vars_prompt rather than all vars (#39304) (#44082)

This allows the use of variables in vars_prompt fields but allows variables entered in the prompt to affect play vars rather than throwing an undefined error.

Add tests for vars_prompt

(cherry picked from commit 6d38167d49eb41c67d0b58061b8555cf7255705f)

13 files changed, 254 insertions, 14 deletions

diff --git a/changelogs/fragments/vars_prompt-allow-prompting-for-play-vars.yaml b/changelogs/fragments/vars_prompt-allow-prompting-for-play-vars.yaml
new file mode 100644
index 0000000000..d95857ae70
--- /dev/null
+++ b/changelogs/fragments/vars_prompt-allow-prompting-for-play-vars.yaml
@@ -0,0 +1,2 @@
+bugfixes:
+  - vars_prompt - properly template play level variables in vars_prompt (https://github.com/ansible/ansible/issues/37984)
diff --git a/lib/ansible/executor/playbook_executor.py b/lib/ansible/executor/playbook_executor.py
index 6c300977e2..25020c4a1c 100644
--- a/lib/ansible/executor/playbook_executor.py
+++ b/lib/ansible/executor/playbook_executor.py
@@ -102,16 +102,13 @@ class PlaybookExecutor:
                     # clear any filters which may have been applied to the inventory
                     self._inventory.remove_restriction()
 
-                    # Create a temporary copy of the play here, so we can run post_validate
-                    # on it without the templating changes affecting the original object.
-                    # Doing this before vars_prompt to allow for using variables in prompt.
+                    # Allow variables to be used in vars_prompt fields.
                     all_vars = self._variable_manager.get_vars(play=play)
                     templar = Templar(loader=self._loader, variables=all_vars)
-                    new_play = play.copy()
-                    new_play.post_validate(templar)
+                    setattr(play, 'vars_prompt', templar.template(play.vars_prompt))
 
                     if play.vars_prompt:
-                        for var in new_play.vars_prompt:
+                        for var in play.vars_prompt:
                             vname = var['name']
                             prompt = var.get("prompt", vname)
                             default = var.get("default", None)
@@ -128,17 +125,17 @@ class PlaybookExecutor:
                                 else:  # we are either in --list-<option> or syntax check
                                     play.vars[vname] = default
 
-                        # Post validating again in case variables were entered in the prompt.
-                        all_vars = self._variable_manager.get_vars(play=play)
-                        templar = Templar(loader=self._loader, variables=all_vars)
-                        new_play.post_validate(templar)
+                    # Post validate so any play level variables are templated
+                    all_vars = self._variable_manager.get_vars(play=play)
+                    templar = Templar(loader=self._loader, variables=all_vars)
+                    play.post_validate(templar)
 
                     if self._options.syntax:
                         continue
 
                     if self._tqm is None:
                         # we are just doing a listing
-                        entry['plays'].append(new_play)
+                        entry['plays'].append(play)
 
                     else:
                         self._tqm._unreachable_hosts.update(self._unreachable_hosts)
@@ -148,9 +145,9 @@ class PlaybookExecutor:
 
                         break_play = False
                         # we are actually running plays
-                        batches = self._get_serialized_batches(new_play)
+                        batches = self._get_serialized_batches(play)
                         if len(batches) == 0:
-                            self._tqm.send_callback('v2_playbook_on_play_start', new_play)
+                            self._tqm.send_callback('v2_playbook_on_play_start', play)
                             self._tqm.send_callback('v2_playbook_on_no_hosts_matched')
                         for batch in batches:
                             # restrict the inventory to the hosts in the serialized batch
diff --git a/lib/ansible/playbook/play.py b/lib/ansible/playbook/play.py
index 1f8b67b617..3aaac55405 100644
--- a/lib/ansible/playbook/play.py
+++ b/lib/ansible/playbook/play.py
@@ -64,7 +64,7 @@ class Play(Base, Taggable, Become):
 
     # Variable Attributes
     _vars_files = FieldAttribute(isa='list', default=[], priority=99)
-    _vars_prompt = FieldAttribute(isa='list', default=[], always_post_validate=True)
+    _vars_prompt = FieldAttribute(isa='list', default=[], always_post_validate=False)
     _vault_password = FieldAttribute(isa='string', always_post_validate=True)
 
     # Role Attributes
diff --git a/test/integration/targets/vars_prompt/aliases b/test/integration/targets/vars_prompt/aliases
new file mode 100644
index 0000000000..765b70da79
--- /dev/null
+++ b/test/integration/targets/vars_prompt/aliases
@@ -0,0 +1 @@
+shippable/posix/group2
diff --git a/test/integration/targets/vars_prompt/runme.sh b/test/integration/targets/vars_prompt/runme.sh
new file mode 100755
index 0000000000..15b72141a4
--- /dev/null
+++ b/test/integration/targets/vars_prompt/runme.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Install passlib on RHEL and FreeBSD
+dist=$(python -c 'import platform; print(platform.dist()[0])')
+system=$(python -c 'import platform; print(platform.system())')
+
+if [[ "$dist" == "redhat" || "$system" == "FreeBSD" ]]; then
+    pip install passlib
+fi
+
+# Interactively test vars_prompt
+pip install pexpect
+python test-vars_prompt.py -i ../../inventory "$@"
diff --git a/test/integration/targets/vars_prompt/test-vars_prompt.py b/test/integration/targets/vars_prompt/test-vars_prompt.py
new file mode 100644
index 0000000000..39f834af4c
--- /dev/null
+++ b/test/integration/targets/vars_prompt/test-vars_prompt.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+
+import os
+import pexpect
+import sys
+
+from ansible.module_utils.six import PY2
+
+if PY2:
+    log_buffer = sys.stdout
+else:
+    log_buffer = sys.stdout.buffer
+
+env_vars = {
+    'ANSIBLE_ROLES_PATH': './roles',
+    'ANSIBLE_NOCOLOR': 'True',
+    'ANSIBLE_RETRY_FILES_ENABLED': 'False',
+}
+
+
+def run_test(playbook, test_spec, args=None, timeout=10, env=None):
+
+    if not env:
+        env = os.environ.copy()
+    env.update(env_vars)
+
+    if not args:
+        args = sys.argv[1:]
+
+    vars_prompt_test = pexpect.spawn(
+        'ansible-playbook',
+        args=[playbook] + args,
+        timeout=timeout,
+        env=env,
+    )
+
+    vars_prompt_test.logfile = log_buffer
+    for item in test_spec[0]:
+        vars_prompt_test.expect(item[0])
+        if item[1]:
+            vars_prompt_test.send(item[1])
+    vars_prompt_test.expect(test_spec[1])
+    vars_prompt_test.expect(pexpect.EOF)
+    vars_prompt_test.close()
+
+
+# These are the tests to run. Each test is a playbook and a test_spec.
+#
+# The test_spec is a list with two elements.
+#
+# The first element is a list of two element tuples. The first is the regexp to look
+# for in the output, the second is the line to send.
+#
+# The last element is the last string of text to look for in the output.
+#
+tests = [
+    # Basic vars_prompt
+    {'playbook': 'vars_prompt-1.yml',
+     'test_spec': [
+        [('input:', 'some input\r')],
+         '"input": "some input"']},
+
+    # Custom prompt
+    {'playbook': 'vars_prompt-2.yml',
+     'test_spec': [
+         [('Enter some input:', 'some more input\r')],
+         '"input": "some more input"']},
+
+    # Test confirm, both correct and incorrect
+    {'playbook': 'vars_prompt-3.yml',
+     'test_spec': [
+         [('input:', 'confirm me\r'),
+          ('confirm input:', 'confirm me\r')],
+         '"input": "confirm me"']},
+
+    {'playbook': 'vars_prompt-3.yml',
+     'test_spec': [
+         [('input:', 'confirm me\r'),
+          ('confirm input:', 'incorrect\r'),
+          (r'\*\*\*\*\* VALUES ENTERED DO NOT MATCH \*\*\*\*', ''),
+          ('input:', 'confirm me\r'),
+          ('confirm input:', 'confirm me\r')],
+         '"input": "confirm me"']},
+
+    # Test private
+    {'playbook': 'vars_prompt-4.yml',
+     'test_spec': [
+         [('not_secret', 'this is displayed\r'),
+          ('this is displayed', '')],
+         '"not_secret": "this is displayed"']},
+
+    # Test hashing
+    {'playbook': 'vars_prompt-5.yml',
+     'test_spec': [
+         [('password', 'Scenic-Improving-Payphone\r'),
+          ('confirm password', 'Scenic-Improving-Payphone\r')],
+         r'"password": "\$6\$rounds=']},
+
+    # Test variables in prompt field
+    # https://github.com/ansible/ansible/issues/32723
+    {'playbook': 'vars_prompt-6.yml',
+     'test_spec': [
+         [('prompt from variable:', 'input\r')],
+         '']},
+
+    # Test play vars coming from vars_prompt
+    # https://github.com/ansible/ansible/issues/37984
+    {'playbook': 'vars_prompt-7.yml',
+     'test_spec': [
+         [('prompting for host:', 'testhost\r')],
+         r'testhost.*ok=1']},
+]
+
+for t in tests:
+    run_test(playbook=t['playbook'], test_spec=t['test_spec'])
diff --git a/test/integration/targets/vars_prompt/vars_prompt-1.yml b/test/integration/targets/vars_prompt/vars_prompt-1.yml
new file mode 100644
index 0000000000..727c60e7fa
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-1.yml
@@ -0,0 +1,15 @@
+- name: Basic vars_prompt test
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: input
+
+  tasks:
+    - assert:
+        that:
+          - input == 'some input'
+
+    - debug:
+        var: input
diff --git a/test/integration/targets/vars_prompt/vars_prompt-2.yml b/test/integration/targets/vars_prompt/vars_prompt-2.yml
new file mode 100644
index 0000000000..d8f20db801
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-2.yml
@@ -0,0 +1,16 @@
+- name: Test vars_prompt custom prompt
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: input
+      prompt: "Enter some input"
+
+  tasks:
+    - assert:
+        that:
+          - input == 'some more input'
+
+    - debug:
+        var: input
diff --git a/test/integration/targets/vars_prompt/vars_prompt-3.yml b/test/integration/targets/vars_prompt/vars_prompt-3.yml
new file mode 100644
index 0000000000..f8148182b8
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-3.yml
@@ -0,0 +1,17 @@
+- name: Test vars_prompt confirm
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: input
+      confirm: yes
+
+  tasks:
+    - name:
+      assert:
+        that:
+          - input == 'confirm me'
+
+    - debug:
+        var: input
diff --git a/test/integration/targets/vars_prompt/vars_prompt-4.yml b/test/integration/targets/vars_prompt/vars_prompt-4.yml
new file mode 100644
index 0000000000..d33cc90215
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-4.yml
@@ -0,0 +1,16 @@
+- name: Test vars_prompt not private
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: not_secret
+      private: no
+
+  tasks:
+    - assert:
+        that:
+          - not_secret == 'this is displayed'
+
+    - debug:
+        var: not_secret
diff --git a/test/integration/targets/vars_prompt/vars_prompt-5.yml b/test/integration/targets/vars_prompt/vars_prompt-5.yml
new file mode 100644
index 0000000000..62c8ad8efa
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-5.yml
@@ -0,0 +1,14 @@
+- name: Test vars_prompt hashing
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: password
+      confirm: yes
+      encrypt: sha512_crypt
+      salt: 'jESIyad4F08hP3Ta'
+
+  tasks:
+    - debug:
+        var: password
diff --git a/test/integration/targets/vars_prompt/vars_prompt-6.yml b/test/integration/targets/vars_prompt/vars_prompt-6.yml
new file mode 100644
index 0000000000..ea3fe6209c
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-6.yml
@@ -0,0 +1,20 @@
+- name: Test vars_prompt custom variables in prompt
+  hosts: testhost
+  become: no
+  gather_facts: no
+
+  vars:
+    prompt_var: prompt from variable
+
+  vars_prompt:
+    - name: input
+      prompt: "{{ prompt_var }}"
+
+  tasks:
+    - name:
+      assert:
+        that:
+          - input == 'input'
+
+    - debug:
+        var: input
diff --git a/test/integration/targets/vars_prompt/vars_prompt-7.yml b/test/integration/targets/vars_prompt/vars_prompt-7.yml
new file mode 100644
index 0000000000..a6b086d0d7
--- /dev/null
+++ b/test/integration/targets/vars_prompt/vars_prompt-7.yml
@@ -0,0 +1,12 @@
+- name: Test vars_prompt play vars
+  hosts: "{{ target_hosts }}"
+  become: no
+  gather_facts: no
+
+  vars_prompt:
+    - name: target_hosts
+      prompt: prompting for host
+      private: no
+
+  tasks:
+    - ping: