hashi_vault lookup: add integration tests (#44814)

* hashi_vault lookup: add integration tests

* hashi_vault lookup tests: use ansible-ci-files

(cherry picked from commit 9984c0fd4056d10c225439e79c31754c8d896ea4)

12 files changed, 338 insertions, 0 deletions

diff --git a/test/integration/targets/lookup_hashi_vault/aliases b/test/integration/targets/lookup_hashi_vault/aliases
new file mode 100644
index 0000000000..ca7c912851
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/aliases
@@ -0,0 +1,2 @@
+shippable/posix/group2
+destructive
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml
new file mode 100644
index 0000000000..4d19195a39
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/defaults/main.yml
@@ -0,0 +1,3 @@
+---
+vault_base_path: 'secret/data/testproject'
+vault_base_path_kv: 'secret/testproject'  # required by KV 2 engine
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml
new file mode 100644
index 0000000000..63307728a3
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_setup.yml
@@ -0,0 +1,21 @@
+- name: 'Create an approle policy'
+  shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write approle-policy -"
+  vars:
+    policy: |
+      path "auth/approle/login" {
+       capabilities = [ "create", "read" ]
+      }
+
+- name: 'Enable the AppRole auth method'
+  command: '{{ vault_cmd }} auth enable approle'
+
+- name: 'Create a named role'
+  command: '{{ vault_cmd }} write auth/approle/role/test-role policies="test-policy,approle-policy"'
+
+- name: 'Fetch the RoleID of the AppRole'
+  command: '{{ vault_cmd }} read -field=role_id auth/approle/role/test-role/role-id'
+  register: role_id_cmd
+
+- name: 'Get a SecretID issued against the AppRole'
+  command: '{{ vault_cmd }} write -field=secret_id -f auth/approle/role/test-role/secret-id'
+  register: secret_id_cmd
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml
new file mode 100644
index 0000000000..a97c427cff
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/approle_test.yml
@@ -0,0 +1,45 @@
+- vars:
+    role_id: '{{ role_id_cmd.stdout }}'
+    secret_id: '{{ secret_id_cmd.stdout }}'
+  block:
+    - name: 'Fetch secrets using "hashi_vault" lookup'
+      set_fact:
+        secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+        secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+
+    - name: 'Check secret values'
+      fail:
+        msg: 'unexpected secret values'
+      when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2'
+
+    - name: 'Failure expected when erroneous credentials are used'
+      vars:
+        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=approle secret_id=toto role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_wrong_cred }})'
+      register: test_wrong_cred
+      ignore_errors: true
+
+    - name: 'Failure expected when unauthorized secret is read'
+      vars:
+        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_unauthorized }})'
+      register: test_unauthorized
+      ignore_errors: true
+
+    - name: 'Failure expected when inexistent secret is read'
+      vars:
+        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 auth_method=approle secret_id=' ~ secret_id ~ ' role_id=' ~ role_id) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_inexistent }})'
+      register: test_inexistent
+      ignore_errors: true
+
+    - name: 'Check expected failures'
+      assert:
+        msg: "an expected failure didn't occur"
+        that:
+          - test_wrong_cred is failed
+          - test_unauthorized is failed
+          - test_inexistent is failed
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml
new file mode 100644
index 0000000000..ff93f83f79
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/main.yml
@@ -0,0 +1,122 @@
+---
+- name: Install Hashi Vault on controlled node and test
+
+  vars:
+    vault_version: '0.11.0'
+    vault_uri: 'https://ansible-ci-files.s3.amazonaws.com/test/integration/targets/lookup_hashi_vault/vault_{{ vault_version }}_{{ ansible_system | lower }}_{{ vault_arch }}.zip'
+    vault_cmd: '{{ local_temp_dir }}/vault'
+
+  connection: local  # already defined in inventory for testhost, usefull when used with another inventory
+
+  block:
+    - name: Create a local temporary directory
+      tempfile:
+        state: directory
+      register: tempfile_result
+
+    - set_fact:
+        local_temp_dir: '{{ tempfile_result.path }}'
+
+    - when: pyopenssl_version.stdout is version('0.15', '>=')
+      block:
+        - name: Generate privatekey
+          openssl_privatekey:
+            path: '{{ local_temp_dir }}/privatekey.pem'
+
+        - name: Generate CSR
+          openssl_csr:
+            path: '{{ local_temp_dir }}/csr.csr'
+            privatekey_path: '{{ local_temp_dir }}/privatekey.pem'
+            subject:
+              commonName: localhost
+
+        - name: Generate selfsigned certificate
+          openssl_certificate:
+            path: '{{ local_temp_dir }}/cert.pem'
+            csr_path: '{{ local_temp_dir }}/csr.csr'
+            privatekey_path: '{{ local_temp_dir }}/privatekey.pem'
+            provider: selfsigned
+            selfsigned_digest: sha256
+          register: selfsigned_certificate
+
+    - name: 'Install unzip'
+      package:
+        name: unzip
+      when: ansible_distribution != "MacOSX"  # unzip already installed
+
+    - assert:
+        # Linux: x86_64, FreeBSD: amd64
+        that: ansible_architecture in ['i386', 'x86_64', 'amd64']
+    - set_fact:
+        vault_arch: '386'
+      when: ansible_architecture == 'i386'
+    - set_fact:
+        vault_arch: amd64
+      when: ansible_architecture in ['x86_64', 'amd64']
+
+    - name: 'Download vault binary'
+      unarchive:
+        src: '{{ vault_uri }}'
+        dest: '{{ local_temp_dir }}'
+        remote_src: true
+
+    - environment:
+        # used by vault command
+        VAULT_DEV_ROOT_TOKEN_ID: '47542cbc-6bf8-4fba-8eda-02e0a0d29a0a'
+      block:
+        - name: 'Create configuration file'
+          template:
+            src: vault_config.hcl.j2
+            dest: '{{ local_temp_dir }}/vault_config.hcl'
+
+        - name: 'Start vault service'
+          environment:
+            VAULT_ADDR: 'http://localhost:8200'
+          block:
+            - name: 'Start vault server (dev mode enabled)'
+              shell: 'nohup {{ vault_cmd }} server -dev -config {{ local_temp_dir }}/vault_config.hcl </dev/null >/dev/null 2>&1 &'
+
+            - name: 'Create a test policy'
+              shell: "echo '{{ policy }}' | {{ vault_cmd }} policy write test-policy -"
+              vars:
+                policy: |
+                  path "{{ vault_base_path }}/secret1" {
+                    capabilities = ["read"]
+                  }
+                  path "{{ vault_base_path }}/secret2" {
+                    capabilities = ["read", "update"]
+                  }
+                  path "{{ vault_base_path }}/secret3" {
+                    capabilities = ["deny"]
+                  }
+
+            - name: 'Create secrets'
+              command: '{{ vault_cmd }} kv put {{ vault_base_path_kv }}/secret{{ item }} value=foo{{ item }}'
+              loop: [1, 2, 3]
+
+            - name: setup approle auth
+              import_tasks: approle_setup.yml
+              when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>')
+
+            - name: setup token auth
+              import_tasks: token_setup.yml
+
+        - import_tasks: tests.yml
+          vars:
+            auth_type: approle
+          when: ansible_distribution != 'RedHat' or ansible_distribution_major_version is version('7', '>')
+
+        - import_tasks: tests.yml
+          vars:
+            auth_type: token
+
+      always:
+        - name: 'Kill vault process'
+          shell: "kill $(cat {{ local_temp_dir }}/vault.pid)"
+          ignore_errors: true
+
+  always:
+    - name: 'Delete temp dir'
+      file:
+        path: '{{ local_temp_dir }}'
+        state: absent
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tests.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tests.yml
new file mode 100644
index 0000000000..91d0134116
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/tests.yml
@@ -0,0 +1,35 @@
+- name: 'test {{ auth_type }} auth without SSL (lookup parameters)'
+  include_tasks: '{{ auth_type }}_test.yml'
+  vars:
+    conn_params: 'url=http://localhost:8200 '
+
+- name: 'test {{ auth_type }} auth without SSL (environment variable)'
+  include_tasks: '{{ auth_type }}_test.yml'
+  args:
+    apply:
+      vars:
+        conn_params: ''
+      environment:
+        VAULT_ADDR: 'http://localhost:8200'
+
+- when: pyopenssl_version.stdout is version('0.15', '>=')
+  block:
+    - name: 'test {{ auth_type }} auth with certs (validation enabled, lookup parameters)'
+      include_tasks: '{{ auth_type }}_test.yml'
+      vars:
+        conn_params: 'url=https://localhost:8201 cacert={{ local_temp_dir }}/cert.pem validate_certs=True '
+
+    - name: 'test {{ auth_type }} auth with certs (validation enabled, environment variables)'
+      include_tasks: '{{ auth_type }}_test.yml'
+      args:
+        apply:
+          vars:
+            conn_params: ''
+          environment:
+            VAULT_ADDR: 'https://localhost:8201'
+            VAULT_CACERT: '{{ local_temp_dir }}/cert.pem'
+
+    - name: 'test {{ auth_type }} auth with certs (validation disabled, lookup parameters)'
+      include_tasks: '{{ auth_type }}_test.yml'
+      vars:
+        conn_params: 'url=https://localhost:8201 validate_certs=False '
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_setup.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_setup.yml
new file mode 100644
index 0000000000..d5ce280346
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_setup.yml
@@ -0,0 +1,3 @@
+- name: 'Create a test credentials (token)'
+  command: '{{ vault_cmd }} token create -policy test-policy -field token'
+  register: user_token_cmd
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml
new file mode 100644
index 0000000000..1f4828e5a3
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/tasks/token_test.yml
@@ -0,0 +1,44 @@
+- vars:
+    user_token: '{{ user_token_cmd.stdout }}'
+  block:
+    - name: 'Fetch secrets using "hashi_vault" lookup'
+      set_fact:
+        secret1: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret1 auth_method=token token=' ~ user_token) }}"
+        secret2: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 token=' ~ user_token) }}"
+
+    - name: 'Check secret values'
+      fail:
+        msg: 'unexpected secret values'
+      when: secret1['data']['value'] != 'foo1' or secret2['data']['value'] != 'foo2'
+
+    - name: 'Failure expected when erroneous credentials are used'
+      vars:
+        secret_wrong_cred: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret2 auth_method=token token=wrong_token') }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_wrong_cred }})'
+      register: test_wrong_cred
+      ignore_errors: true
+
+    - name: 'Failure expected when unauthorized secret is read'
+      vars:
+        secret_unauthorized: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret3 token=' ~ user_token) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_unauthorized }})'
+      register: test_unauthorized
+      ignore_errors: true
+
+    - name: 'Failure expected when inexistent secret is read'
+      vars:
+        secret_inexistent: "{{ lookup('hashi_vault', conn_params ~ 'secret=' ~ vault_base_path ~ '/secret4 token=' ~ user_token) }}"
+      debug:
+        msg: 'Failure is expected ({{ secret_inexistent }})'
+      register: test_inexistent
+      ignore_errors: true
+
+    - name: 'Check expected failures'
+      assert:
+        msg: "an expected failure didn't occur"
+        that:
+          - test_wrong_cred is failed
+          - test_unauthorized is failed
+          - test_inexistent is failed
diff --git a/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/templates/vault_config.hcl.j2 b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/templates/vault_config.hcl.j2
new file mode 100644
index 0000000000..effc90ba90
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/lookup_hashi_vault/templates/vault_config.hcl.j2
@@ -0,0 +1,10 @@
+# {{ ansible_managed }}
+pid_file = "{{ local_temp_dir }}/vault.pid"
+{% if  pyopenssl_version.stdout is version('0.15', '>=') %}
+listener "tcp" {
+  tls_key_file = "{{ local_temp_dir }}/privatekey.pem"
+  tls_cert_file = "{{ local_temp_dir }}/cert.pem"
+  tls_disable  = false
+  address = "localhost:8201"
+}
+{% endif %}
diff --git a/test/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml b/test/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml
new file mode 100644
index 0000000000..49c250462f
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/playbooks/install_dependencies.yml
@@ -0,0 +1,19 @@
+- hosts: testhost
+  tasks:
+    - name: Install openssl
+      import_role:
+        name: setup_openssl
+
+    - name: "RedHat <= 7, select last version compatible with request 2.6.0 (this version doesn't support approle auth)"
+      set_fact:
+        hvac_package: 'hvac==0.2.5'
+      when: ansible_distribution == 'RedHat' and ansible_distribution_major_version is version('7', '<=')
+
+    - name: 'CentOS < 7, select last version compatible with Python 2.6'
+      set_fact:
+        hvac_package: 'hvac==0.5.0'
+      when: ansible_distribution == 'CentOS' and ansible_distribution_major_version is version('7', '<')
+
+    - name: 'Install hvac Python package'
+      pip:
+        name: "{{ hvac_package|default('hvac') }}"
diff --git a/test/integration/targets/lookup_hashi_vault/playbooks/test_lookup_hashi_vault.yml b/test/integration/targets/lookup_hashi_vault/playbooks/test_lookup_hashi_vault.yml
new file mode 100644
index 0000000000..26b11f22e3
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/playbooks/test_lookup_hashi_vault.yml
@@ -0,0 +1,9 @@
+- hosts: testhost
+  tasks:
+    - name: register pyOpenSSL version
+      command: python -c 'import OpenSSL; print(OpenSSL.__version__)'
+      register: pyopenssl_version
+
+    - name: Test lookup hashi_vault
+      import_role:
+        name: lookup_hashi_vault/lookup_hashi_vault
diff --git a/test/integration/targets/lookup_hashi_vault/runme.sh b/test/integration/targets/lookup_hashi_vault/runme.sh
new file mode 100755
index 0000000000..e726ca1915
--- /dev/null
+++ b/test/integration/targets/lookup_hashi_vault/runme.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# First install pyOpenSSL, then test lookup in a second playbook in order to
+# workaround this error which occurs on OS X 10.11 only:
+#
+# TASK [lookup_hashi_vault : test token auth with certs (validation enabled, lookup parameters)] ***
+# included: lookup_hashi_vault/tasks/token_test.yml for testhost
+#
+#   TASK [lookup_hashi_vault : Fetch secrets using "hashi_vault" lookup] ***
+#   From cffi callback <function _verify_callback at 0x106f995f0>:
+#   Traceback (most recent call last):
+#     File "/usr/local/lib/python2.7/site-packages/OpenSSL/SSL.py", line 309, in wrapper
+#       _lib.X509_up_ref(x509)
+#   AttributeError: 'module' object has no attribute 'X509_up_ref'
+#   fatal: [testhost]: FAILED! => { "msg": "An unhandled exception occurred while running the lookup plugin 'hashi_vault'. Error was a <class 'requests.exceptions.SSLError'>, original message: HTTPSConnectionPool(host='localhost', port=8201): Max retries exceeded with url: /v1/auth/token/lookup-self (Caused by SSLError(SSLError(\"bad handshake: Error([('SSL routines', 'ssl3_get_server_certificate', 'certificate verify failed')],)\",),))"}
+
+ANSIBLE_ROLES_PATH=../ \
+ANSIBLE_CONFIG=../../integration.cfg \
+    ansible-playbook -i ../../inventory -e@../../integration_config.yml playbooks/install_dependencies.yml -v "$@"
+
+ANSIBLE_ROLES_PATH=../ \
+ANSIBLE_CONFIG=../../integration.cfg \
+    ansible-playbook -i ../../inventory -e@../../integration_config.yml playbooks/test_lookup_hashi_vault.yml -v "$@"