Fix Ansible version testing

Several of our tests which validate Ansible behavior with Zuul are
not versioned so that they test all supported versions of Ansible.
For those cases, add versioned tests and fix any descrepancies that
have been uncovered by the additional tests (fortunately all are
minor test syntax issues and do not affect real-world usage).

One of our largest versioned Ansible tests was not actually testing
multiple Ansible versions -- we just ran it 3 times on the default
version.  Correct that and add validation that the version ran was
the expected version.

Change-Id: I26213f69fe844776408fce24322749a197e07551

18 files changed, 172 insertions, 15 deletions

diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml
index 50bbbbfc5..13ddac988 100644
--- a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml
+++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback.yaml
@@ -1,4 +1,8 @@
 - hosts: localhost
-  gather_facts: smart
+  gather_facts: false
   tasks:
     - command: echo test
+
+    - name: Echo ansible version.
+      debug:
+        msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }}
diff --git a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py
index 39ff7cd49..2597370bc 100644
--- a/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py
+++ b/tests/fixtures/config/ansible-callbacks/git/common-config/playbooks/callback_plugins/test_callback.py
@@ -15,17 +15,20 @@ DOCUMENTATION = '''
 
 
 class CallbackModule(CallbackBase):
-    CALLBACK_VERSION = 1.0
+    """
+    test callback
+    """
+    CALLBACK_VERSION = 2.0
     CALLBACK_NEEDS_WHITELIST = True
+    # aggregate means we can be loaded and not be the stdout plugin
+    CALLBACK_TYPE = 'aggregate'
+    CALLBACK_NAME = 'test_callback'
 
     def __init__(self):
         super(CallbackModule, self).__init__()
 
-    def set_options(self, task_keys=None, var_options=None, direct=None):
-        super(CallbackModule, self).set_options(task_keys=task_keys,
-                                                var_options=var_options,
-                                                direct=direct)
-
+    def set_options(self, *args, **kw):
+        super(CallbackModule, self).set_options(*args, **kw)
         self.file_name = self.get_option('file_name')
 
     def v2_on_any(self, *args, **kwargs):
diff --git a/tests/fixtures/config/ansible-callbacks/main.yaml b/tests/fixtures/config/ansible-callbacks/main.yaml
index 9d01f542f..1e5247e4a 100644
--- a/tests/fixtures/config/ansible-callbacks/main.yaml
+++ b/tests/fixtures/config/ansible-callbacks/main.yaml
@@ -1,5 +1,6 @@
 - tenant:
     name: tenant-one
+    default-ansible-version: SETME
     source:
       gerrit:
         config-projects:
diff --git a/tests/fixtures/config/ansible-callbacks/main28.yaml b/tests/fixtures/config/ansible-callbacks/main28.yaml
new file mode 100644
index 000000000..371710b4f
--- /dev/null
+++ b/tests/fixtures/config/ansible-callbacks/main28.yaml
@@ -0,0 +1,7 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.8'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
diff --git a/tests/fixtures/config/ansible-callbacks/main29.yaml b/tests/fixtures/config/ansible-callbacks/main29.yaml
new file mode 100644
index 000000000..b127139a9
--- /dev/null
+++ b/tests/fixtures/config/ansible-callbacks/main29.yaml
@@ -0,0 +1,7 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.9'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
diff --git a/tests/fixtures/config/ansible-callbacks/main5.yaml b/tests/fixtures/config/ansible-callbacks/main5.yaml
new file mode 100644
index 000000000..5efc12339
--- /dev/null
+++ b/tests/fixtures/config/ansible-callbacks/main5.yaml
@@ -0,0 +1,7 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '5'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
diff --git a/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml b/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml
index 17ddc1661..d0458c710 100644
--- a/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml
+++ b/tests/fixtures/config/ansible/git/org_ansible/playbooks/hello-ansible.yaml
@@ -3,3 +3,7 @@
     - name: hello
       debug:
         msg: hello ansible
+
+    - name: Echo ansible version.
+      debug:
+        msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }}
diff --git a/tests/fixtures/config/ansible/main.yaml b/tests/fixtures/config/ansible/main.yaml
index 94e7aa78c..473bb5ef8 100644
--- a/tests/fixtures/config/ansible/main.yaml
+++ b/tests/fixtures/config/ansible/main.yaml
@@ -1,5 +1,6 @@
 - tenant:
     name: tenant-one
+    default-ansible-version: SETME
     source:
       gerrit:
         config-projects:
diff --git a/tests/fixtures/config/ansible/main28.yaml b/tests/fixtures/config/ansible/main28.yaml
new file mode 100644
index 000000000..f2add49c7
--- /dev/null
+++ b/tests/fixtures/config/ansible/main28.yaml
@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.8'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - bare-role
+          - org/ansible
diff --git a/tests/fixtures/config/ansible/main29.yaml b/tests/fixtures/config/ansible/main29.yaml
new file mode 100644
index 000000000..758292950
--- /dev/null
+++ b/tests/fixtures/config/ansible/main29.yaml
@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.9'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - bare-role
+          - org/ansible
diff --git a/tests/fixtures/config/ansible/main5.yaml b/tests/fixtures/config/ansible/main5.yaml
new file mode 100644
index 000000000..b2364e80b
--- /dev/null
+++ b/tests/fixtures/config/ansible/main5.yaml
@@ -0,0 +1,11 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '5'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
+          - bare-role
+          - org/ansible
diff --git a/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml b/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml
index 300dfa5f0..53819aa00 100644
--- a/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml
+++ b/tests/fixtures/config/executor-facts/git/org_project/playbooks/datetime-fact.yaml
@@ -1,5 +1,5 @@
 - hosts: localhost
-  gather_facts: smart
+  gather_facts: no
   tasks:
     - debug:
         var: date_time
@@ -9,3 +9,6 @@
         var: ansible_date_time
     - assert:
         that: ansible_date_time is not defined
+    - name: Echo ansible version
+      debug:
+        msg: Ansible version={{ ansible_version.major }}.{{ ansible_version.minor }}
diff --git a/tests/fixtures/config/executor-facts/main.yaml b/tests/fixtures/config/executor-facts/main.yaml
index 208e274b1..37c9dd4fc 100644
--- a/tests/fixtures/config/executor-facts/main.yaml
+++ b/tests/fixtures/config/executor-facts/main.yaml
@@ -1,5 +1,6 @@
 - tenant:
     name: tenant-one
+    default-ansible-version: SETME
     source:
       gerrit:
         config-projects:
diff --git a/tests/fixtures/config/executor-facts/main28.yaml b/tests/fixtures/config/executor-facts/main28.yaml
new file mode 100644
index 000000000..686899bf8
--- /dev/null
+++ b/tests/fixtures/config/executor-facts/main28.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.8'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
diff --git a/tests/fixtures/config/executor-facts/main29.yaml b/tests/fixtures/config/executor-facts/main29.yaml
new file mode 100644
index 000000000..df934ff22
--- /dev/null
+++ b/tests/fixtures/config/executor-facts/main29.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '2.9'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
diff --git a/tests/fixtures/config/executor-facts/main5.yaml b/tests/fixtures/config/executor-facts/main5.yaml
new file mode 100644
index 000000000..55d9d10c0
--- /dev/null
+++ b/tests/fixtures/config/executor-facts/main5.yaml
@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    default-ansible-version: '5'
+    source:
+      gerrit:
+        config-projects:
+          - common-config
+        untrusted-projects:
+          - org/project
diff --git a/tests/unit/test_executor.py b/tests/unit/test_executor.py
index d18cf4448..6296ebe59 100644
--- a/tests/unit/test_executor.py
+++ b/tests/unit/test_executor.py
@@ -838,8 +838,10 @@ class TestLineMapping(AnsibleZuulTestCase):
         )
 
 
-class TestExecutorFacts(AnsibleZuulTestCase):
+class ExecutorFactsMixin:
+    # These should be overridden in child classes.
     tenant_config_file = 'config/executor-facts/main.yaml'
+    ansible_major_minor = 'X.Y'
 
     def _get_file(self, build, path):
         p = os.path.join(build.jobdir.root, path)
@@ -861,12 +863,34 @@ class TestExecutorFacts(AnsibleZuulTestCase):
         date_time = \
             j[0]['plays'][0]['tasks'][0]['hosts']['localhost']['date_time']
         self.assertEqual(18, len(date_time))
+        build = self.getJobFromHistory('datetime-fact', result='SUCCESS')
+        with open(build.jobdir.job_output_file) as f:
+            output = f.read()
+            self.assertIn(f'Ansible version={self.ansible_major_minor}',
+                          output)
 
 
-class TestAnsibleCallbackConfigs(AnsibleZuulTestCase):
+class TestExecutorFacts28(AnsibleZuulTestCase, ExecutorFactsMixin):
+    tenant_config_file = 'config/executor-facts/main28.yaml'
+    ansible_major_minor = '2.8'
 
+
+class TestExecutorFacts29(AnsibleZuulTestCase, ExecutorFactsMixin):
+    tenant_config_file = 'config/executor-facts/main29.yaml'
+    ansible_major_minor = '2.9'
+
+
+class TestExecutorFacts5(AnsibleZuulTestCase, ExecutorFactsMixin):
+    tenant_config_file = 'config/executor-facts/main5.yaml'
+    ansible_major_minor = '2.12'
+
+
+class AnsibleCallbackConfigsMixin:
     config_file = 'zuul-executor-ansible-callback.conf'
+
+    # These should be overridden in child classes.
     tenant_config_file = 'config/ansible-callbacks/main.yaml'
+    ansible_major_minor = 'X.Y'
 
     def test_ansible_callback_config(self):
         self.executor_server.keep_jobdir = True
@@ -905,6 +929,32 @@ class TestAnsibleCallbackConfigs(AnsibleZuulTestCase):
             'common-config/playbooks/callback_plugins/',
             c['callback_test_callback']['file_name'])
         self.assertTrue(os.path.isfile(callback_result_file))
+        build = self.getJobFromHistory('callback-test', result='SUCCESS')
+        with open(build.jobdir.job_output_file) as f:
+            output = f.read()
+            self.assertIn(f'Ansible version={self.ansible_major_minor}',
+                          output)
+
+
+class TestAnsibleCallbackConfigs28(AnsibleZuulTestCase,
+                                   AnsibleCallbackConfigsMixin):
+    config_file = 'zuul-executor-ansible-callback.conf'
+    tenant_config_file = 'config/ansible-callbacks/main28.yaml'
+    ansible_major_minor = '2.8'
+
+
+class TestAnsibleCallbackConfigs29(AnsibleZuulTestCase,
+                                   AnsibleCallbackConfigsMixin):
+    config_file = 'zuul-executor-ansible-callback.conf'
+    tenant_config_file = 'config/ansible-callbacks/main29.yaml'
+    ansible_major_minor = '2.9'
+
+
+class TestAnsibleCallbackConfigs5(AnsibleZuulTestCase,
+                                  AnsibleCallbackConfigsMixin):
+    config_file = 'zuul-executor-ansible-callback.conf'
+    tenant_config_file = 'config/ansible-callbacks/main5.yaml'
+    ansible_major_minor = '2.12'
 
 
 class TestExecutorEnvironment(AnsibleZuulTestCase):
diff --git a/tests/unit/test_v3.py b/tests/unit/test_v3.py
index a89bb3007..4c2befd61 100644
--- a/tests/unit/test_v3.py
+++ b/tests/unit/test_v3.py
@@ -3733,9 +3733,9 @@ class TestInRepoJoin(ZuulTestCase):
 class FunctionalAnsibleMixIn(object):
     # A temporary class to hold new tests while others are disabled
 
+    # These should be overridden in child classes.
     tenant_config_file = 'config/ansible/main.yaml'
-    # This should be overriden in child classes.
-    ansible_version = '2.9'
+    ansible_major_minor = 'X.Y'
 
     def test_playbook(self):
         # This test runs a bit long and needs extra time.
@@ -3826,6 +3826,7 @@ class FunctionalAnsibleMixIn(object):
             self.assertEqual(build_bubblewrap.result, 'SUCCESS')
 
     def test_repo_ansible(self):
+        self.executor_server.keep_jobdir = True
         A = self.fake_gerrit.addFakeChange('org/ansible', 'master', 'A')
         self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
         self.waitUntilSettled()
@@ -3835,18 +3836,26 @@ class FunctionalAnsibleMixIn(object):
         self.assertHistory([
             dict(name='hello-ansible', result='SUCCESS', changes='1,1'),
         ])
+        build = self.getJobFromHistory('hello-ansible', result='SUCCESS')
+        with open(build.jobdir.job_output_file) as f:
+            output = f.read()
+            self.assertIn(f'Ansible version={self.ansible_major_minor}',
+                          output)
 
 
 class TestAnsible28(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
-    ansible_version = '2.8'
+    tenant_config_file = 'config/ansible/main28.yaml'
+    ansible_major_minor = '2.8'
 
 
 class TestAnsible29(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
-    ansible_version = '2.9'
+    tenant_config_file = 'config/ansible/main29.yaml'
+    ansible_major_minor = '2.9'
 
 
 class TestAnsible5(AnsibleZuulTestCase, FunctionalAnsibleMixIn):
-    ansible_version = '5'
+    tenant_config_file = 'config/ansible/main5.yaml'
+    ansible_major_minor = '2.12'
 
 
 class TestPrePlaybooks(AnsibleZuulTestCase):