Move API (functional) tests to separate jobs

It becomes increasingly harder to maintain functional tests using the fake
hardware type and integration tests in the same jobs. This change creates
two new jobs (for Python 2 and Python 3) and makes API tests run only in them.

Change-Id: I3570323b22d8d4f19635f26733b3ae92bd0c661d

10 files changed, 243 insertions, 8 deletions

diff --git a/playbooks/legacy/tempest-dsvm-ironic-functional-python2/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-functional-python2/run.yaml
new file mode 100644
index 000000000..086c01d25
--- /dev/null
+++ b/playbooks/legacy/tempest-dsvm-ironic-functional-python2/run.yaml
@@ -0,0 +1,113 @@
+- hosts: all
+  name: Legacy functional (API) tests job on Python 2
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=384"
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_RAMDISK_TYPE=tinyipa"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.api"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-vars-early
+            # use tempest plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-tempest-plugin'"
+            export TEMPEST_CONCURRENCY=1
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/ironic-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/pyghmi $PROJECTS"
+          export PROJECTS="openstack/virtualbmc $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_IRONIC=1
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_VIRT_DRIVER=ironic
+          export DEVSTACK_GATE_CONFIGDRIVE=1
+          export DEVSTACK_GATE_IRONIC_DRIVER=ipmi
+          # Only keystone is required for the functional tests
+          export OVERRIDE_ENABLED_SERVICES="key,mysql,rabbit,ir-api,ir-cond,tempest"
+          # We do not need testing nodes pre-configured
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BAREMETAL_BASIC_OPS=False"
+          # There is no neutron, so using noop networks.
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_NETWORK_INTERFACE=noop"
+
+          export BRANCH_OVERRIDE="{{ zuul.override_checkout | default('default') }}"
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          if [[ "$ZUUL_BRANCH" != "stable/ocata" && "$BRANCH_OVERRIDE" != "stable/ocata" ]]; then
+              export DEVSTACK_GATE_TLSPROXY=1
+          fi
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_TEMPEST_WHOLE_DISK_IMAGE=True"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=0"
+
+          export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=0
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_BUILD_RAMDISK=False"
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_COUNT=1"
+
+          # Ensure the ironic-vars-EARLY file exists
+          touch ironic-vars-early
+          # Pull in the EARLY variables injected by the optional builders
+          source ironic-vars-early
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic git://git.openstack.org/openstack/ironic"
+
+          # Ensure the ironic-EXTRA-vars file exists
+          touch ironic-extra-vars
+          # Pull in the EXTRA variables injected by the optional builders
+          source ironic-extra-vars
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-functional-python3/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-functional-python3/run.yaml
new file mode 100644
index 000000000..67c1d55c7
--- /dev/null
+++ b/playbooks/legacy/tempest-dsvm-ironic-functional-python3/run.yaml
@@ -0,0 +1,122 @@
+- hosts: all
+  name: Legacy functional (API) tests job on Python 3
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=384"
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_RAMDISK_TYPE=tinyipa"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.api"
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-vars-early
+            # use tempest plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-tempest-plugin'"
+            export TEMPEST_CONCURRENCY=1
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_USE_PYTHON3=True
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/ironic-tempest-plugin $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/pyghmi $PROJECTS"
+          export PROJECTS="openstack/virtualbmc $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_IRONIC=1
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_VIRT_DRIVER=ironic
+          export DEVSTACK_GATE_CONFIGDRIVE=1
+          export DEVSTACK_GATE_IRONIC_DRIVER=ipmi
+          # Only keystone is required for the functional tests
+          export OVERRIDE_ENABLED_SERVICES="key,mysql,rabbit,ir-api,ir-cond,tempest"
+          # We do not need testing nodes pre-configured
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BAREMETAL_BASIC_OPS=False"
+          # There is no neutron, so using noop networks.
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_DEFAULT_NETWORK_INTERFACE=noop"
+
+          export BRANCH_OVERRIDE="{{ zuul.override_checkout | default('default') }}"
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          if [[ "$ZUUL_BRANCH" != "stable/ocata" && "$BRANCH_OVERRIDE" != "stable/ocata" ]]; then
+              export DEVSTACK_GATE_TLSPROXY=1
+          fi
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_TEMPEST_WHOLE_DISK_IMAGE=False"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_EPHEMERAL_DISK=1"
+
+          export DEVSTACK_GATE_IRONIC_BUILD_RAMDISK=0
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_BUILD_RAMDISK=False"
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_COUNT=1"
+
+          # Ensure the ironic-vars-EARLY file exists
+          touch ironic-vars-early
+          # Pull in the EARLY variables injected by the optional builders
+          source ironic-vars-early
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic git://git.openstack.org/openstack/ironic"
+
+          # Ensure the ironic-EXTRA-vars file exists
+          touch ironic-extra-vars
+          # Pull in the EXTRA variables injected by the optional builders
+          source ironic-extra-vars
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-pxe_ipmitool-tinyipa-python3/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-pxe_ipmitool-tinyipa-python3/run.yaml
index d4958e643..950eccd02 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-pxe_ipmitool-tinyipa-python3/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-pxe_ipmitool-tinyipa-python3/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-redfish-tinyipa/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-redfish-tinyipa/run.yaml
index 3f427024b..376b45b63 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-redfish-tinyipa/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-redfish-tinyipa/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-uefi-pxe_ipmitool-tinyipa/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-uefi-pxe_ipmitool-tinyipa/run.yaml
index 9d04adac6..83f6b610b 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-uefi-pxe_ipmitool-tinyipa/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-partition-uefi-pxe_ipmitool-tinyipa/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-agent_ipmitool-tinyipa-multinode/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-agent_ipmitool-tinyipa-multinode/run.yaml
index ec0a0c943..5b7a1f3f8 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-agent_ipmitool-tinyipa-multinode/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-agent_ipmitool-tinyipa-multinode/run.yaml
@@ -79,7 +79,7 @@
             # Run all ironic tests and the multinode smoke test from nova.
             # TODO(dtantsur): add test_attach_interfaces from nova when our
             # devstack plugin creates more than one port per node.
-            export DEVSTACK_GATE_TEMPEST_REGEX="(ironic|test_schedule_to_all_nodes)"
+            export DEVSTACK_GATE_TEMPEST_REGEX="(ironic_tempest_plugin.tests.scenario|test_schedule_to_all_nodes)"
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
       environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-agent_ipmitool-tinyipa/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-agent_ipmitool-tinyipa/run.yaml
index fba6d9929..37779c8ab 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-agent_ipmitool-tinyipa/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-agent_ipmitool-tinyipa/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-pxe_snmp-tinyipa/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-pxe_snmp-tinyipa/run.yaml
index c1fdb8e13..a92175797 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-pxe_snmp-tinyipa/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-ipa-wholedisk-bios-pxe_snmp-tinyipa/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'
diff --git a/playbooks/legacy/tempest-dsvm-ironic-parallel/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-parallel/run.yaml
index ddcfb9331..ace592b7a 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-parallel/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-parallel/run.yaml
@@ -46,7 +46,7 @@
               export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
           fi
           # Run only baremetal tests
-          export DEVSTACK_GATE_TEMPEST_REGEX='ironic'
+          export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
           ./safe-devstack-vm-gate-wrap.sh
diff --git a/playbooks/legacy/tempest-dsvm-ironic-pxe_ipmitool-postgres/run.yaml b/playbooks/legacy/tempest-dsvm-ironic-pxe_ipmitool-postgres/run.yaml
index cfa4b6521..348551c2d 100644
--- a/playbooks/legacy/tempest-dsvm-ironic-pxe_ipmitool-postgres/run.yaml
+++ b/playbooks/legacy/tempest-dsvm-ironic-pxe_ipmitool-postgres/run.yaml
@@ -37,7 +37,7 @@
     - shell:
         cmd: |
           cat << 'EOF' >> ironic-extra-vars
-            export DEVSTACK_GATE_TEMPEST_REGEX="ironic"
+            export DEVSTACK_GATE_TEMPEST_REGEX="ironic_tempest_plugin.tests.scenario"
 
           EOF
         chdir: '{{ ansible_user_dir }}/workspace'