Migrate functional test jobs to zuul v3

Something are introduced in this patch:
* As devstack-gate/devstack-vm-gate-wrap.sh is not really zuul v3
native, we move all configs in to `devstack/lib/heat` and .zuul.yaml.

* Remove extra configs process in devstack. Like setup tempest(which
is well covered by tempest itself.) or overlapping heat test configs setup.
Use tempest config for all heat_integration tests. Also remove
heat_integrationtests/common/configs since they're no longer required.

* copy post.yaml for grenade jobs. As we migrate to zuul v3 for
functional tests but not grenade (not yet), the post.yaml should exists
under grenade dir. since it's only required by grenade jobs.

* Use post.yaml in functional tests for cleanup test environments.

Story: #2007056
Task: #37908

Depends-On: https://review.opendev.org/701105
Change-Id: I4f531161a7222e2c2a21f8d483f9c2a1d91dc38d

14 files changed, 209 insertions, 408 deletions

diff --git a/.zuul.yaml b/.zuul.yaml
index 367ed273a..fa08156ad 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,26 +1,106 @@
 - job:
-    name: heat-functional-devstack-base
-    parent: legacy-dsvm-base
+    name: heat-functional-base
+    parent: devstack
+    abstract: true
     run: playbooks/devstack/functional/run.yaml
     post-run: playbooks/devstack/functional/post.yaml
+    description: Base heat functional test job
     timeout: 7800
+    roles:
+      - zuul: opendev.org/openstack/devstack
+      - zuul: opendev.org/openstack/tempest
     required-projects:
       - openstack/devstack-gate
       - openstack/aodh
       - openstack/barbican
-      - openstack/barbican-tempest-plugin
       - openstack/ceilometer
-      - openstack/devstack-plugin-amqp1
       - openstack/heat
+      - openstack/heat-templates
       - openstack/heat-tempest-plugin
-      - openstack/neutron
       - openstack/octavia
+      - openstack/neutron
       - openstack/oslo.messaging
       - openstack/python-barbicanclient
       - openstack/python-heatclient
       - openstack/heat-agents
       - openstack/python-zaqarclient
       - openstack/zaqar
+      - openstack/tempest
+    vars:
+      gabbi_tempest_path: heat_tempest_plugin.tests.api.gabbits
+      tempest_plugins:
+        - heat-tempest-plugin
+      devstack_localrc:
+        TEMPEST_PLUGINS: '/opt/stack/heat-tempest-plugin'
+        HEAT_USE_MOD_WSGI: True
+        CEILOMETER_PIPELINE_INTERVAL: 60
+      devstack_services:
+        tls-proxy: false
+        s-account: false
+        s-container: false
+        s-object: false
+        s-proxy: false
+        tempest: true
+      devstack_plugins:
+        barbican: https://opendev.org/openstack/barbican
+        ceilometer: https://opendev.org/openstack/ceilometer
+        aodh: https://opendev.org/openstack/aodh
+        zaqar: https://opendev.org/openstack/zaqar
+        heat: https://opendev.org/openstack/heat
+        octavia: https://opendev.org/openstack/octavia
+      devstack_local_conf:
+        post-config:
+          $HEAT_CONF:
+            DEFAULT:
+              convergence_engine: true
+              stack_scheduler_hints: true
+              hidden_stack_tags: hidden
+              encrypt_parameters_and_properties: True
+              logging_exception_prefix: "%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s"
+              enable_stack_adopt: true
+              enable_stack_abandon: true
+              heat_api:
+                workers: 2
+              heat_api_cfn:
+                workers: 2
+              cache:
+                enabled: True
+              eventlet_opts:
+                client_socket_timeout: 120
+              oslo_messaging_notifications:
+                driver: messagingv2
+        test-config:
+          $TEMPEST_CONFIG:
+            service_available:
+              heat: True
+            heat_plugin:
+              convergence_engine_enabled: true
+              minimal_image_ref: ${DEFAULT_IMAGE_NAME:-cirros-0.3.6-x86_64-disk}
+              instance_type: m1.heat_int
+              minimal_instance_type: m1.heat_micro
+              image_ref: Fedora-Cloud-Base-29-1.2.x86_64
+              hidden_stack_tag: hidden
+              heat_config_notify_script: /opt/stack/heat-agents/heat-config/bin/heat-config-notify
+              boot_config_env: /opt/stack/heat-templates/hot/software-config/boot-config/test_image_env.yaml
+              credential_secret_id: $OS_CREDENTIAL_SECRET_ID
+            heat_features_enabled:
+              multi_cloud: True
+            # disable cinder backup feature
+            volume-feature-enabled:
+              backup: False
+      test_results_stage_name: test_results
+      zuul_copy_output:
+        '{{ devstack_base_dir }}/tempest/etc/tempest.conf': logs
+        '{{ devstack_base_dir }}/tempest/etc/accounts.yaml': logs
+        '{{ devstack_base_dir }}/tempest/tempest.log': logs
+        '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs
+        '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs
+        '{{ stage_dir }}/stackviz': logs
+      extensions_to_txt:
+        conf: true
+        log: true
+        yaml: true
+        yml: true
     irrelevant-files:
       - ^.*\.rst$
       - ^api-ref/.*$
@@ -28,44 +108,31 @@
       - ^heat/locale/.*$
       - ^heat/tests/.*$
       - ^releasenotes/.*$
-    vars:
-      disable_convergence: 'false'
-      sql: mysql
-      use_amqp1: 0
-      use_apache: 1
-      use_python3: 1
-      branch_override: default
 
-- job:
-    name: heat-functional-legacy
-    parent: heat-functional-devstack-base
-    vars:
-      disable_convergence: 'true'
 
 - job:
     name: heat-functional
-    parent: heat-functional-devstack-base
+    parent: heat-functional-base
 
 - job:
-    name: heat-functional-amqp1
-    parent: heat-functional-devstack-base
-    voting: false
-    branches: master
-    vars:
-      use_amqp1: 1
-
-- job:
-    name: heat-functional-non-apache
-    parent: heat-functional-devstack-base
-    voting: false
+    name: heat-functional-legacy
+    parent: heat-functional-base
     vars:
-      use_apache: 0
+      devstack_local_conf:
+        post-config:
+          $HEAT_CONF:
+            DEFAULT:
+              convergence_engine: false
+        test-config:
+          $TEMPEST_CONFIG:
+            heat_plugin:
+              convergence_engine_enabled: false
 
 - job:
     name: grenade-heat
     parent: legacy-dsvm-base
     run: playbooks/devstack/grenade/run.yaml
-    post-run: playbooks/devstack/functional/post.yaml
+    post-run: playbooks/devstack/grenade/post.yaml
     timeout: 7800
     required-projects:
       - openstack/grenade
@@ -110,8 +177,6 @@
         - grenade-heat-multinode
         - heat-functional
         - heat-functional-legacy
-        - heat-functional-amqp1
-        - heat-functional-non-apache
     gate:
       queue: heat
       jobs:
diff --git a/devstack/lib/heat b/devstack/lib/heat
index ca4b166e0..1db18e89f 100644
--- a/devstack/lib/heat
+++ b/devstack/lib/heat
@@ -441,15 +441,56 @@ function create_heat_accounts {
 # NOTE (gmann): Configure all the Tempest setting for Heat service in
 # this function.
 function configure_tempest_for_heat {
-    if is_service_enabled tempest; then
-        iniset $TEMPEST_CONFIG service_available heat True
+    # Skip SoftwareConfigIntegrationTest because it requires a custom image
+    # Skip AutoscalingLoadBalancerTest and AutoscalingLoadBalancerv2Test as deprecated neutron-lbaas service is not enabled
+    iniset $TEMPEST_CONFIG heat_plugin skip_scenario_test_list 'AutoscalingLoadBalancerTest, AutoscalingLoadBalancerv2Test, SoftwareConfigIntegrationTest'
+    # Skip LoadBalancerv2Test as deprecated neutron-lbaas service is not enabled
+    iniset $TEMPEST_CONFIG heat_plugin skip_functional_test_list 'LoadBalancerv2Test, NotificationTest'
+
+    openstack flavor show m1.heat_int || openstack flavor create m1.heat_int --ram 512 --disk 4
+    openstack flavor show m1.heat_micro || openstack flavor create m1.heat_micro --ram 128 --disk 1
+
+    source $TOP_DIR/openrc demo demo
+    openstack network show heat-net || openstack network create heat-net
+    openstack subnet show heat-subnet || openstack subnet create heat-subnet --network heat-net --subnet-range 10.0.5.0/24
+    openstack router add subnet router1 heat-subnet
+
+    iniset $TEMPEST_CONFIG heat_plugin username $OS_USERNAME
+    iniset $TEMPEST_CONFIG heat_plugin password $OS_PASSWORD
+    iniset $TEMPEST_CONFIG heat_plugin project_name $OS_PROJECT_NAME
+    iniset $TEMPEST_CONFIG heat_plugin auth_url $OS_AUTH_URL
+    iniset $TEMPEST_CONFIG heat_plugin user_domain_id $OS_USER_DOMAIN_ID
+    iniset $TEMPEST_CONFIG heat_plugin project_domain_id $OS_PROJECT_DOMAIN_ID
+    iniset $TEMPEST_CONFIG heat_plugin user_domain_name $OS_USER_DOMAIN_NAME
+    iniset $TEMPEST_CONFIG heat_plugin project_domain_name $OS_PROJECT_DOMAIN_NAME
+    iniset $TEMPEST_CONFIG heat_plugin region $OS_REGION_NAME
+    iniset $TEMPEST_CONFIG heat_plugin auth_version $OS_IDENTITY_API_VERSION
+
+    source $TOP_DIR/openrc admin admin
+    iniset $TEMPEST_CONFIG heat_plugin admin_username $OS_USERNAME
+    iniset $TEMPEST_CONFIG heat_plugin admin_password $OS_PASSWORD
+    if [[ -e /etc/ci/mirror_info.sh ]]; then
+        source /etc/ci/mirror_info.sh
     fi
+    export HEAT_TEST_FEDORA_IMAGE=${NODEPOOL_FEDORA_MIRROR:-https://download.fedoraproject.org/pub/fedora/linux}/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2
+    TOKEN=$(openstack token issue -c id -f value)
+    local image_exists=$( openstack image list | grep "Fedora-Cloud-Base-29-1.2.x86_64" )
+    if [[ -z $image_exists ]]; then
+        if is_service_enabled g-reg; then
+            upload_image $HEAT_TEST_FEDORA_IMAGE $TOKEN
+        fi
+    fi
+
+    if is_service_enabled tls-proxy; then
+        iniset $TEMPEST_CONFIG heat_plugin ca_file $SSL_BUNDLE_FILE
+    fi
+    # add application credential and secret to support test multi-cloud
+    app_cred_id=$(openstack application credential show  heat_multicloud || openstack application credential create heat_multicloud \
+        --secret secret --unrestricted -c id -f value)
+    export OS_CREDENTIAL_SECRET_ID=$(openstack secret store -n heat-multi-cloud-test-cred --payload \
+        '{"auth_type": "v3applicationcredential", "auth": {"auth_url": $OS_AUTH_URL, "application_credential_id": $app_cred_id, "application_credential_secret": "secret"}}'\
+        -c "Secret href" -f value)
 }
 
 # Restore xtrace
 $_XTRACE_HEAT
-
-# Tell emacs to use shell-script-mode
-## Local variables:
-## mode: shell-script
-## End:
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index ae919b6bb..86c8b0985 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -18,7 +18,8 @@ if is_heat_enabled; then
 
     elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
         if is_service_enabled tempest; then
-            setup_develop $TEMPEST_DIR
+            echo_summary "Configuring Tempest for Heat"
+            configure_tempest_for_heat
         fi
 
     elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
@@ -35,10 +36,6 @@ if is_heat_enabled; then
         # Start the heat API and heat taskmgr components
         echo_summary "Starting heat"
         start_heat
-
-    elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
-        echo_summary "Configuring Tempest for Heat"
-        configure_tempest_for_heat
     fi
 
     if [[ "$1" == "unstack" ]]; then
diff --git a/heat_integrationtests/__init__.py b/heat_integrationtests/__init__.py
index ec0913317..625951c3c 100644
--- a/heat_integrationtests/__init__.py
+++ b/heat_integrationtests/__init__.py
@@ -14,16 +14,14 @@
 import os
 import unittest
 
-from heat_integrationtests.common import config
-
 from oslo_log import log as logging
+from tempest import config
 
 LOG = logging.getLogger(__name__, project=__name__)
 
 
 def load_tests(loader, standard_tests, pattern):
-    logging.setup(config.init_conf(), __name__)
-
+    logging.setup(config.CONF, __name__)
     suite = unittest.TestSuite()
 
     heat_integration_dir = os.path.dirname(os.path.abspath(__file__))
diff --git a/heat_integrationtests/cleanup_test_env.sh b/heat_integrationtests/cleanup_test_env.sh
index ee32d17a3..25500cc92 100755
--- a/heat_integrationtests/cleanup_test_env.sh
+++ b/heat_integrationtests/cleanup_test_env.sh
@@ -12,7 +12,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
-# This script is executed inside post_test_hook function in devstack gate.
+# This script is executed in devstack gate.
 
 set -ex
 
diff --git a/heat_integrationtests/common/config.py b/heat_integrationtests/common/config.py
deleted file mode 100644
index 8ec24d833..000000000
--- a/heat_integrationtests/common/config.py
+++ /dev/null
@@ -1,147 +0,0 @@
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import os
-
-from oslo_config import cfg
-from oslo_log import log as logging
-
-import heat_integrationtests
-
-_CONF = None
-
-heat_group = cfg.OptGroup(name="heat_plugin",
-                          title="Heat Service Options")
-
-HeatGroup = [
-    cfg.StrOpt("catalog_type",
-               default="orchestration",
-               help="Catalog type of the orchestration service."),
-    cfg.StrOpt('username',
-               help="Username to use for non admin API requests."),
-    cfg.StrOpt('password',
-               help="Non admin API key to use when authenticating.",
-               secret=True),
-    cfg.StrOpt('admin_username',
-               help="Username to use for admin API requests."),
-    cfg.StrOpt('admin_password',
-               help="Admin API key to use when authentication.",
-               secret=True),
-    cfg.StrOpt('project_name',
-               help="Project name to use for API requests.",
-               deprecated_opts=[cfg.DeprecatedOpt('tenant_name',
-                                                  group='heat_plugin')]),
-    cfg.StrOpt('admin_project_name',
-               default='admin',
-               help="Admin project name to use for admin API requests.",
-               deprecated_opts=[cfg.DeprecatedOpt('admin_tenant_name',
-                                                  group='heat_plugin')]),
-    cfg.StrOpt('auth_url',
-               help="Full URI of the OpenStack Identity API (Keystone)."),
-    cfg.StrOpt('auth_version',
-               help="OpenStack Identity API version."),
-    cfg.StrOpt('user_domain_name',
-               help="User domain name, if keystone v3 auth_url "
-                    "is used"),
-    cfg.StrOpt('project_domain_name',
-               help="Project domain name, if keystone v3 auth_url "
-                    "is used"),
-    cfg.StrOpt('user_domain_id',
-               help="User domain id, if keystone v3 auth_url "
-                    "is used"),
-    cfg.StrOpt('project_domain_id',
-               help="Project domain id, if keystone v3 auth_url "
-                    "is used"),
-    cfg.StrOpt('region',
-               help="The region name to use"),
-    cfg.StrOpt('instance_type',
-               help="Instance type for tests. Needs to be big enough for a "
-                    "full OS plus the test workload"),
-    cfg.StrOpt('minimal_instance_type',
-               help="Instance type enough for simplest cases."),
-    cfg.StrOpt('image_ref',
-               help="Name of image to use for tests which boot servers."),
-    cfg.StrOpt('keypair_name',
-               help="Name of existing keypair to launch servers with."),
-    cfg.StrOpt('minimal_image_ref',
-               help="Name of minimal (e.g cirros) image to use when "
-                    "launching test instances."),
-    cfg.BoolOpt('disable_ssl_certificate_validation',
-                default=False,
-                help="Set to True if using self-signed SSL certificates."),
-    cfg.StrOpt('ca_file',
-               help="CA certificate to pass for servers that have "
-                    "https endpoint."),
-    cfg.IntOpt('build_interval',
-               default=4,
-               help="Time in seconds between build status checks."),
-    cfg.IntOpt('build_timeout',
-               default=1200,
-               help="Timeout in seconds to wait for a stack to build."),
-    cfg.StrOpt('network_for_ssh',
-               default='heat-net',
-               help="Network used for SSH connections."),
-    cfg.StrOpt('fixed_network_name',
-               default='heat-net',
-               help="Visible fixed network name "),
-    cfg.StrOpt('floating_network_name',
-               default='public',
-               help="Visible floating network name "),
-    cfg.StrOpt('fixed_subnet_name',
-               default='heat-subnet',
-               help="Visible fixed sub-network name "),
-    cfg.BoolOpt('skip_functional_tests',
-                default=False,
-                help="Skip all functional tests"),
-    cfg.ListOpt('skip_functional_test_list',
-                help="List of functional test class or class.method "
-                     "names to skip ex. AutoscalingGroupTest, "
-                     "InstanceGroupBasicTest.test_size_updates_work"),
-    cfg.ListOpt('skip_test_stack_action_list',
-                help="List of stack actions in tests to skip "
-                     "ex. ABANDON, ADOPT, SUSPEND, RESUME"),
-    cfg.BoolOpt('convergence_engine_enabled',
-                default=True,
-                help="Test features that are only present for stacks with "
-                     "convergence enabled."),
-    cfg.IntOpt('connectivity_timeout',
-               default=120,
-               help="Timeout in seconds to wait for connectivity to "
-                    "server."),
-]
-
-
-def init_conf(read_conf=True):
-    global _CONF
-    if _CONF is not None:
-        return _CONF
-
-    default_config_files = None
-    if read_conf:
-        confpath = os.path.join(
-            os.path.dirname(os.path.realpath(heat_integrationtests.__file__)),
-            'heat_integrationtests.conf')
-        if os.path.isfile(confpath):
-            default_config_files = [confpath]
-
-    _CONF = cfg.ConfigOpts()
-    logging.register_options(_CONF)
-    _CONF(args=[], project='heat_integrationtests',
-          default_config_files=default_config_files)
-
-    for group, opts in list_opts():
-        _CONF.register_opts(opts, group=group)
-    return _CONF
-
-
-def list_opts():
-    yield heat_group.name, HeatGroup
diff --git a/heat_integrationtests/common/test.py b/heat_integrationtests/common/test.py
index 86d7d0a55..d5c259501 100644
--- a/heat_integrationtests/common/test.py
+++ b/heat_integrationtests/common/test.py
@@ -23,11 +23,11 @@ from oslo_log import log as logging
 from oslo_utils import timeutils
 import six
 from six.moves import urllib
+from tempest import config
 import testscenarios
 import testtools
 
 from heat_integrationtests.common import clients
-from heat_integrationtests.common import config
 from heat_integrationtests.common import exceptions
 
 LOG = logging.getLogger(__name__)
@@ -71,8 +71,7 @@ def requires_convergence(test_method):
 
     The decorated test will be skipped when convergence is disabled.
     '''
-    convergence_enabled = config.init_conf(
-    ).heat_plugin.convergence_engine_enabled
+    convergence_enabled = config.CONF.heat_plugin.convergence_engine_enabled
     skipper = testtools.skipUnless(convergence_enabled,
                                    "Convergence-only tests are disabled")
     return skipper(test_method)
@@ -84,7 +83,7 @@ class HeatIntegrationTest(testscenarios.WithScenarios,
     def setUp(self):
         super(HeatIntegrationTest, self).setUp()
 
-        self.conf = config.init_conf().heat_plugin
+        self.conf = config.CONF.heat_plugin
 
         self.assertIsNotNone(self.conf.auth_url,
                              'No auth_url configured')
diff --git a/heat_integrationtests/post_test_hook.sh b/heat_integrationtests/post_test_hook.sh
deleted file mode 100755
index 3d975c657..000000000
--- a/heat_integrationtests/post_test_hook.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This script is executed inside post_test_hook function in devstack gate.
-
-set -ex
-
-export DEST=${DEST:-/opt/stack/new}
-export TOP_DIR=${TOP_DIR:-/opt/stack/new/devstack}
-sudo -E $DEST/heat/heat_integrationtests/prepare_test_env.sh
-sudo -E $DEST/heat/heat_integrationtests/prepare_test_network.sh
-
-cd $DEST/tempest
-sudo tox -evenv-tempest -- stestr --test-path=$DEST/heat/heat_integrationtests --top-dir=$DEST/heat --group_regex='heat_tempest_plugin\.tests\.api\.test_heat_api[._]([^_]+)' run
-
-sudo -E $DEST/heat/heat_integrationtests/cleanup_test_env.sh
diff --git a/heat_integrationtests/pre_test_hook.sh b/heat_integrationtests/pre_test_hook.sh
deleted file mode 100755
index 8194e97cd..000000000
--- a/heat_integrationtests/pre_test_hook.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/bin/bash
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This script is executed inside pre_test_hook function in devstack gate.
-
-set -x
-
-localconf=$BASE/new/devstack/local.conf
-
-echo -e '[[post-config|$HEAT_CONF]]\n[DEFAULT]\n' >> $localconf
-
-if [ "$DISABLE_CONVERGENCE" == "true" ] ; then
-    echo -e 'convergence_engine=false\n' >> $localconf
-fi
-
-echo -e 'stack_scheduler_hints=true\n' >> $localconf
-echo -e 'hidden_stack_tags=hidden\n' >> $localconf
-echo -e 'encrypt_parameters_and_properties=True\n' >> $localconf
-echo -e 'logging_exception_prefix=%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s %(instance)s\n' >> $localconf
-
-echo -e '[heat_api]\nworkers=2\n' >> $localconf
-echo -e '[heat_api_cfn]\nworkers=2\n' >> $localconf
-
-echo -e '[cache]\nenabled=True\n' >> $localconf
-
-echo -e '[eventlet_opts]\nclient_socket_timeout=120\n' >> $localconf
-
-echo -e '[oslo_messaging_notifications]\ndriver=messagingv2\n' >> $localconf
-
-echo "[[local|localrc]]" >> $localconf
-
-# NOTE(mnaser): This will use the region local mirrors to avoid going out
-#               to network
-if [[ -e /etc/ci/mirror_info.sh ]]; then
-	source /etc/ci/mirror_info.sh
-	echo "IMAGE_URLS+=${NODEPOOL_FEDORA_MIRROR}/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" >> $localconf
-else
-	echo "IMAGE_URLS+=https://download.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" >> $localconf
-fi
-
-echo "CEILOMETER_PIPELINE_INTERVAL=60" >> $localconf
-echo "HEAT_ENABLE_ADOPT_ABANDON=True" >> $localconf
diff --git a/playbooks/devstack/functional/post.yaml b/playbooks/devstack/functional/post.yaml
index e07f5510a..5f3cf3282 100644
--- a/playbooks/devstack/functional/post.yaml
+++ b/playbooks/devstack/functional/post.yaml
@@ -1,15 +1,19 @@
 - hosts: primary
+  name: Clear test env
   tasks:
+    - name: clear test env
+      shell:
+        cmd: |
+            /opt/stack/heat/heat_integrationtests/cleanup_test_env.sh
+        executable: /bin/bash
+        chdir: "{{ zuul.project.src_dir }}"
+      environment:
+        DEVSTACK_BASE_DIR: "{{ devstack_base_dir }}"
+      become: true
 
-    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
-      synchronize:
-        src: '{{ ansible_user_dir }}/workspace/'
-        dest: '{{ zuul.executor.log_root }}'
-        mode: pull
-        copy_links: true
-        verify_host: true
-        rsync_opts:
-          - --include=/logs/**
-          - --include=*/
-          - --exclude=*
-          - --prune-empty-dirs
+- hosts: tempest
+  become: true
+  roles:
+    - role: fetch-subunit-output
+      zuul_work_dir: '{{ devstack_base_dir }}/tempest'
+    - role: process-stackviz
diff --git a/playbooks/devstack/functional/run.yaml b/playbooks/devstack/functional/run.yaml
index 6c7f80d19..a510da4eb 100644
--- a/playbooks/devstack/functional/run.yaml
+++ b/playbooks/devstack/functional/run.yaml
@@ -1,116 +1,15 @@
 - hosts: all
-  name: Job for functional tests
-  tasks:
-
-    - name: Ensure legacy workspace directory
-      file:
-        path: '{{ ansible_user_dir }}/workspace'
-        state: directory
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          cat > clonemap.yaml << EOF
-          clonemap:
-            - name: openstack/devstack-gate
-              dest: devstack-gate
-          EOF
-          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
-              https://opendev.org \
-              openstack/devstack-gate
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-
-    - shell:
-        cmd: |
-          set -e
-          set -x
-          export PYTHONUNBUFFERED=true
-          services=rabbit,tempest,mysql,dstat,key
-          services+=,n-api,n-api-meta,n-cpu,n-cond,n-sch,n-crt
-          services+=,placement-api,placement-client
-          services+=,g-api,g-reg
-          services+=,c-sch,c-api,c-vol,c-bak
-          services+=,neutron-api,neutron-dhcp,neutron-metadata-agent,neutron-agent,neutron-l3,neutron-trunk
-
-          if [ "{{ use_python3 }}" -eq 1 ] ; then
-             export DEVSTACK_GATE_USE_PYTHON3=True
-             # Swift does not work so skip s-* for python3x for now
-          else
-             export DEVSTACK_GATE_USE_PYTHON3=False
-             services+=,s-proxy,s-object,s-container,s-account
-          fi
-
-          export DEVSTACK_GATE_NEUTRON=1
-          export DEVSTACK_GATE_TEMPEST=1
-          export DEVSTACK_GATE_TEMPEST_NOTESTS=1
-          export DEVSTACK_GATE_INSTALL_TESTONLY=1
-          export DEVSTACK_PROJECT_FROM_GIT=python-heatclient
-          export KEEP_LOCALRC=1
-          export PROJECTS="openstack/ceilometer $PROJECTS"
-          export PROJECTS="openstack/aodh $PROJECTS"
-          export PROJECTS="openstack/zaqar $PROJECTS"
-          export PROJECTS="openstack/heat-agents $PROJECTS"
-          export PROJECTS="openstack/python-zaqarclient $PROJECTS"
-          export PROJECTS="openstack/neutron $PROJECTS"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ceilometer https://opendev.org/openstack/ceilometer"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin aodh https://opendev.org/openstack/aodh"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin zaqar https://opendev.org/openstack/zaqar"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin neutron https://opendev.org/openstack/neutron"
-
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat https://opendev.org/openstack/heat"
-
-          # Enable octavia plugin and services
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin octavia https://opendev.org/openstack/octavia"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_AMP_IMAGE_FILE=/tmp/test-only-amphora-x64-haproxy-ubuntu-bionic.qcow2"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_AMP_IMAGE_SIZE=3"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_AMP_IMAGE_NAME=test-only-amphora-x64-haproxy-ubuntu-bionic"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_AMPHORA_DRIVER=amphora_noop_driver"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_COMPUTE_DRIVER=compute_noop_driver"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"OCTAVIA_NETWORK_DRIVER=network_noop_driver"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"DISABLE_AMP_IMAGE_BUILD=True"
-          services+=,octavia,o-cw,o-hk,o-hm,o-api
-          export PROJECTS="openstack/octavia $PROJECTS"
-          export PROJECTS="openstack/barbican $PROJECTS"
-          export PROJECTS="openstack/python-barbicanclient $PROJECTS"
-          export PROJECTS="openstack/barbican-tempest-plugin $PROJECTS"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin barbican https://opendev.org/openstack/barbican"
-
-          # use heat-tempest-plugin
-          export PROJECTS="openstack/heat-tempest-plugin $PROJECTS"
-          export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/heat-tempest-plugin'"
-
-          export OVERRIDE_ENABLED_SERVICES=$services
-
-          if [ "{{ branch_override }}" != "default" ] ; then
-              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
-          fi
-          if [ "{{ use_apache  }}" -eq 0 ] ; then
-              export DEVSTACK_LOCAL_CONFIG+=$'\n'"HEAT_USE_MOD_WSGI=False"
-          fi
-          if [ "{{ use_amqp1 }}" -eq 1 ] ; then
-              export PROJECTS="openstack/devstack-plugin-amqp1 $PROJECTS"
-              export DEVSTACK_LOCAL_CONFIG+=$'\n'"export AMQP1_SERVICE=qpid-hybrid"
-              export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin devstack-plugin-amqp1 https://opendev.org/openstack/devstack-plugin-amqp1"
-              export DEVSTACK_LOCAL_CONFIG+=$'\n'"export CELLSV2_SETUP=singleconductor"
-              export DEVSTACK_PROJECT_FROM_GIT="oslo.messaging,$DEVSTACK_PROJECT_FROM_GIT"
-          fi
-          export DISABLE_CONVERGENCE="{{ disable_convergence }}"
-          function pre_test_hook {
-              cd /opt/stack/new/heat/heat_integrationtests
-              source ./pre_test_hook.sh
-          }
-          export -f pre_test_hook
-
-          function post_test_hook {
-              cd /opt/stack/new/heat/heat_integrationtests
-              source ./post_test_hook.sh
-          }
-          export -f post_test_hook
-
-          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
-          ./safe-devstack-vm-gate-wrap.sh
-        executable: /bin/bash
-        chdir: '{{ ansible_user_dir }}/workspace'
-      environment: '{{ zuul | zuul_legacy_vars }}'
+  roles:
+    - orchestrate-devstack
+
+- hosts: tempest
+  environment:
+    # This enviroment variable is used by the optional tempest-gabbi
+    # job provided by the gabbi-tempest plugin. It can be safely ignored
+    # if that plugin is not being used.
+    GABBI_TEMPEST_PATH: "{{ gabbi_tempest_path | default('') }}"
+  roles:
+    - setup-tempest-run-dir
+    - setup-tempest-data-dir
+    - acl-devstack-files
+    - run-heat-tests
diff --git a/playbooks/devstack/grenade/post.yaml b/playbooks/devstack/grenade/post.yaml
new file mode 100644
index 000000000..baf876086
--- /dev/null
+++ b/playbooks/devstack/grenade/post.yaml
@@ -0,0 +1,14 @@
+- hosts: primary
+  tasks:
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/roles/run-heat-tests/defaults/main.yaml b/roles/run-heat-tests/defaults/main.yaml
new file mode 100644
index 000000000..b601d49d9
--- /dev/null
+++ b/roles/run-heat-tests/defaults/main.yaml
@@ -0,0 +1,2 @@
+devstack_base_dir: /opt/stack
+tempest_test_timeout: ''
diff --git a/roles/run-heat-tests/tasks/main.yaml b/roles/run-heat-tests/tasks/main.yaml
new file mode 100644
index 000000000..75122f2a1
--- /dev/null
+++ b/roles/run-heat-tests/tasks/main.yaml
@@ -0,0 +1,9 @@
+- name: Run heat tests
+  command: tox -evenv-tempest -- stestr --test-path={{devstack_base_dir}}/heat/heat_integrationtests \
+            --top-dir={{devstack_base_dir}}/heat \
+            --group_regex='heat_tempest_plugin\.tests\.api\.test_heat_api[._]([^_]+)' run
+  args:
+    chdir: "{{devstack_base_dir}}/tempest"
+  become: true
+  become_user: tempest
+  environment: '{{ {"OS_TEST_TIMEOUT": tempest_test_timeout} if tempest_test_timeout else {} }}'