WIP: add guest image build check

This PR adds trove-guest-image-build check in ci.

Change-Id: If512cb7c094d14850555215d412b8e7b418c35ce

3 files changed, 516 insertions, 0 deletions

diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml
new file mode 100644
index 00000000..2decd526
--- /dev/null
+++ b/zuul.d/jobs.yaml
@@ -0,0 +1,459 @@
+- job:
+    name: trove-devstack-base
+    parent: devstack
+    abstract: true
+    timeout: 10800
+    required-projects:
+      - openstack/tempest
+      - openstack/diskimage-builder
+      - openstack/tripleo-image-elements
+      - openstack/trove
+      - openstack/trove-dashboard
+    vars:
+      devstack_services:
+        tempest: true
+        etcd3: false
+        ceilometer-acentral: false
+        ceilometer-acompute: false
+        ceilometer-alarm-evaluator: false
+        ceilometer-alarm-notifier: false
+        ceilometer-anotification: false
+        ceilometer-api: false
+        ceilometer-collector: false
+        q-svc: true
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-ovn-metadata-agent: false
+        ovn-controller: false
+        ovn-northd: false
+      devstack_plugins:
+        trove: https://opendev.org/openstack/trove
+      # NOTE(zhaochao): we have to override the default settings from the
+      # the base devstack job about "SERVICE_HOST" and "HOST_IP", as trove
+      # guestagent should use public addresses to access the underlying
+      # components(MQ and Swift), and the previous devstack-gate jobs didn't
+      # set these variables. However we cannot unset variables in job
+      # definition, so we just set SERVICE_HOST and HOST_IP to empty strings,
+      # and VNCSERVER_PROXYCLIENT_ADDRESS to 127.0.0.1 instead.
+      devstack_localrc:
+        SERVICE_HOST: ''
+        HOST_IP: ''
+        VNCSERVER_PROXYCLIENT_ADDRESS: 127.0.0.1
+        USE_PYTHON3: True
+        Q_AGENT: openvswitch
+        Q_PLUGIN: ml2
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+    run: playbooks/trove-devstack-base.yaml
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^tools/.*$
+      - ^trove/hacking/.*$
+      - ^trove/tests/unittests/.*$
+
+- job:
+    name: trove-fakemodetests-base
+    parent: openstack-tox-py36
+    abstract: true
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^contrib/.*$
+      - ^doc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^trove/hacking/.*$
+      - ^trove/tests/api/.*$
+      - ^trove/tests/db/.*$
+      - ^trove/tests/scenario/.*$
+      - ^trove/tests/unittests/.*$
+
+- job:
+    name: trove-functional-mysql
+    parent: trove-devstack-base
+    nodeset: trove-ubuntu-focal
+    timeout: 10800
+    vars:
+      devstack_localrc:
+        TROVE_RESIZE_TIME_OUT: 1800
+      trove_resize_time_out: 1800
+      trove_test_datastore: 'mysql'
+      trove_test_group: 'mysql'
+      trove_test_datastore_version: '5.7.29'
+
+- job:
+    name: trove-functional-mysql-nondev
+    parent: trove-devstack-base
+    vars:
+      devstack_localrc:
+        TROVE_RESIZE_TIME_OUT: 1800
+        TROVE_NON_DEV_IMAGE_URL: https://tarballs.opendev.org/openstack/trove/images/trove-master-mysql-ubuntu-xenial.qcow2
+      trove_resize_time_out: 1800
+      trove_test_datastore: 'mysql'
+      trove_test_group: 'mysql'
+      trove_test_datastore_version: '5.7.29'
+
+- job:
+    name: trove-scenario-cassandra-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: cassandra
+      trove_test_group: cassandra-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-cassandra-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: cassandra
+      trove_test_group: cassandra-supported-multi
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-couchdb-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: couchdb
+      trove_test_group: couchdb-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-couchbase-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: couchbase
+      trove_test_group: couchbase-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-mariadb-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: mariadb
+      trove_test_group: mariadb-supported-single
+      trove_test_datastore_version: 10.4.12
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-mariadb-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: mariadb
+      trove_test_group: mariadb-supported-multi
+      trove_test_datastore_version: 10.4.12
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-mysql-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: mysql
+      trove_test_group: mysql-supported-single
+      trove_test_datastore_version: 5.7.29
+
+- job:
+    name: trove-scenario-mysql-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: mysql
+      trove_test_group: mysql-supported-multi
+      trove_test_datastore_version: 5.7.29
+
+- job:
+    name: trove-scenario-percona-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: percona
+      trove_test_group: percona-supported-multi
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-percona-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: percona
+      trove_test_group: percona-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-postgresql-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: postgresql
+      trove_test_group: postgresql-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-postgresql-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: postgresql
+      trove_test_group: postgresql-supported-multi
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-pxc-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: pxc
+      trove_test_group: pxc-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-pxc-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: pxc
+      trove_test_group: pxc-supported-multi
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-redis-single
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: redis
+      trove_test_group: redis-supported-single
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-scenario-redis-multi
+    parent: trove-devstack-base
+    vars:
+      trove_test_datastore: redis
+      trove_test_group: redis-supported-multi
+      devstack_localrc:
+        TROVE_ENABLE_IMAGE_BUILD: false
+
+- job:
+    name: trove-tox-bandit-baseline
+    parent: openstack-tox
+    timeout: 2400
+    vars:
+      tox_envlist: bandit-baseline
+    required-projects:
+      - openstack/requirements
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^.*\.txt$
+      - ^api-ref/.*$
+      - ^contrib/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^trove/hacking/.*$
+      - ^trove/tests/scenario/.*$
+      - ^trove/tests/unittests/.*$
+
+- job:
+    name: trove-tempest
+    parent: devstack-tempest
+    nodeset: trove-ubuntu-focal
+    timeout: 10800
+    required-projects: &base_required_projects
+      - openstack/trove
+      - openstack/trove-tempest-plugin
+      - openstack/tempest
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+    vars: &base_vars
+      tox_envlist: all
+      tempest_concurrency: 1
+      devstack_localrc:
+        TEMPEST_PLUGINS: /opt/stack/trove-tempest-plugin
+        USE_PYTHON3: true
+        Q_AGENT: openvswitch
+        Q_PLUGIN: ml2
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+      devstack_local_conf:
+        post-config:
+          $TROVE_CONF:
+            DEFAULT:
+              usage_timeout: 1800
+      devstack_plugins:
+        trove: https://opendev.org/openstack/trove.git
+      devstack_services:
+        etcd3: false
+        ceilometer-acentral: false
+        ceilometer-acompute: false
+        ceilometer-alarm-evaluator: false
+        ceilometer-alarm-notifier: false
+        ceilometer-anotification: false
+        ceilometer-api: false
+        ceilometer-collector: false
+        cinder: true
+        c-sch: true
+        c-api: true
+        c-vol: true
+        c-bak: false
+        swift: true
+        s-account: true
+        s-container: true
+        s-object: true
+        s-proxy: true
+        tempest: true
+        q-svc: true
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-ovn-metadata-agent: false
+        ovn-controller: false
+        ovn-northd: false
+      tempest_test_regex: ^trove_tempest_plugin\.tests
+      tempest_test_blacklist: '{{ ansible_user_dir }}/{{ zuul.projects["opendev.org/openstack/trove-tempest-plugin"].src_dir }}/tempest_blacklist.txt'
+
+- job:
+    name: trove-tempest-postgres
+    parent: devstack-tempest
+    nodeset: trove-ubuntu-focal
+    timeout: 10800
+    required-projects:
+      - openstack/trove
+      - openstack/trove-tempest-plugin
+      - openstack/tempest
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+      - ^trove/guestagent/(mysql.*|mariadb)/.*$
+      - ^trove/guestagent/strategies/replication/(mysql.*|mariadb.*)\.py$
+    vars:
+      tox_envlist: all
+      tempest_concurrency: 1
+      devstack_localrc:
+        TEMPEST_PLUGINS: /opt/stack/trove-tempest-plugin
+        USE_PYTHON3: true
+        Q_AGENT: openvswitch
+        Q_PLUGIN: ml2
+        Q_ML2_TENANT_NETWORK_TYPE: vxlan
+        Q_ML2_PLUGIN_MECHANISM_DRIVERS: openvswitch
+        TROVE_DATASTORE_TYPE: postgresql
+        TROVE_DATASTORE_VERSION: 12.4
+      devstack_local_conf:
+        post-config:
+          $TROVE_CONF:
+            DEFAULT:
+              usage_timeout: 1800
+        test-config:
+          $TEMPEST_CONFIG:
+            database:
+              enabled_datastores: "postgresql"
+              default_datastore_versions: "postgresql:12.4"
+      devstack_plugins:
+        trove: https://opendev.org/openstack/trove.git
+      devstack_services:
+        etcd3: false
+        ceilometer-acentral: false
+        ceilometer-acompute: false
+        ceilometer-alarm-evaluator: false
+        ceilometer-alarm-notifier: false
+        ceilometer-anotification: false
+        ceilometer-api: false
+        ceilometer-collector: false
+        cinder: true
+        c-sch: true
+        c-api: true
+        c-vol: true
+        c-bak: false
+        swift: true
+        s-account: true
+        s-container: true
+        s-object: true
+        s-proxy: true
+        tempest: true
+        q-svc: true
+        q-agt: true
+        q-dhcp: true
+        q-l3: true
+        q-meta: true
+        q-ovn-metadata-agent: false
+        ovn-controller: false
+        ovn-northd: false
+      tempest_test_regex: ^trove_tempest_plugin\.tests
+      tempest_test_blacklist: '{{ ansible_user_dir }}/{{ zuul.projects["opendev.org/openstack/trove-tempest-plugin"].src_dir }}/tempest_blacklist.txt'
+
+- job:
+    name: publish-trove-guest-image
+    parent: publish-openstack-artifacts
+    run: playbooks/image-build/run.yaml
+    post-run: playbooks/image-build/post.yaml
+    required-projects:
+      - openstack/diskimage-builder
+      - openstack/trove
+      - openstack/tripleo-image-elements
+
+- job:
+    name: trove-ubuntu-guest-image-build
+    run: playbooks/image-build/run.yaml
+    nodeset: trove-ubuntu-focal-single
+    description: |
+      Build Ubuntu Bionic based image only on ubuntu distro.
+    required-projects:
+      - openstack/diskimage-builder
+      - openstack/trove
+      - openstack/tripleo-image-elements
+    files:
+      - ^integration/(scripts|tests)/
+    vars:
+      guest_os: ubuntu
+      guest_os_release: bionic
+      guest_username: ubuntu
+      branch: master
+      dev_mode: false
+      image_suffix: ""
+
+- job:
+    name: publish-trove-guest-image-ubuntu-bionic
+    parent: publish-trove-guest-image
+    description: |
+      Build and publish Ubuntu Bionic based Trove guest image to
+      tarballs.openstack.org.
+    vars:
+      guest_os: ubuntu
+      guest_os_release: bionic
+      guest_username: ubuntu
+      branch: master
+      dev_mode: false
+      image_suffix: ""
+
+- job:
+    name: publish-trove-guest-image-ubuntu-bionic-dev
+    parent: publish-trove-guest-image
+    description: |
+      Build and publish Ubuntu Bionic based Trove guest image to
+      tarballs.openstack.org.
+    vars:
+      guest_os: ubuntu
+      guest_os_release: bionic
+      guest_username: ubuntu
+      branch: master
+      dev_mode: true
+      image_suffix: "-dev"
diff --git a/zuul.d/nodesets.yaml b/zuul.d/nodesets.yaml
new file mode 100644
index 00000000..88688f9c
--- /dev/null
+++ b/zuul.d/nodesets.yaml
@@ -0,0 +1,15 @@
+- nodeset:
+    name: trove-ubuntu-focal
+    nodes:
+      - name: controller
+        label: nested-virt-ubuntu-focal
+    groups:
+      - name: tempest
+        nodes:
+          - controller
+
+- nodeset:
+    name: trove-ubuntu-focal-single
+    nodes:
+      - name: primary
+        label: ubuntu-focal
\ No newline at end of file
diff --git a/zuul.d/projects.yaml b/zuul.d/projects.yaml
new file mode 100644
index 00000000..0628d051
--- /dev/null
+++ b/zuul.d/projects.yaml
@@ -0,0 +1,42 @@
+
+- project:
+    templates:
+      - check-requirements
+      - openstack-cover-jobs
+      # - openstack-lower-constraints-jobs
+      - openstack-python3-zed-jobs
+      - periodic-stable-jobs
+      - publish-openstack-docs-pti
+      - release-notes-jobs-python3
+    check:
+      jobs:
+        - openstack-tox-cover:
+            voting: false
+        - trove-tox-bandit-baseline:
+            voting: false
+        - trove-tempest:
+            voting: false
+        - trove-ubuntu-guest-image-build:
+            voting: true
+        - trove-tempest-postgres:
+            voting: false
+        - trove-tempest-ipv6-only:
+            voting: false
+        - trove-functional-mysql:
+            voting: false
+    gate:
+      queue: trove
+      jobs:
+        - trove-tempest:
+            voting: false
+    experimental:
+      jobs:
+        - trove-functional-mysql
+        - trove-scenario-mysql-single
+        - trove-scenario-mysql-multi
+        - trove-scenario-mariadb-single
+        - trove-scenario-mariadb-multi
+    periodic:
+      jobs:
+        - publish-trove-guest-image-ubuntu-bionic
+        - publish-trove-guest-image-ubuntu-bionic-dev
\ No newline at end of file