Move pbr-installation jobs in-tree

These jobs are currently broken and to fix them we may need to make
changes to the job definitions.  Since they're pbr-specific we might
as well move them in-tree so we can iterate on them without going
through the global repo.


Needed-By: https://review.openstack.org/596535
Change-Id: I10313a8a534a64113fa03d7ee3b0ed28e49fbf8e

5 files changed, 420 insertions, 10 deletions

diff --git a/.zuul.yaml b/.zuul.yaml
index de05749..dd8cd77 100644
--- a/.zuul.yaml
+++ b/.zuul.yaml
@@ -1,22 +1,128 @@
+- job:
+    name: pbr-installation-devstack-base
+    parent: legacy-dsvm-base
+    timeout: 7800
+    required-projects:
+      - openstack-dev/devstack
+      - openstack-dev/grenade
+      - openstack-dev/pbr
+      - openstack-infra/devstack-gate
+      - openstack-infra/tripleo-ci
+      - openstack/aodh
+      - openstack/automaton
+      - openstack/ceilometer
+      - openstack/ceilometermiddleware
+      - openstack/cinder
+      - openstack/cliff
+      - openstack/debtcollector
+      - openstack/dib-utils
+      - openstack/diskimage-builder
+      - openstack/django_openstack_auth
+      - openstack/futurist
+      - openstack/glance
+      - openstack/glance_store
+      - openstack/heat
+      - openstack/heat-cfntools
+      - openstack/heat-templates
+      - openstack/horizon
+      - openstack/ironic
+      - openstack/ironic-lib
+      - openstack/ironic-python-agent
+      - openstack/keystone
+      - openstack/keystoneauth
+      - openstack/keystonemiddleware
+      - openstack/manila
+      - openstack/manila-ui
+      - openstack/neutron
+      - openstack/neutron-fwaas
+      - openstack/neutron-lbaas
+      - openstack/neutron-vpnaas
+      - openstack/nova
+      - openstack/octavia
+      - openstack/os-apply-config
+      - openstack/os-brick
+      - openstack/os-client-config
+      - openstack/os-collect-config
+      - openstack/os-net-config
+      - openstack/os-refresh-config
+      - openstack/osc-lib
+      - openstack/oslo.cache
+      - openstack/oslo.concurrency
+      - openstack/oslo.config
+      - openstack/oslo.context
+      - openstack/oslo.db
+      - openstack/oslo.i18n
+      - openstack/oslo.log
+      - openstack/oslo.messaging
+      - openstack/oslo.middleware
+      - openstack/oslo.policy
+      - openstack/oslo.reports
+      - openstack/oslo.rootwrap
+      - openstack/oslo.serialization
+      - openstack/oslo.service
+      - openstack/oslo.utils
+      - openstack/oslo.versionedobjects
+      - openstack/oslo.vmware
+      - openstack/pycadf
+      - openstack/python-cinderclient
+      - openstack/python-glanceclient
+      - openstack/python-heatclient
+      - openstack/python-ironicclient
+      - openstack/python-keystoneclient
+      - openstack/python-manilaclient
+      - openstack/python-neutronclient
+      - openstack/python-novaclient
+      - openstack/python-openstackclient
+      - openstack/python-saharaclient
+      - openstack/python-swiftclient
+      - openstack/python-troveclient
+      - openstack/python-zaqarclient
+      - openstack/requirements
+      - openstack/sahara
+      - openstack/sahara-dashboard
+      - openstack/stevedore
+      - openstack/swift
+      - openstack/taskflow
+      - openstack/tempest
+      - openstack/tooz
+      - openstack/tripleo-heat-templates
+      - openstack/tripleo-image-elements
+      - openstack/trove
+      - openstack/trove-dashboard
+      - openstack/zaqar
+
+- job:
+    name: pbr-installation-devstack
+    parent: pbr-installation-devstack-base
+    run: playbooks/legacy/pbr-installation-devstack/run.yaml
+    post-run: playbooks/legacy/pbr-installation-devstack/post.yaml
+
+- job:
+    name: pbr-installation-upstream-devstack
+    parent: pbr-installation-devstack-base
+    run: playbooks/legacy/pbr-installation-upstream-devstack/run.yaml
+    post-run: playbooks/legacy/pbr-installation-upstream-devstack/post.yaml
+
 - project:
-    check:
-      jobs:
-        - openstack-tox-lower-constraints
-        - legacy-pbr-installation-dsvm
-        - legacy-pbr-installation-dsvm-upstream
-    gate:
-      jobs:
-        - openstack-tox-lower-constraints
-        - legacy-pbr-installation-dsvm
     templates:
       - openstack-python-jobs
       - openstack-python35-jobs
+      - openstack-python36-jobs
       - lib-forward-testing
       - lib-forward-testing-python3
       - openstack-pypy-jobs-nonvoting
       - publish-openstack-docs-pti
       - periodic-stable-jobs
-      - openstack-python36-jobs
+    check:
+      jobs:
+        - openstack-tox-lower-constraints
+        - pbr-installation-devstack
+        - pbr-installation-upstream-devstack
+    gate:
+      jobs:
+        - openstack-tox-lower-constraints
+        - pbr-installation-devstack
+        - pbr-installation-upstream-devstack
     post:
       jobs:
         - openstack-tox-cover
diff --git a/playbooks/legacy/pbr-installation-devstack/post.yaml b/playbooks/legacy/pbr-installation-devstack/post.yaml
new file mode 100644
index 0000000..e07f551
--- /dev/null
+++ b/playbooks/legacy/pbr-installation-devstack/post.yaml
@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/legacy/pbr-installation-devstack/run.yaml b/playbooks/legacy/pbr-installation-devstack/run.yaml
new file mode 100644
index 0000000..9b48a34
--- /dev/null
+++ b/playbooks/legacy/pbr-installation-devstack/run.yaml
@@ -0,0 +1,137 @@
+- hosts: all
+  name: Autoconverted job legacy-pbr-installation-dsvm from old job gate-pbr-installation-dsvm-ubuntu-xenial
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          # Define the entire projects list here so that what we
+          # test requirements against is independent of what d-g
+          # thinks is relevant.
+          export PROJECTS="openstack-infra/devstack-gate $PROJECTS"
+          export PROJECTS="openstack-dev/devstack $PROJECTS"
+          export PROJECTS="openstack-dev/grenade $PROJECTS"
+          export PROJECTS="openstack-dev/pbr $PROJECTS"
+          export PROJECTS="openstack-infra/tripleo-ci $PROJECTS"
+          export PROJECTS="openstack/aodh $PROJECTS"
+          export PROJECTS="openstack/automaton $PROJECTS"
+          export PROJECTS="openstack/ceilometer $PROJECTS"
+          export PROJECTS="openstack/ceilometermiddleware $PROJECTS"
+          export PROJECTS="openstack/cinder $PROJECTS"
+          export PROJECTS="openstack/cliff $PROJECTS"
+          export PROJECTS="openstack/debtcollector $PROJECTS"
+          export PROJECTS="openstack/dib-utils $PROJECTS"
+          export PROJECTS="openstack/diskimage-builder $PROJECTS"
+          export PROJECTS="openstack/django_openstack_auth $PROJECTS"
+          export PROJECTS="openstack/futurist $PROJECTS"
+          export PROJECTS="openstack/glance $PROJECTS"
+          export PROJECTS="openstack/glance_store $PROJECTS"
+          export PROJECTS="openstack/heat $PROJECTS"
+          export PROJECTS="openstack/heat-cfntools $PROJECTS"
+          export PROJECTS="openstack/heat-templates $PROJECTS"
+          export PROJECTS="openstack/horizon $PROJECTS"
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/keystone $PROJECTS"
+          export PROJECTS="openstack/keystoneauth $PROJECTS"
+          export PROJECTS="openstack/keystonemiddleware $PROJECTS"
+          export PROJECTS="openstack/manila $PROJECTS"
+          export PROJECTS="openstack/manila-ui $PROJECTS"
+          export PROJECTS="openstack/zaqar $PROJECTS"
+          export PROJECTS="openstack/neutron $PROJECTS"
+          export PROJECTS="openstack/neutron-fwaas $PROJECTS"
+          export PROJECTS="openstack/neutron-lbaas $PROJECTS"
+          export PROJECTS="openstack/octavia $PROJECTS"
+          export PROJECTS="openstack/neutron-vpnaas $PROJECTS"
+          export PROJECTS="openstack/nova $PROJECTS"
+          export PROJECTS="openstack/os-apply-config $PROJECTS"
+          export PROJECTS="openstack/os-brick $PROJECTS"
+          export PROJECTS="openstack/os-client-config $PROJECTS"
+          export PROJECTS="openstack/os-collect-config $PROJECTS"
+          export PROJECTS="openstack/os-net-config $PROJECTS"
+          export PROJECTS="openstack/os-refresh-config $PROJECTS"
+          export PROJECTS="openstack/osc-lib $PROJECTS"
+          export PROJECTS="openstack/oslo.cache $PROJECTS"
+          export PROJECTS="openstack/oslo.concurrency $PROJECTS"
+          export PROJECTS="openstack/oslo.config $PROJECTS"
+          export PROJECTS="openstack/oslo.context $PROJECTS"
+          export PROJECTS="openstack/oslo.db $PROJECTS"
+          export PROJECTS="openstack/oslo.i18n $PROJECTS"
+          export PROJECTS="openstack/oslo.log $PROJECTS"
+          export PROJECTS="openstack/oslo.messaging $PROJECTS"
+          export PROJECTS="openstack/oslo.middleware $PROJECTS"
+          export PROJECTS="openstack/oslo.policy $PROJECTS"
+          export PROJECTS="openstack/oslo.reports $PROJECTS"
+          export PROJECTS="openstack/oslo.rootwrap $PROJECTS"
+          export PROJECTS="openstack/oslo.utils $PROJECTS"
+          export PROJECTS="openstack/oslo.serialization $PROJECTS"
+          export PROJECTS="openstack/oslo.service $PROJECTS"
+          export PROJECTS="openstack/oslo.versionedobjects $PROJECTS"
+          export PROJECTS="openstack/oslo.vmware $PROJECTS"
+          export PROJECTS="openstack/pycadf $PROJECTS"
+          export PROJECTS="openstack/python-cinderclient $PROJECTS"
+          export PROJECTS="openstack/python-glanceclient $PROJECTS"
+          export PROJECTS="openstack/python-heatclient $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/python-keystoneclient $PROJECTS"
+          export PROJECTS="openstack/python-manilaclient $PROJECTS"
+          export PROJECTS="openstack/python-zaqarclient $PROJECTS"
+          export PROJECTS="openstack/python-neutronclient $PROJECTS"
+          export PROJECTS="openstack/python-novaclient $PROJECTS"
+          export PROJECTS="openstack/python-openstackclient $PROJECTS"
+          export PROJECTS="openstack/python-saharaclient $PROJECTS"
+          export PROJECTS="openstack/python-swiftclient $PROJECTS"
+          export PROJECTS="openstack/python-troveclient $PROJECTS"
+          export PROJECTS="openstack/requirements $PROJECTS"
+          export PROJECTS="openstack/sahara $PROJECTS"
+          export PROJECTS="openstack/sahara-dashboard $PROJECTS"
+          export PROJECTS="openstack/stevedore $PROJECTS"
+          export PROJECTS="openstack/swift $PROJECTS"
+          export PROJECTS="openstack/taskflow $PROJECTS"
+          export PROJECTS="openstack/tempest $PROJECTS"
+          export PROJECTS="openstack/tooz $PROJECTS"
+          export PROJECTS="openstack/tripleo-heat-templates $PROJECTS"
+          export PROJECTS="openstack/tripleo-image-elements $PROJECTS"
+          export PROJECTS="openstack/trove $PROJECTS"
+          export PROJECTS="openstack/trove-dashboard $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=0
+          export DEVSTACK_GATE_EXERCISES=0
+          export DEVSTACK_GATE_REQS_INTEGRATION=1
+          export PIP_PROCESS_DEPENDENCY_LINKS=1
+          if [ "" = "-upstream" ]; then
+              export PBR_PIP_VERSION=git+https://github.com/pypa/pip.git#egg=pip
+          fi
+          function gate_hook {
+              set -x
+              bash -xe $BASE/new/pbr/tools/integration.sh $(cat $BASE/new/requirements/projects.txt)
+          }
+          export -f gate_hook
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/legacy/pbr-installation-upstream-devstack/post.yaml b/playbooks/legacy/pbr-installation-upstream-devstack/post.yaml
new file mode 100644
index 0000000..e07f551
--- /dev/null
+++ b/playbooks/legacy/pbr-installation-upstream-devstack/post.yaml
@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml
new file mode 100644
index 0000000..a8fb806
--- /dev/null
+++ b/playbooks/legacy/pbr-installation-upstream-devstack/run.yaml
@@ -0,0 +1,137 @@
+- hosts: all
+  name: Autoconverted job legacy-pbr-installation-dsvm-upstream from old job gate-pbr-installation-dsvm-upstream-ubuntu-xenial
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          # Define the entire projects list here so that what we
+          # test requirements against is independent of what d-g
+          # thinks is relevant.
+          export PROJECTS="openstack-infra/devstack-gate $PROJECTS"
+          export PROJECTS="openstack-dev/devstack $PROJECTS"
+          export PROJECTS="openstack-dev/grenade $PROJECTS"
+          export PROJECTS="openstack-dev/pbr $PROJECTS"
+          export PROJECTS="openstack-infra/tripleo-ci $PROJECTS"
+          export PROJECTS="openstack/aodh $PROJECTS"
+          export PROJECTS="openstack/automaton $PROJECTS"
+          export PROJECTS="openstack/ceilometer $PROJECTS"
+          export PROJECTS="openstack/ceilometermiddleware $PROJECTS"
+          export PROJECTS="openstack/cinder $PROJECTS"
+          export PROJECTS="openstack/cliff $PROJECTS"
+          export PROJECTS="openstack/debtcollector $PROJECTS"
+          export PROJECTS="openstack/dib-utils $PROJECTS"
+          export PROJECTS="openstack/diskimage-builder $PROJECTS"
+          export PROJECTS="openstack/django_openstack_auth $PROJECTS"
+          export PROJECTS="openstack/futurist $PROJECTS"
+          export PROJECTS="openstack/glance $PROJECTS"
+          export PROJECTS="openstack/glance_store $PROJECTS"
+          export PROJECTS="openstack/heat $PROJECTS"
+          export PROJECTS="openstack/heat-cfntools $PROJECTS"
+          export PROJECTS="openstack/heat-templates $PROJECTS"
+          export PROJECTS="openstack/horizon $PROJECTS"
+          export PROJECTS="openstack/ironic $PROJECTS"
+          export PROJECTS="openstack/ironic-lib $PROJECTS"
+          export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+          export PROJECTS="openstack/keystone $PROJECTS"
+          export PROJECTS="openstack/keystoneauth $PROJECTS"
+          export PROJECTS="openstack/keystonemiddleware $PROJECTS"
+          export PROJECTS="openstack/manila $PROJECTS"
+          export PROJECTS="openstack/manila-ui $PROJECTS"
+          export PROJECTS="openstack/zaqar $PROJECTS"
+          export PROJECTS="openstack/neutron $PROJECTS"
+          export PROJECTS="openstack/neutron-fwaas $PROJECTS"
+          export PROJECTS="openstack/neutron-lbaas $PROJECTS"
+          export PROJECTS="openstack/octavia $PROJECTS"
+          export PROJECTS="openstack/neutron-vpnaas $PROJECTS"
+          export PROJECTS="openstack/nova $PROJECTS"
+          export PROJECTS="openstack/os-apply-config $PROJECTS"
+          export PROJECTS="openstack/os-brick $PROJECTS"
+          export PROJECTS="openstack/os-client-config $PROJECTS"
+          export PROJECTS="openstack/os-collect-config $PROJECTS"
+          export PROJECTS="openstack/os-net-config $PROJECTS"
+          export PROJECTS="openstack/os-refresh-config $PROJECTS"
+          export PROJECTS="openstack/osc-lib $PROJECTS"
+          export PROJECTS="openstack/oslo.cache $PROJECTS"
+          export PROJECTS="openstack/oslo.concurrency $PROJECTS"
+          export PROJECTS="openstack/oslo.config $PROJECTS"
+          export PROJECTS="openstack/oslo.context $PROJECTS"
+          export PROJECTS="openstack/oslo.db $PROJECTS"
+          export PROJECTS="openstack/oslo.i18n $PROJECTS"
+          export PROJECTS="openstack/oslo.log $PROJECTS"
+          export PROJECTS="openstack/oslo.messaging $PROJECTS"
+          export PROJECTS="openstack/oslo.middleware $PROJECTS"
+          export PROJECTS="openstack/oslo.policy $PROJECTS"
+          export PROJECTS="openstack/oslo.reports $PROJECTS"
+          export PROJECTS="openstack/oslo.rootwrap $PROJECTS"
+          export PROJECTS="openstack/oslo.utils $PROJECTS"
+          export PROJECTS="openstack/oslo.serialization $PROJECTS"
+          export PROJECTS="openstack/oslo.service $PROJECTS"
+          export PROJECTS="openstack/oslo.versionedobjects $PROJECTS"
+          export PROJECTS="openstack/oslo.vmware $PROJECTS"
+          export PROJECTS="openstack/pycadf $PROJECTS"
+          export PROJECTS="openstack/python-cinderclient $PROJECTS"
+          export PROJECTS="openstack/python-glanceclient $PROJECTS"
+          export PROJECTS="openstack/python-heatclient $PROJECTS"
+          export PROJECTS="openstack/python-ironicclient $PROJECTS"
+          export PROJECTS="openstack/python-keystoneclient $PROJECTS"
+          export PROJECTS="openstack/python-manilaclient $PROJECTS"
+          export PROJECTS="openstack/python-zaqarclient $PROJECTS"
+          export PROJECTS="openstack/python-neutronclient $PROJECTS"
+          export PROJECTS="openstack/python-novaclient $PROJECTS"
+          export PROJECTS="openstack/python-openstackclient $PROJECTS"
+          export PROJECTS="openstack/python-saharaclient $PROJECTS"
+          export PROJECTS="openstack/python-swiftclient $PROJECTS"
+          export PROJECTS="openstack/python-troveclient $PROJECTS"
+          export PROJECTS="openstack/requirements $PROJECTS"
+          export PROJECTS="openstack/sahara $PROJECTS"
+          export PROJECTS="openstack/sahara-dashboard $PROJECTS"
+          export PROJECTS="openstack/stevedore $PROJECTS"
+          export PROJECTS="openstack/swift $PROJECTS"
+          export PROJECTS="openstack/taskflow $PROJECTS"
+          export PROJECTS="openstack/tempest $PROJECTS"
+          export PROJECTS="openstack/tooz $PROJECTS"
+          export PROJECTS="openstack/tripleo-heat-templates $PROJECTS"
+          export PROJECTS="openstack/tripleo-image-elements $PROJECTS"
+          export PROJECTS="openstack/trove $PROJECTS"
+          export PROJECTS="openstack/trove-dashboard $PROJECTS"
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=0
+          export DEVSTACK_GATE_EXERCISES=0
+          export DEVSTACK_GATE_REQS_INTEGRATION=1
+          export PIP_PROCESS_DEPENDENCY_LINKS=1
+          if [ "-upstream" = "-upstream" ]; then
+              export PBR_PIP_VERSION=git+https://github.com/pypa/pip.git#egg=pip
+          fi
+          function gate_hook {
+              set -x
+              bash -xe $BASE/new/pbr/tools/integration.sh $(cat $BASE/new/requirements/projects.txt)
+          }
+          export -f gate_hook
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'