summaryrefslogtreecommitdiff
path: root/integration/scripts
diff options
context:
space:
mode:
authorAmrith Kumar <amrith@tesora.com>2016-10-09 07:13:29 -0400
committerAmrith Kumar <amrith@tesora.com>2016-10-09 07:14:25 -0400
commitb120be1772f2416730ea1066782bf0813ca2dd1a (patch)
treee86dddeed295fdb97489f66170d6965a58474cd9 /integration/scripts
parent473d360b906dba80f0f914e88b461221c1bfd5f3 (diff)
downloadtrove-b120be1772f2416730ea1066782bf0813ca2dd1a.tar.gz
Merge trove-integration into trove
This commit will merge into trove, the trove-integration tree as of commit 9f92ca853f8aa2f72921e54682c918941a8f0919. This is in preparation for making trove-integration go away. In addition, it supresses any consideration of the integration directory in the trove tox tests as it is understandably a small pile of pooh and in need of much cleanup. Change-Id: Ib7f2655c4c5ed86b5454708c04371ee55e37ec2d Partially-Implements-Blueprint: eliminate-trove-integration-and-redstack
Diffstat (limited to 'integration/scripts')
-rw-r--r--integration/scripts/conf.json.example12
-rw-r--r--integration/scripts/conf/cassandra.conf6
-rw-r--r--integration/scripts/conf/couchbase.conf6
-rw-r--r--integration/scripts/conf/couchdb.conf6
-rw-r--r--integration/scripts/conf/db2.conf6
-rw-r--r--integration/scripts/conf/mariadb.conf6
-rw-r--r--integration/scripts/conf/mongodb.conf6
-rw-r--r--integration/scripts/conf/mysql.conf6
-rw-r--r--integration/scripts/conf/percona.conf6
-rw-r--r--integration/scripts/conf/postgresql.conf6
-rw-r--r--integration/scripts/conf/pxc.conf6
-rw-r--r--integration/scripts/conf/redis.conf6
-rw-r--r--integration/scripts/conf/test_begin.conf101
-rw-r--r--integration/scripts/conf/test_end.conf2
-rw-r--r--integration/scripts/conf/vertica.conf6
-rwxr-xr-xintegration/scripts/create_vm84
-rw-r--r--integration/scripts/files/elements/apt-conf-dir/README.rst16
-rwxr-xr-xintegration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd21
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep48
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd21
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key31
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep30
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/install.d/20-etc8
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/install.d/50-user17
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/install.d/62-ssh-key29
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding5
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers15
-rwxr-xr-xintegration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update9
-rw-r--r--integration/scripts/files/elements/fedora-mariadb/README.md3
-rwxr-xr-xintegration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb9
-rwxr-xr-xintegration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr10
-rw-r--r--integration/scripts/files/elements/fedora-mongodb/README.md1
-rwxr-xr-xintegration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb24
-rwxr-xr-xintegration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep9
-rw-r--r--integration/scripts/files/elements/fedora-mysql/README.md3
-rwxr-xr-xintegration/scripts/files/elements/fedora-mysql/install.d/10-mysql16
-rwxr-xr-xintegration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup10
-rw-r--r--integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service6
-rwxr-xr-xintegration/scripts/files/elements/fedora-percona/install.d/05-percona-server17
-rwxr-xr-xintegration/scripts/files/elements/fedora-percona/install.d/10-mysql16
-rwxr-xr-xintegration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql83
-rw-r--r--integration/scripts/files/elements/fedora-redis/README.md1
-rwxr-xr-xintegration/scripts/files/elements/fedora-redis/install.d/10-redis9
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra25
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb19
-rw-r--r--integration/scripts/files/elements/ubuntu-db2/README.md36
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs25
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-db2/install.d/10-db252
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep48
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart21
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key31
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps10
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep31
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/20-etc8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/50-user18
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key28
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/98-ssh8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt11
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding4
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp10
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers15
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update9
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs117
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools7
-rw-r--r--integration/scripts/files/elements/ubuntu-mariadb/README.md3
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb34
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key40
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local11
-rw-r--r--integration/scripts/files/elements/ubuntu-mongodb/README.md1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp42
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep9
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf26
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init46
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init30
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key14
-rw-r--r--integration/scripts/files/elements/ubuntu-mysql/README.md3
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql23
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key40
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local11
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-percona/install.d/30-mysql17
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key42
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local11
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql79
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo12
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql14
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key42
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local11
-rw-r--r--integration/scripts/files/elements/ubuntu-redis/README.md1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-redis/install.d/10-redis53
-rw-r--r--integration/scripts/files/elements/ubuntu-vertica/README.md1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb14
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica54
-rw-r--r--integration/scripts/files/keys/authorized_keys1
-rw-r--r--integration/scripts/files/keys/id_rsa27
-rw-r--r--integration/scripts/files/keys/id_rsa.pub1
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-default.txt31
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-juno.txt19
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-kilo.txt24
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-liberty.txt27
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-mitaka.txt28
-rw-r--r--integration/scripts/files/requirements/fedora-requirements-newton.txt31
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-default.txt30
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-juno.txt19
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-kilo.txt24
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-liberty.txt26
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt27
-rw-r--r--integration/scripts/files/requirements/ubuntu-requirements-newton.txt30
-rw-r--r--integration/scripts/files/trove-guest.systemd.conf32
-rw-r--r--integration/scripts/files/trove-guest.upstart.conf40
-rw-r--r--integration/scripts/functions324
-rw-r--r--integration/scripts/functions_qemu174
-rw-r--r--integration/scripts/image-projects-list3
-rw-r--r--integration/scripts/local.conf.d/ceilometer_cinder.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/ceilometer_nova.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/ceilometer_services.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/sample.rc42
-rw-r--r--integration/scripts/local.conf.d/trove_services.conf.rc24
-rw-r--r--integration/scripts/local.conf.d/use_kvm.rc4
-rw-r--r--integration/scripts/local.conf.d/use_uuid_token.rc3
-rw-r--r--integration/scripts/local.conf.d/using_vagrant.rc9
-rw-r--r--integration/scripts/local.conf.rc37
-rw-r--r--integration/scripts/localrc.rc100
-rw-r--r--integration/scripts/projects-list12
-rwxr-xr-xintegration/scripts/redstack1433
-rw-r--r--integration/scripts/redstack.rc120
-rw-r--r--integration/scripts/reviews.rc5
128 files changed, 4687 insertions, 0 deletions
diff --git a/integration/scripts/conf.json.example b/integration/scripts/conf.json.example
new file mode 100644
index 00000000..ee6bc7bf
--- /dev/null
+++ b/integration/scripts/conf.json.example
@@ -0,0 +1,12 @@
+{
+ "devstack":null,
+ "glance":null,
+ "horizon":null,
+ "keystone":null,
+ "nova":null,
+ "python_openstackclient":null,
+ "python_novaclient":null,
+ "trove":null,
+ "python_troveclient":null,
+ "tempest":null
+}
diff --git a/integration/scripts/conf/cassandra.conf b/integration/scripts/conf/cassandra.conf
new file mode 100644
index 00000000..5a2f0acc
--- /dev/null
+++ b/integration/scripts/conf/cassandra.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.large-5",
+ "instance_bigger_flavor_name": "test.large-5.resize",
+ "instance_eph_flavor_name": "test.eph.large-5",
+ "instance_bigger_eph_flavor_name": "test.eph.large-5.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/couchbase.conf b/integration/scripts/conf/couchbase.conf
new file mode 100644
index 00000000..5a2f0acc
--- /dev/null
+++ b/integration/scripts/conf/couchbase.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.large-5",
+ "instance_bigger_flavor_name": "test.large-5.resize",
+ "instance_eph_flavor_name": "test.eph.large-5",
+ "instance_bigger_eph_flavor_name": "test.eph.large-5.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/couchdb.conf b/integration/scripts/conf/couchdb.conf
new file mode 100644
index 00000000..65bd382c
--- /dev/null
+++ b/integration/scripts/conf/couchdb.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.tiny-3",
+ "instance_bigger_flavor_name": "test.tiny-3.resize",
+ "instance_eph_flavor_name": "test.eph.tiny-3",
+ "instance_bigger_eph_flavor_name": "test.eph.tiny-3.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/db2.conf b/integration/scripts/conf/db2.conf
new file mode 100644
index 00000000..a842ea02
--- /dev/null
+++ b/integration/scripts/conf/db2.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.small-5",
+ "instance_bigger_flavor_name": "test.small-5.resize",
+ "instance_eph_flavor_name": "test.eph.small-5",
+ "instance_bigger_eph_flavor_name": "test.eph.small-5.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/mariadb.conf b/integration/scripts/conf/mariadb.conf
new file mode 100644
index 00000000..0e29d114
--- /dev/null
+++ b/integration/scripts/conf/mariadb.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.small-4",
+ "instance_bigger_flavor_name": "test.small-4.resize",
+ "instance_eph_flavor_name": "test.eph.small-4",
+ "instance_bigger_eph_flavor_name": "test.eph.small-4.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/mongodb.conf b/integration/scripts/conf/mongodb.conf
new file mode 100644
index 00000000..0e0aae02
--- /dev/null
+++ b/integration/scripts/conf/mongodb.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.large-5",
+ "instance_bigger_flavor_name": "test.large-5.resize",
+ "instance_eph_flavor_name": "test.eph.large-5",
+ "instance_bigger_eph_flavor_name": "test.eph.large-5.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 5,
diff --git a/integration/scripts/conf/mysql.conf b/integration/scripts/conf/mysql.conf
new file mode 100644
index 00000000..eac3c579
--- /dev/null
+++ b/integration/scripts/conf/mysql.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.small-3",
+ "instance_bigger_flavor_name": "test.small-3.resize",
+ "instance_eph_flavor_name": "test.eph.small-3",
+ "instance_bigger_eph_flavor_name": "test.eph.small-3.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/percona.conf b/integration/scripts/conf/percona.conf
new file mode 100644
index 00000000..eac3c579
--- /dev/null
+++ b/integration/scripts/conf/percona.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.small-3",
+ "instance_bigger_flavor_name": "test.small-3.resize",
+ "instance_eph_flavor_name": "test.eph.small-3",
+ "instance_bigger_eph_flavor_name": "test.eph.small-3.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/postgresql.conf b/integration/scripts/conf/postgresql.conf
new file mode 100644
index 00000000..8033f582
--- /dev/null
+++ b/integration/scripts/conf/postgresql.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.medium-4",
+ "instance_bigger_flavor_name": "test.medium-4.resize",
+ "instance_eph_flavor_name": "test.eph.medium-4",
+ "instance_bigger_eph_flavor_name": "test.eph.medium-4.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/pxc.conf b/integration/scripts/conf/pxc.conf
new file mode 100644
index 00000000..eac3c579
--- /dev/null
+++ b/integration/scripts/conf/pxc.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.small-3",
+ "instance_bigger_flavor_name": "test.small-3.resize",
+ "instance_eph_flavor_name": "test.eph.small-3",
+ "instance_bigger_eph_flavor_name": "test.eph.small-3.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/redis.conf b/integration/scripts/conf/redis.conf
new file mode 100644
index 00000000..65bd382c
--- /dev/null
+++ b/integration/scripts/conf/redis.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.tiny-3",
+ "instance_bigger_flavor_name": "test.tiny-3.resize",
+ "instance_eph_flavor_name": "test.eph.tiny-3",
+ "instance_bigger_eph_flavor_name": "test.eph.tiny-3.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 1,
diff --git a/integration/scripts/conf/test_begin.conf b/integration/scripts/conf/test_begin.conf
new file mode 100644
index 00000000..eff96dea
--- /dev/null
+++ b/integration/scripts/conf/test_begin.conf
@@ -0,0 +1,101 @@
+{
+ "dbaas_url":"http://%service_host%:8779/v1.0",
+ "version_url":"http://%service_host%:8779",
+ "trove_auth_url":"http://%service_host%:35357/v2.0/tokens",
+ "trove_client_insecure":false,
+ "auth_strategy":null,
+ "trove_client_region_name": "%region_name%",
+
+ "nova_client": {
+ "url":"http://%service_host%:8774/v1.1",
+ "auth_url":"http://%service_host%:35357/v2.0",
+ "nova_service_type":"compute",
+ "volume_service_type":"volume"
+ },
+
+ "flavors": null,
+
+ "white_box":false,
+ "start_services": true,
+ "test_mgmt":false,
+ "use_local_ovz":false,
+ "use_venv":false,
+ "glance_code_root":"%glance_path%",
+ "glance_api_conf":"/vagrant/conf/glance-api.conf",
+ "glance_reg_conf":"/vagrant/conf/glance-reg.conf",
+ "glance_images_directory": "/glance_images",
+ "glance_image": "debian-squeeze-x86_64-openvz.tar.gz",
+ "report_directory":"%report_directory%",
+ "usr_bin_dir":"%bin_path%",
+ "nova_code_root":"%nova_path%",
+ "nova_conf":"/home/vagrant/nova.conf",
+ "keystone_code_root":"%keystone_path%",
+ "keystone_conf":"/etc/keystone/keystone.conf",
+ "keystone_use_combined":true,
+ "trove_code_root":"%trove_path%",
+ "trove_conf":"/tmp/trove.conf",
+ "trove_version":"v1.0",
+ "trove_api_updated":"2012-08-01T00:00:00Z",
+ "trove_max_accepted_volume_size": 1000,
+ "trove_max_instances_per_user": 55,
+ "trove_max_volumes_per_user": 100,
+ "use_reaper":false,
+ "users": [
+ { "auth_user":"admin",
+ "auth_key":"%admin_password%",
+ "tenant":"admin",
+ "tenant_id":"%admin_tenant_id%",
+ "requirements": {
+ "is_admin":true,
+ "services": ["trove"]
+ }
+ },
+ { "auth_user":"alt_demo",
+ "auth_key":"%admin_password%",
+ "tenant":"alt_demo",
+ "tenant_id":"%alt_demo_tenant_id%",
+ "requirements": {
+ "is_admin":false,
+ "services": ["trove"]
+ }
+ },
+ { "auth_user":"admin_alt_demo",
+ "auth_key":"%admin_password%",
+ "tenant":"alt_demo",
+ "tenant_id":"%alt_demo_tenant_id%",
+ "requirements": {
+ "is_admin":true,
+ "services": ["swift"]
+ }
+ },
+ { "auth_user":"demo",
+ "auth_key":"%admin_password%",
+ "tenant":"demo",
+ "tenant_id":"%demo_tenant_id%",
+ "requirements": {
+ "is_admin":false,
+ "services": ["nova", "trove"]
+ }
+ }
+ ],
+ "root_removed_from_instance_api": true,
+ "root_timestamp_disabled": false,
+ "openvz_disabled": true,
+ "management_api_disabled": true,
+ "dbaas_image": 1,
+ "dns_driver":"trove.dns.rsdns.driver.RsDnsDriver",
+ "dns_instance_entry_factory":"trove.dns.rsdns.driver.RsDnsInstanceEntryFactory",
+ "trove_dns_support": false,
+ "databases_page_size": 20,
+ "instances_page_size": 20,
+ "users_page_size": 20,
+ "rabbit_runs_locally": true,
+ "dbaas_datastore": "%datastore_type%",
+ "dbaas_datastore_version": "%datastore_version%",
+ "neutron_enabled": %neutron_enabled%,
+ "shared_network": "%shared_network%",
+ "shared_network_subnet": "%shared_network_subnet%",
+ "instance_fault_1_flavor_name": "test.fault_1-1",
+ "instance_fault_1_eph_flavor_name": "test.eph.fault_1-1",
+ "instance_fault_2_flavor_name": "test.fault_2-5",
+ "instance_fault_2_eph_flavor_name": "test.eph.fault_2-5",
diff --git a/integration/scripts/conf/test_end.conf b/integration/scripts/conf/test_end.conf
new file mode 100644
index 00000000..10fc2de1
--- /dev/null
+++ b/integration/scripts/conf/test_end.conf
@@ -0,0 +1,2 @@
+ "sentinel": null
+}
diff --git a/integration/scripts/conf/vertica.conf b/integration/scripts/conf/vertica.conf
new file mode 100644
index 00000000..242d7fa1
--- /dev/null
+++ b/integration/scripts/conf/vertica.conf
@@ -0,0 +1,6 @@
+ "instance_flavor_name": "test.large-10",
+ "instance_bigger_flavor_name": "test.large-10.resize",
+ "instance_eph_flavor_name": "test.eph.large-10",
+ "instance_bigger_eph_flavor_name": "test.eph.large-10.resize",
+ "trove_volume_support": true,
+ "trove_volume_size": 5,
diff --git a/integration/scripts/create_vm b/integration/scripts/create_vm
new file mode 100755
index 00000000..7b1b3767
--- /dev/null
+++ b/integration/scripts/create_vm
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+"""
+Sets up a VM with hardcoded paths to multiple source trees.
+Creates a script for use with VMWare or a Vagrantfile.
+
+Uses a configuration file (in JSON format) that stores the paths to checked-out
+copies of OpenStack projects on the host machine. If the path is None then
+it lets devstack download them.
+"""
+
+import json
+
+
+class Config(object):
+ """
+ Very simple configuration file thats just some JSON.
+ """
+
+ vm_paths = {
+ 'devstack':"/devstack",
+ 'glance': '/opt/stack/glance',
+ 'horizon': '/opt/stack/horizon',
+ 'keystone': "/opt/stack/keystone",
+ 'nova': "/opt/stack/nova",
+ 'python_openstackclient': "/opt/stack/python-openstackclient",
+ 'python_novaclient': "/opt/stack/python-novaclient",
+ 'trove':"/opt/stack/trove",
+ 'python_troveclient':"/opt/stack/python-troveclient",
+ 'tempest':"/opt/stack/tempest"
+ }
+
+ def __init__(self, **kwargs):
+ for name in Config.vm_paths.keys():
+ if name not in kwargs:
+ raise RuntimeError('Missing configuration value "%s".' % name)
+ value = kwargs[name]
+ if value is not None and type(value) is not str \
+ and type(value) is not unicode:
+ raise RuntimeError('Path "%s" must be a string or None but is '
+ 'of type %s.' % (name, type(value)))
+ setattr(self, name, kwargs[name])
+ self.vagrant_path = kwargs.get("vagrant_path", "Vagrantfile")
+
+ @staticmethod
+ def load(file_path):
+ file_contents = open(file_path, "r").read()
+ dict = json.loads(file_contents);
+ return Config(**dict)
+
+ def write_vagrant_file(self):
+ with open(self.vagrant_path, 'w') as file:
+ file.write("""
+Vagrant::Config.run do |global_config|
+ # Host config
+ global_config.vm.define :host do |config|
+
+ config.vm.network "33.33.44.11"
+
+ config.vm.box = "precise"
+ config.vm.host_name = "host"
+
+ config.ssh.timeout = 3600
+ config.vm.customize do |vm|
+ vm.memory_size = 2048
+ end
+
+ config.vm.share_folder "integration", "/integration", "../"
+
+""")
+ for key in Config.vm_paths.keys():
+ local_path = getattr(self, key)
+ vm_path = Config.vm_paths[key]
+ if local_path is not None:
+ file.write('\tconfig.vm.share_folder "%s", "%s", "%s" \n'
+ % (key, vm_path, local_path))
+ file.write("""
+
+ end
+end
+ """)
+
+if __name__=="__main__":
+ conf = Config.load("conf.json")
+ conf.write_vagrant_file()
diff --git a/integration/scripts/files/elements/apt-conf-dir/README.rst b/integration/scripts/files/elements/apt-conf-dir/README.rst
new file mode 100644
index 00000000..c94e00ea
--- /dev/null
+++ b/integration/scripts/files/elements/apt-conf-dir/README.rst
@@ -0,0 +1,16 @@
+============
+apt-conf-dir
+============
+
+This element overrides the default apt.conf.d directory for APT based systems.
+
+Environment Variables
+---------------------
+
+DIB_APT_CONF_DIR:
+ :Required: No
+ :Default: None
+ :Description: To override `DIB_APT_CONF_DIR`, set it to the path to your
+ apt.conf.d. The new apt.conf.d will take effect at build time
+ and run time.
+ :Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf``
diff --git a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd
new file mode 100755
index 00000000..e286d684
--- /dev/null
+++ b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd
@@ -0,0 +1,21 @@
+#!/bin/bash
+# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+# exit directly if DIB_APT_CONF_DIR is not defined properly
+if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then
+ echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in"
+ exit 0
+elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then
+ echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory."
+ echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR"
+ exit 1
+fi
+
+# copy the apt.conf to cloudimg
+sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt
diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep b/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep
new file mode 100755
index 00000000..97a5e438
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep
+
+source $_LIB/die
+
+BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default}
+ADD_BRANCH=$(basename ${BRANCH_OVERRIDE})
+REQUIREMENTS_FILE=${REDSTACK_SCRIPTS}/files/requirements/fedora-requirements-${ADD_BRANCH}.txt
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found"
+[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set"
+
+sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt
+
+# Grab the upper constraints file, but don't fail if we can't find it.
+# If we are running in the CI environment, $DEST will be set and stackrc
+# will use $DEST/requirements as the location for the requirements repo.
+# Use that as it will help us chain a job with something that is changing UC.
+
+UC_FILE=upper-constraints.txt
+
+if [ -f "${DEST}/requirements/${UC_FILE}" ]; then
+ echo "Found ${DEST}/requirements/${UC_FILE}, using that"
+ sudo -Hiu ${HOST_USERNAME} dd if="${DEST}/requirements/${UC_FILE}" \
+ of="${TMP_HOOKS_PATH}/${UC_FILE}"
+else
+ UC_DIR=$(pwd)
+ UC_BRANCH=${BRANCH_OVERRIDE}
+ if [ "${ADD_BRANCH}" == "default" ]; then
+ UC_BRANCH=master
+ fi
+
+ set +e
+ curl -o "${UC_DIR}/${UC_FILE}" \
+ https://git.openstack.org/cgit/openstack/requirements/plain/${UC_FILE}?h=${UC_BRANCH}
+ set -e
+
+ if [ -f "${UC_DIR}/${UC_FILE}" ]; then
+ sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE}
+ rm -f "${UC_DIR}/${UC_FILE}"
+ fi
+fi
diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd b/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd
new file mode 100755
index 00000000..960342e3
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
+# configured
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
+[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
+[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host"
+[ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir"
+[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
+
+sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key
new file mode 100755
index 00000000..63453a75
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where
+# they will be duplicated in the guest VM.
+# This process allows the host to log into the guest but more importantly the guest phones home to get the trove
+# source
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host"
+
+if [ `whoami` = "root" ]; then
+ die "This should not be run as root"
+fi
+
+# copy files over the "staging" area for the guest image (they'll later be put in the correct location by the guest user
+# not these keys should not be overridden otherwise a) you won't be able to ssh in and b) the guest won't be able to
+# rsync the files
+if [ -e ${SSH_DIR}/authorized_keys ]; then
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/authorized_keys of=${TMP_HOOKS_PATH}/ssh-authorized-keys
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
+else
+ die "SSH Authorized Keys file must exist along with pub and private key"
+fi
diff --git a/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep b/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep
new file mode 100755
index 00000000..98fb24ba
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu
+
+set -e
+set -o xtrace
+
+dnf install -y python-devel libxml2-devel libxslt-devel python-setuptools \
+ python-sqlalchemy python-lxml \
+ python-routes python-eventlet python-webob \
+ python-kombu python-paste-deploy python-paste python-netaddr \
+ python-httplib2 python-iso8601 python-pip python-mysql \
+ python-migrate python-anyjson gcc python-pexpect
+
+# pick up the requirements file left for us by
+# extra-data.d/15-reddwarf-dep
+
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+UPPER_CONSTRAINTS=
+if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then
+ UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt"
+fi
+
+pip install -q --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS}
+
+echo "diagnostic pip freeze output follows"
+pip freeze
+echo "diagnostic pip freeze output above"
diff --git a/integration/scripts/files/elements/fedora-guest/install.d/20-etc b/integration/scripts/files/elements/fedora-guest/install.d/20-etc
new file mode 100755
index 00000000..bec065ef
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/install.d/20-etc
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
+
+dd if=/tmp/in_target.d/trove-guest.service of=/usr/lib/systemd/system/trove-guest.service
+
+systemctl enable trove-guest.service
diff --git a/integration/scripts/files/elements/fedora-guest/install.d/50-user b/integration/scripts/files/elements/fedora-guest/install.d/50-user
new file mode 100755
index 00000000..a4b666bf
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/install.d/50-user
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist
+
+set -e
+set -o xtrace
+
+# Difference from apt, -G admin option
+if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
+ echo "Adding ${GUEST_USERNAME} user"
+ useradd -m ${GUEST_USERNAME} -s /bin/bash
+ passwd ${GUEST_USERNAME} <<_EOF_
+${GUEST_USERNAME}
+${GUEST_USERNAME}
+_EOF_
+fi \ No newline at end of file
diff --git a/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key b/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key
new file mode 100755
index 00000000..80c1e65c
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory
+
+set -e
+set -o xtrace
+
+SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+if [ -e "${TMP_HOOKS_DIR}/ssh-authorized-keys" ]; then
+ if [ ! -e ${SSH_DIR} ]; then
+ # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
+ mkdir ${SSH_DIR}
+ chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR}
+ fi
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/authorized_keys conv=notrunc if=${TMP_HOOKS_DIR}/ssh-authorized-keys
+ sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/authorized_keys
+ if [ ! -e "${SSH_DIR}/id_rsa" ]; then
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
+ # perms have to be right on this file for ssh to work
+ sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
+ fi
+else
+ echo "SSH Keys were not staged by host"
+ exit -1
+fi
diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding
new file mode 100755
index 00000000..4824cfcf
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+set -o xtrace
+sed -i -r 's/^\s*#(net\.ipv4\.ip_forward=1.*)/\1/' /etc/sysctl.conf
+echo 1 > /proc/sys/net/ipv4/ip_forward
diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers
new file mode 100755
index 00000000..0581fd2b
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+set -o xtrace
+
+# CONTEXT: HOST after IMAGE BUILD as SCRIPT USER
+# PURPOSE: add the guest user account to the /etc/sudoers files with NOPASSWD
+
+# Adds user to the sudoers file so they can do everything w/o a pass
+# Some binaries might be under /sbin or /usr/sbin, so make sure sudo will
+# see them by forcing PATH
+TEMPFILE=`mktemp`
+echo "${GUEST_USERNAME} ALL=(ALL) NOPASSWD:ALL" > $TEMPFILE
+chmod 0440 $TEMPFILE
+sudo chown root:root $TEMPFILE
+sudo mv $TEMPFILE /etc/sudoers.d/60_trove_guest
diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update b/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update
new file mode 100755
index 00000000..cd2992c1
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# CONTEXT: GUEST after packages installed
+# PURPOSE: do dnf update to save each instance having to do all the work
+
+set -e
+set -o xtrace
+
+dnf -y update
diff --git a/integration/scripts/files/elements/fedora-mariadb/README.md b/integration/scripts/files/elements/fedora-mariadb/README.md
new file mode 100644
index 00000000..757f00b8
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mariadb/README.md
@@ -0,0 +1,3 @@
+Sets up a MariaDB server install in the image.
+
+TODO: auto-tune settings based on host resources or metadata service.
diff --git a/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb b/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb
new file mode 100755
index 00000000..a5cc2c0e
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+dnf -y install mariadb-server percona-xtrabackup
diff --git a/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr b/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr
new file mode 100755
index 00000000..bcc55205
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup COPR Percona repository
+
+set -e
+set -o xtrace
+
+# install from Fedora repos
+dnf -y install percona-xtrabackup
diff --git a/integration/scripts/files/elements/fedora-mongodb/README.md b/integration/scripts/files/elements/fedora-mongodb/README.md
new file mode 100644
index 00000000..2518abf2
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mongodb/README.md
@@ -0,0 +1 @@
+Sets up a MongoDB install in the image. \ No newline at end of file
diff --git a/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb b/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb
new file mode 100755
index 00000000..3b53a18b
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+cat > "/etc/rc.local" << _EOF_
+# Make sure to disable Linux kernel feature transparent huge pages,
+# it will affect greatly both memory usage and latency in a negative way.
+# See: http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/
+if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then
+ echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
+fi
+if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then
+ echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
+fi
+
+exit \$?
+
+_EOF_
+
+dnf -y install mongodb-server
diff --git a/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep b/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep
new file mode 100755
index 00000000..a3e2633b
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu
+
+set -e
+set -o xtrace
+
+pip install pymongo>=3.0.2,!=3.1
diff --git a/integration/scripts/files/elements/fedora-mysql/README.md b/integration/scripts/files/elements/fedora-mysql/README.md
new file mode 100644
index 00000000..39a6ab8c
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mysql/README.md
@@ -0,0 +1,3 @@
+Sets up a MySQL server install in the image.
+
+TODO: auto-tune settings based on host resources or metadata service.
diff --git a/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql b/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql
new file mode 100755
index 00000000..ff7efa36
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+dnf -y install https://repo.mysql.com/mysql-community-release-fc22.rpm
+dnf -y install mysql-community-server
+
+# move the config dir for now but leave /etc/my.cnf alone
+# ln -s creates problems for the systemd script
+mkdir /etc/mysql
+mv /etc/my.cnf.d /etc/mysql/conf.d
+chown -R mysql:mysql /etc/mysql
diff --git a/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup b/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup
new file mode 100755
index 00000000..9c9709ca
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+dnf -y install percona-xtrabackup
+
diff --git a/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service b/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service
new file mode 100644
index 00000000..a7db5d92
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# DO NOT enable or start mysqld for systemd, let the guestagent coordinate startup
diff --git a/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server b/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server
new file mode 100755
index 00000000..9c43c6ef
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo
+
+set -e
+set -o xtrace
+
+curl -o /etc/pki/rpm-gpg/RPM-GPG-KEY-percona http://www.percona.com/downloads/RPM-GPG-KEY-percona
+cat <<EOL > /etc/yum.repos.d/Percona.repo
+[percona]
+name = CentOS \$releasever - Percona
+baseurl=http://repo.percona.com/centos/latest/os/\$basearch/
+enabled = 1
+gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-percona
+gpgcheck = 1
+EOL \ No newline at end of file
diff --git a/integration/scripts/files/elements/fedora-percona/install.d/10-mysql b/integration/scripts/files/elements/fedora-percona/install.d/10-mysql
new file mode 100755
index 00000000..284e81ac
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-percona/install.d/10-mysql
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
+# was put into the mysql guestagent code for Mitaka. There are no current
+# plans to backport so we need to make sure the guest generated when the
+# tests are run for Kilo or Liberty get the 2.2 verson of PXB
+if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
+ PXB_VERSION_OVERRIDE="-22"
+fi
+dnf -y install percona-toolkit Percona-Server-shared-55 Percona-Server-server-55 Percona-Server-test-55 Percona-Server-client-55 percona-xtrabackup${PXB_VERSION_OVERRIDE}
diff --git a/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql b/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql
new file mode 100755
index 00000000..a2ee4633
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql
@@ -0,0 +1,83 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
+# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html'
+# for best practices.
+# It is recommended to disable memory overcommit,
+# but the Python interpreter may require it on smaller flavors.
+# We therefore stick with the heuristic overcommit setting.
+vm.overcommit_memory=0
+
+_EOF_
+
+cat > "/etc/rc.local" << _EOF_
+# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html'
+# Disable Linux kernel transparent huge pages. This feature is not supported by
+# by Postgres 9.3 and may negatively impact performance of the database.
+if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then
+ echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
+fi
+if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then
+ echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
+fi
+
+exit \$?
+
+_EOF_
+
+dnf install -y http://yum.postgresql.org/9.4/fedora/fedora-22-x86_64/pgdg-fedora94-9.4-4.noarch.rpm
+
+dnf install -y postgresql94-server postgresql94-contrib postgresql94-devel gcc
+
+###########################################
+# Hack alert:
+# For Postgresql 9.4, pg_rewind is not in the main source tree and
+# no packages exist in the repos, so it must be compiled manually
+# and installed on the image until we can move to 9.5
+# See README at
+# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE
+
+dev_pkgs="readline-devel zlib-devel krb5-devel openssl-devel pam-devel libxml2-devel libxslt-devel"
+
+yum install -y $dev_pkgs
+
+# We need pg_config to be accessible on the path
+mkdir -p /tmp/build
+cd /tmp/build
+git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE
+git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE
+
+ln -s /usr/pgsql-9.4/bin/pg_config /usr/bin/pg_config
+
+cd pg_rewind
+make USE_PGXS=1 top_srcdir=/tmp/build/postgres install
+ln -s /usr/pgsql-9.4/bin/pg_rewind /usr/bin/pg_rewind
+
+# Cleanup
+
+cd
+rm -rf /tmp/build
+yum remove -y $dev_pkgs
+
+
+# Though /var/lib/pgsql is the preferred directory, need to move it as
+# this is where the volume will be mounted
+su - postgres -c "/usr/pgsql-9.4/bin/initdb /var/lib/pgsql/9.4/data"
+mv /var/lib/pgsql /var/lib/postgresql
+
+mv /lib/systemd/system/postgresql-9.4.service /lib/systemd/system/postgresql.service
+
+sed -i 's/PGDATA=\/var\/lib\/pgsql\/9.4\/data/PGDATA=\/var\/lib\/postgresql\/9.4\/data/' /lib/systemd/system/postgresql.service
+
+# Create a volatile directory for runtime files.
+echo "d /var/run/postgresql/ 0755 postgres postgres" > /lib/tmpfiles.d/postgresql.conf
+
+# Install the native Python client.
+dnf install -y postgresql-devel python-devel
+pip install psycopg2
+
+systemctl enable postgresql.service
+systemctl start postgresql.service
diff --git a/integration/scripts/files/elements/fedora-redis/README.md b/integration/scripts/files/elements/fedora-redis/README.md
new file mode 100644
index 00000000..426072cf
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-redis/README.md
@@ -0,0 +1 @@
+Sets up a redis server install in the image.
diff --git a/integration/scripts/files/elements/fedora-redis/install.d/10-redis b/integration/scripts/files/elements/fedora-redis/install.d/10-redis
new file mode 100755
index 00000000..8c23b5f6
--- /dev/null
+++ b/integration/scripts/files/elements/fedora-redis/install.d/10-redis
@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -ex
+
+dnf -y install redis
+service redis start
diff --git a/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra b/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra
new file mode 100755
index 00000000..2bd40181
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -ex
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -qy curl
+echo "deb http://debian.datastax.com/community stable main" >> /etc/apt/sources.list.d/cassandra.sources.list
+curl -L http://debian.datastax.com/debian/repo_key | apt-key add -
+apt-get update
+apt-get install -qy openjdk-7-jdk expect python-dev
+apt-get install -qy libxml2-dev ntp mc
+apt-get install -qy libxslt1-dev python-pexpect
+apt-get install -qy python-migrate build-essential
+
+apt-get install dsc21=2.1.* cassandra=2.1.* -qy
+
+# The Python Driver 2.0 for Apache Cassandra.
+pip install cassandra-driver
+# Sorted sets support for the Python driver.
+pip install blist
+
+service cassandra stop
+rm -rf /var/lib/cassandra/data/system/*
+service cassandra start
diff --git a/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase b/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase
new file mode 100755
index 00000000..0dc67d8b
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase
@@ -0,0 +1,8 @@
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -qy curl
+apt-get install -qy libssl0.9.8
+curl -O http://packages.couchbase.com/releases/2.2.0/couchbase-server-community_2.2.0_x86_64.deb
+INSTALL_DONT_START_SERVER=1 dpkg -i couchbase-server-community_2.2.0_x86_64.deb
diff --git a/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb b/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb
new file mode 100755
index 00000000..77871d3f
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+# install the ppa-finding tool for ubuntu 12.0.4 release
+apt-get install -y python-software-properties
+add-apt-repository -y ppa:couchdb/stable
+# update cached list of packages
+apt-get update -y
+# remove any existing couchdb binaries
+apt-get remove -yf couchdb couchdb-bin couchdb-common
+# install couchdb
+apt-get install -yV couchdb
+# install curl to provide a way to interact with CouchDB
+# over HTTP REST API
+apt-get install -qy curl
diff --git a/integration/scripts/files/elements/ubuntu-db2/README.md b/integration/scripts/files/elements/ubuntu-db2/README.md
new file mode 100644
index 00000000..b488f2db
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-db2/README.md
@@ -0,0 +1,36 @@
+ Creates an image for DB2 Express-C v10.5
+
+ The packages for DB2 Express-C can be downloaded from:
+ http://www-01.ibm.com/software/data/db2/express-c/download.html
+ and click on the link "DB2 Express-C for Linux 64-bit".
+ New users can either get an IBM ID or click on the "Proceed without an
+ IBM ID". User is provided with a registration form which needs to be
+ completed in order to proceed further to download the DB2 Express-C
+ packages. After accepting the license agreement, user can download the
+ the DB2 Express-C package (.tar.gz file).
+
+ There are 2 options for making the DB2 Express-C package accessible to
+ the Trove disk-image building process:
+ - place the package in a private repository and set the environment
+ variable DATASTORE_PKG_LOCATION with the url to this private
+ repository.
+ e.g. export DATASTORE_PKG_LOCATION="http://www.foo.com/db2/v10.5_linuxx64_expc.tar.gz"
+
+ - download the package and place it in any directory on the local
+ filesystem that the trove-integration scripts can access. Set the
+ environment variable DATASTORE_PKG_LOCATION with the full path to
+ the downloaded package.
+ e.g. export DATASTORE_PKG_LOCATION="/home/stack/db2/v10.5_linuxx64_expc.tar.gz"
+
+ The environment variables used are as follows:
+
+ DATASTORE_PKG_LOCATION - is the place where user stores the DB2
+ Express-C package after registration. This can either be a
+ url to a private repository or the full path to the
+ downloaded package on a local filesystem.
+ DATASTORE_DOWNLOAD_OPTS - defines any wget options user wants to specify
+ like user,password, etc. This is an optional variable and is
+ needed only if specifying a private repository to download
+ the packages from.
+ e.g. export DATASTORE_DOWNLOAD_OPTS="--user=foo --password='secret'"
+
diff --git a/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs b/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs
new file mode 100755
index 00000000..f82fd32a
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: Download the DB2 Express-C v10.5 packages to a directory on the local filesystem or
+# to a private repository. The download location is specified using the env variable:
+# DATASTORE_PKG_LOCATION
+
+[ -n "${TMP_HOOKS_PATH}" ] || die "Temp hook path not set"
+[ -n "${DATASTORE_PKG_LOCATION}" ] || die "DATASTORE_PKG_LOCATION not set"
+
+# First check if the package is available on the local filesystem.
+if [ -f "${DATASTORE_PKG_LOCATION}" ]; then
+ echo "Found the DB2 Express-C packages in ${DATASTORE_PKG_LOCATION}."
+ dd if="${DATASTORE_PKG_LOCATION}" of=${TMP_HOOKS_PATH}/db2.tar.gz
+# else, check if the package is available for download in a private repository.
+elif wget ${DATASTORE_DOWNLOAD_OPTS} "${DATASTORE_PKG_LOCATION}" -O ${TMP_HOOKS_PATH}/db2.tar.gz; then
+ echo "Downloaded the DB2 Express-C package from the private repository"
+else
+ echo "Unable to find the DB2 package at ${DATASTORE_PKG_LOCATION}"
+ echo "Please register and download the DB2 Express-C packages to a private repository or local filesystem."
+ exit -1
+fi
diff --git a/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2 b/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2
new file mode 100755
index 00000000..63bc3bf7
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Uncompress the DB2 packages and install and configure DB2 on Ubuntu.
+
+# DB2_PKG_LOCATION points to the directory where the DB2 packages
+# are located to install.
+DB2_PKG_LOCATION="/db2"
+mkdir ${DB2_PKG_LOCATION}
+cd ${DB2_PKG_LOCATION}
+
+# DB2 install requires the hostname to be resolved correctly
+host_name=`hostname`
+echo "127.0.0.1 ${host_name}" >> /etc/hosts
+
+tar -xvzf /tmp/in_target.d/db2.tar.gz
+
+# installing dependencies
+apt-get install libaio1
+apt-get install libstdc++6
+
+# start the installation process. Accepts the default installation directory '/opt/ibm/db2/V10.5'
+${DB2_PKG_LOCATION}/expc/db2_install -b /opt/ibm/db2/V10.5 -f sysreq -l ${DB2_PKG_LOCATION}/db2_install.log
+
+# create the DB2 users.
+# DB2 instance owner - db2inst1
+# DB2 fence user - db2fenc1
+# DB2 admin user - db2das1
+useradd -m db2inst1
+useradd -m db2fenc1
+useradd -m db2das1
+
+# Create the DB2 server instance
+/opt/ibm/db2/V10.5/instance/db2icrt -a server -u db2fenc1 db2inst1
+/opt/ibm/db2/V10.5/cfg/db2ln
+
+# Configure DB2 server instance to communicate via TCP/IP on a particulat port.
+echo 'db2c_db2inst1 50000/tcp # DB2 connection service port' >> /etc/services
+
+# Configure DB2 to use the TCP/IP settings defined above.
+su - db2inst1 -c "db2 update database manager configuration using svcename db2c_db2inst1"
+
+# Start the actual TCP/IP communication.
+su - db2inst1 -c "db2set DB2COMM=tcpip"
+
+# DB2 requires the hostname to be resolved correctly. Delete this entry from the
+# /etc/hosts since this is the hostname of the instance where the image is being
+# built. The correct hostname will be set in the guest agent.
+sed -i "/127.0.0.1[[:space:]]*${host_name}/d" /etc/hosts
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep
new file mode 100755
index 00000000..33b42306
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep
+
+source $_LIB/die
+
+BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default}
+ADD_BRANCH=$(basename ${BRANCH_OVERRIDE})
+REQUIREMENTS_FILE=${REDSTACK_SCRIPTS}/files/requirements/ubuntu-requirements-${ADD_BRANCH}.txt
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found"
+[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set"
+
+sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt
+
+# Grab the upper constraints file, but don't fail if we can't find it.
+# If we are running in the CI environment, $DEST will be set and stackrc
+# will use $DEST/requirements as the location for the requirements repo.
+# Use that as it will help us chain a job with something that is changing UC.
+
+UC_FILE=upper-constraints.txt
+
+if [ -f "${DEST}/requirements/${UC_FILE}" ]; then
+ echo "Found ${DEST}/requirements/${UC_FILE}, using that"
+ sudo -Hiu ${HOST_USERNAME} dd if="${DEST}/requirements/${UC_FILE}" \
+ of="${TMP_HOOKS_PATH}/${UC_FILE}"
+else
+ UC_DIR=$(pwd)
+ UC_BRANCH=${BRANCH_OVERRIDE}
+ if [ "${ADD_BRANCH}" == "default" ]; then
+ UC_BRANCH=master
+ fi
+
+ set +e
+ curl -o "${UC_DIR}/${UC_FILE}" \
+ https://git.openstack.org/cgit/openstack/requirements/plain/${UC_FILE}?h=${UC_BRANCH}
+ set -e
+
+ if [ -f "${UC_DIR}/${UC_FILE}" ]; then
+ sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE}
+ rm -f "${UC_DIR}/${UC_FILE}"
+ fi
+fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart
new file mode 100755
index 00000000..2b221592
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart
@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
+# configured
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
+[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
+[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host"
+[ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir"
+[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
+
+sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key
new file mode 100755
index 00000000..63453a75
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+set -o xtrace
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where
+# they will be duplicated in the guest VM.
+# This process allows the host to log into the guest but more importantly the guest phones home to get the trove
+# source
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host"
+
+if [ `whoami` = "root" ]; then
+ die "This should not be run as root"
+fi
+
+# copy files over the "staging" area for the guest image (they'll later be put in the correct location by the guest user
+# not these keys should not be overridden otherwise a) you won't be able to ssh in and b) the guest won't be able to
+# rsync the files
+if [ -e ${SSH_DIR}/authorized_keys ]; then
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/authorized_keys of=${TMP_HOOKS_PATH}/ssh-authorized-keys
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
+ sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
+else
+ die "SSH Authorized Keys file must exist along with pub and private key"
+fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps
new file mode 100755
index 00000000..5cd392b2
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install basic services and applications
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install ntp apparmor-utils
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep b/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep
new file mode 100755
index 00000000..8d989fe0
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install python-dev libxml2-dev libxslt1-dev python-setuptools \
+ python-pip python-sqlalchemy python-lxml \
+ python-routes python-eventlet python-webob \
+ python-pastedeploy python-paste python-netaddr \
+ python-httplib2 python-iso8601 python-pexpect python-mysqldb python-migrate
+
+
+# pick up the requirements file left for us by
+# extra-data.d/15-reddwarf-dep
+
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+UPPER_CONSTRAINTS=
+if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then
+ UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt"
+fi
+
+pip install -q --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS}
+
+echo "diagnostic pip freeze output follows"
+pip freeze
+echo "diagnostic pip freeze output above"
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc b/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc
new file mode 100755
index 00000000..8ac19f7c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
+
+dd if=/tmp/in_target.d/trove-guest.conf of=/etc/init/trove-guest.conf
+
+
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user
new file mode 100755
index 00000000..99f68966
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist
+
+set -e
+set -o xtrace
+
+if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
+ echo "Adding ${GUEST_USERNAME} user"
+ useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash
+ chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME}
+ passwd ${GUEST_USERNAME} <<_EOF_
+${GUEST_USERNAME}
+${GUEST_USERNAME}
+_EOF_
+fi
+
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key
new file mode 100755
index 00000000..8085cbaa
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory
+
+set -e
+set -o xtrace
+
+SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+if [ -e "${TMP_HOOKS_DIR}/ssh-authorized-keys" ]; then
+ if [ ! -e ${SSH_DIR} ]; then
+ # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
+ mkdir ${SSH_DIR}
+ chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR}
+ fi
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/authorized_keys conv=notrunc if=${TMP_HOOKS_DIR}/ssh-authorized-keys
+ if [ ! -e "${SSH_DIR}/id_rsa" ]; then
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
+ # perms have to be right on this file for ssh to work
+ sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa
+ sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
+ fi
+else
+ echo "SSH Keys were not staged by host"
+ exit -1
+fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh
new file mode 100755
index 00000000..2134c198
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh
@@ -0,0 +1,8 @@
+#!/bin/bash
+# Regenerate host keys now. XXX: Really should be a cloud-init task, should get
+# that working.
+
+set -e
+set -o xtrace
+
+dpkg-reconfigure openssh-server
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt
new file mode 100755
index 00000000..cc348c5c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Delete contents of apt cache on guest (saves image disk space)
+
+set -e
+set -o xtrace
+
+apt-get clean
+
+
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding
new file mode 100755
index 00000000..48b7ad0d
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+set -o xtrace
+sed -i -r 's/^\s*#(net\.ipv4\.ip_forward=1.*)/\1/' /etc/sysctl.conf
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp b/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp
new file mode 100755
index 00000000..bc494ae3
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp
@@ -0,0 +1,10 @@
+#!/bin/bash
+ntpfile=`mktemp`
+cat << EOF > $ntpfile
+server ntp.ubuntu.com iburst
+server 127.127.1.0
+fudge 127.127.1.0 stratum 10
+EOF
+
+mv /etc/ntp.conf /etc/ntp.conf.orig
+mv $ntpfile /etc/ntp.conf
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers
new file mode 100755
index 00000000..0581fd2b
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers
@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+set -o xtrace
+
+# CONTEXT: HOST after IMAGE BUILD as SCRIPT USER
+# PURPOSE: add the guest user account to the /etc/sudoers files with NOPASSWD
+
+# Adds user to the sudoers file so they can do everything w/o a pass
+# Some binaries might be under /sbin or /usr/sbin, so make sure sudo will
+# see them by forcing PATH
+TEMPFILE=`mktemp`
+echo "${GUEST_USERNAME} ALL=(ALL) NOPASSWD:ALL" > $TEMPFILE
+chmod 0440 $TEMPFILE
+sudo chown root:root $TEMPFILE
+sudo mv $TEMPFILE /etc/sudoers.d/60_trove_guest
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update b/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update
new file mode 100755
index 00000000..6cc087a7
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# CONTEXT: GUEST after packages installed
+# PURPOSE: do apt-get update to save each instance having to do all the work
+
+set -e
+set -o xtrace
+
+apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs
new file mode 100755
index 00000000..8787df7b
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs
@@ -0,0 +1,117 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install basic services and applications
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y purge acpid\
+ apport\
+ apport-symptoms\
+ apt-transport-https\
+ apt-xapian-index\
+ aptitude\
+ at\
+ bash-completion\
+ bc\
+ bind9-host\
+ bsdmainutils\
+ busybox-static\
+ byobu\
+ command-not-found\
+ command-not-found-data\
+ curl\
+ dbus\
+ dmidecode\
+ dosfstools\
+ ed\
+ fonts-ubuntu-font-family-console\
+ friendly-recovery\
+ ftp\
+ fuse\
+ geoip-database\
+ groff-base\
+ hdparm\
+ info\
+ install-info\
+ iptables\
+ iputils-tracepath\
+ irqbalance\
+ landscape-client\
+ landscape-common\
+ language-selector-common\
+ laptop-detect\
+ libaccountsservice0\
+ libbind9-90\
+ libclass-accessor-perl\
+ libcwidget3\
+ libdns100\
+ libept1.4.12\
+ libevent-2.0-5\
+ libgc1c2\
+ libgeoip1\
+ libio-string-perl\
+ libisc95\
+ liblwres90\
+ libnfnetlink0\
+ libparse-debianchangelog-perl\
+ libparted0debian1\
+ libpcap0.8\
+ libpci3\
+ libpipeline1\
+ libpolkit-gobject-1-0\
+ libsasl2-modules\
+ libsigc++-2.0-0c2a\
+ libsub-name-perl\
+ libusb-1.0-0\
+ libxapian22\
+ lshw\
+ lsof\
+ ltrace\
+ man-db\
+ mlocate\
+ mtr-tiny\
+ nano\
+ ntfs-3g\
+ parted\
+ patch\
+ plymouth-theme-ubuntu-text\
+ popularity-contest\
+ powermgmt-base\
+ ppp\
+ pppoeconf\
+ python-debian\
+ python-gdbm\
+ python-pam\
+ python-twisted-bin\
+ python-xapian\
+ python-zope.interface\
+ screen\
+ shared-mime-info\
+ strace\
+ tasksel\
+ tcpdump\
+ telnet\
+ time\
+ tmux\
+ ubuntu-standard\
+ ufw\
+ update-manager-core\
+ update-notifier-common\
+ usbutils\
+ uuid-runtime\
+ w3m
+
+# The following packages cannot be removed as they cause cloud-init to be
+# uninstalled in Ubuntu 14.04
+# gir1.2-glib-2.0
+# libdbus-glib-1-2
+# libgirepository-1.0-1
+# python-chardet
+# python-serial
+# xz-utils
+
+apt-get -y autoremove
+
diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools
new file mode 100755
index 00000000..1a8647f5
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools
@@ -0,0 +1,7 @@
+#!/bin/bash
+# Install baseline packages and tools.
+
+set -e
+set -o xtrace
+
+apt-get install -y language-pack-en python-software-properties \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/README.md b/integration/scripts/files/elements/ubuntu-mariadb/README.md
new file mode 100644
index 00000000..757f00b8
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mariadb/README.md
@@ -0,0 +1,3 @@
+Sets up a MariaDB server install in the image.
+
+TODO: auto-tune settings based on host resources or metadata service.
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb
new file mode 100755
index 00000000..065c2f98
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+# NOTE(vkmc): Using MariaDB repositories is required
+# https://mariadb.com/kb/en/mariadb/installing-mariadb-deb-files/
+apt-get -y install software-properties-common
+apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db
+add-apt-repository 'deb http://ftp.osuosl.org/pub/mariadb/repo/10.1/ubuntu trusty main'
+
+# Pin MariaDB repository
+sudo echo -e "Package: *\nPin: origin ftp.osuosl.org\nPin-Priority: 1000" > /etc/apt/preferences.d/mariadb.pref
+
+apt-get -y update
+# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
+# was put into the mysql guestagent code for Mitaka. There are no current
+# plans to backport so we need to make sure the guest generated when the
+# tests are run for Kilo or Liberty get the 2.2 verson of PXB
+if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
+ PXB_VERSION_OVERRIDE="-22"
+fi
+apt-get -y install socat percona-xtrabackup${PXB_VERSION_OVERRIDE}
+apt-get -y install libmariadbclient18 mariadb-server
+
+cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
+[mysqld]
+performance_schema = off
+_EOF_
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key
new file mode 100755
index 00000000..ec1d89d5
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo
+
+set -e
+set -o xtrace
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${RELEASE}" ] || die "RELEASE must be set to either Trusty or Precise"
+
+# Add Percona GPG key
+mkdir -p /home/${GUEST_USERNAME}/.gnupg
+
+# sometimes the primary key server is unavailable and we should try an
+# alternate. see
+# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
+# shell errexit so we can interrogate the exit code and take action
+# based on the exit code. We will reenable it later.
+set +e
+apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A
+
+if [ "$?" -ne "0" ];
+then
+ echo "Trying alternate keyserver hkp://keyserver.ubuntu.com"
+ set -e
+ apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A
+fi
+
+set -e
+
+# Add Percona repo
+# Creates the Percona sources list
+cat <<EOL > /etc/apt/sources.list.d/percona.list
+deb http://repo.percona.com/apt $RELEASE main
+deb-src http://repo.percona.com/apt $RELEASE main
+EOL
+
+# Force an update
+apt-get -y update
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local
new file mode 100755
index 00000000..a3e1dc7c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+#CONTEXT: chroot on host
+#PURPOSE: Allows mysqld to create temporary files when restoring backups
+
+cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
+ /tmp/ rw,
+ /tmp/** rwk,
+EOF
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/README.md b/integration/scripts/files/elements/ubuntu-mongodb/README.md
new file mode 100644
index 00000000..5b9e33bb
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/README.md
@@ -0,0 +1 @@
+Sets up a MongoDB install in the image.
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp b/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp
new file mode 100755
index 00000000..48d6c8bf
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+cat > /etc/init.d/disable-transparent-hugepages << '_EOF_'
+#!/bin/sh
+### BEGIN INIT INFO
+# Provides: disable-transparent-hugepages
+# Required-Start: $local_fs
+# Required-Stop:
+# X-Start-Before: mongod mongodb-mms-automation-agent
+# Default-Start: 2 3 4 5
+# Default-Stop: 0 1 6
+# Short-Description: Disable Linux transparent huge pages
+# Description: Disable Linux transparent huge pages, to improve
+# database performance.
+### END INIT INFO
+
+case $1 in
+ start)
+ if [ -d /sys/kernel/mm/transparent_hugepage ]; then
+ thp_path=/sys/kernel/mm/transparent_hugepage
+ elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then
+ thp_path=/sys/kernel/mm/redhat_transparent_hugepage
+ else
+ return 0
+ fi
+
+ echo 'never' > ${thp_path}/enabled
+ echo 'never' > ${thp_path}/defrag
+
+ unset thp_path
+ ;;
+esac
+_EOF_
+
+chmod 755 /etc/init.d/disable-transparent-hugepages
+
+update-rc.d disable-transparent-hugepages defaults
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb b/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb
new file mode 100755
index 00000000..6a95d2f0
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+apt-get -y install mongodb-org=3.2.6
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep b/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep
new file mode 100755
index 00000000..a3e2633b
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu
+
+set -e
+set -o xtrace
+
+pip install pymongo>=3.0.2,!=3.1
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf b/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf
new file mode 100755
index 00000000..afb58284
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+# Remove the default pid file
+rm -f /var/run/mongodb.pid
+
+
+cat > /etc/mongod.conf << '_EOF_'
+storage.dbPath: /var/lib/mongodb
+security.authorization: enabled
+storage.engine: wiredTiger
+storage.journal.enabled: true
+systemLog.destination: file
+systemLog.logAppend: true
+systemLog.path: /var/log/mongodb/mongod.log
+_EOF_
+
+
+cat > /etc/mongos.conf << '_EOF_'
+security.authorization: enabled
+systemLog.destination: file
+systemLog.logAppend: true
+systemLog.path: /var/log/mongodb/mongos.log
+_EOF_
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init b/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init
new file mode 100755
index 00000000..77a12927
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+cat > /etc/init/mongod.conf << '_EOF_'
+limit fsize unlimited unlimited
+limit cpu unlimited unlimited
+limit as unlimited unlimited
+limit nofile 64000 64000
+limit rss unlimited unlimited
+limit nproc 64000 64000
+
+kill timeout 300 # wait 300s between SIGTERM and SIGKILL.
+
+pre-start script
+ mkdir -p /var/run/mongodb/
+ touch /var/run/mongodb/mongod.pid
+ chown mongodb -R /var/run/mongodb/
+end script
+
+start on runlevel [2345]
+stop on runlevel [06]
+
+script
+ CONF=/etc/mongod.conf
+
+ # Handle NUMA access to CPUs (SERVER-3574)
+ # This verifies the existence of numactl as well as testing that the command works
+ NUMACTL_ARGS="--interleave=all"
+ if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null
+ then
+ NUMACTL="$(which numactl) -- $NUMACTL_ARGS"
+ DAEMON_OPTS=${DAEMON_OPTS:-"--config $CONF"}
+ else
+ NUMACTL=""
+ DAEMON_OPTS="-- "${DAEMON_OPTS:-"--config $CONF"}
+ fi
+
+ exec start-stop-daemon --start \
+ --chuid mongodb \
+ --pidfile /var/run/mongod.pid \
+ --make-pidfile \
+ --exec $NUMACTL /usr/bin/mongod $DAEMON_OPTS
+end script
+_EOF_
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init b/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init
new file mode 100755
index 00000000..dc91fdba
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+cat > /etc/init/mongos.conf << '_EOF_'
+limit fsize unlimited unlimited
+limit cpu unlimited unlimited
+limit as unlimited unlimited
+limit nofile 64000 64000
+limit rss unlimited unlimited
+limit nproc 64000 64000
+
+pre-start script
+ mkdir -p /var/run/mongodb/
+ touch /var/run/mongodb/mongos.pid
+ chown mongodb -R /var/run/mongodb/
+end script
+
+start on runlevel [2345]
+stop on runlevel [06]
+
+script
+ exec start-stop-daemon --start \
+ --chuid mongodb \
+ --pidfile /var/run/mongos.pid \
+ --make-pidfile \
+ --exec /usr/bin/mongos -- --config /etc/mongos.conf
+end script
+_EOF_
diff --git a/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key b/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key
new file mode 100755
index 00000000..1538d61c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
+
+apt-get -y install software-properties-common
+
+apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927
+
+echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list
+
+apt-get -y update
diff --git a/integration/scripts/files/elements/ubuntu-mysql/README.md b/integration/scripts/files/elements/ubuntu-mysql/README.md
new file mode 100644
index 00000000..39a6ab8c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mysql/README.md
@@ -0,0 +1,3 @@
+Sets up a MySQL server install in the image.
+
+TODO: auto-tune settings based on host resources or metadata service.
diff --git a/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql
new file mode 100755
index 00000000..d31292ec
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
+# was put into the mysql guestagent code for Mitaka. There are no current
+# plans to backport so we need to make sure the guest generated when the
+# tests are run for Kilo or Liberty get the 2.2 verson of PXB
+if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
+ PXB_VERSION_OVERRIDE="-22"
+fi
+apt-get -y install libmysqlclient18 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE}
+
+cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
+[mysqld]
+performance_schema = off
+_EOF_
diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key
new file mode 100755
index 00000000..2a03ad50
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key
@@ -0,0 +1,40 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo
+
+set -e
+set -o xtrace
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
+
+# Add Percona GPG key
+mkdir -p /home/${GUEST_USERNAME}/.gnupg
+
+# sometimes the primary key server is unavailable and we should try an
+# alternate. see
+# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
+# shell errexit so we can interrogate the exit code and take action
+# based on the exit code. We will reenable it later.
+set +e
+apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A
+
+if [ "$?" -ne "0" ];
+then
+ echo "Trying alternate keyserver hkp://keyserver.ubuntu.com"
+ set -e
+ apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A
+fi
+
+set -e
+
+# Add Percona repo
+# Creates the percona sources list
+cat <<EOL > /etc/apt/sources.list.d/percona.list
+deb http://repo.percona.com/apt $RELEASE main
+deb-src http://repo.percona.com/apt $RELEASE main
+EOL
+
+# Force an update
+apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local
new file mode 100755
index 00000000..a3e1dc7c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+#CONTEXT: chroot on host
+#PURPOSE: Allows mysqld to create temporary files when restoring backups
+
+cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
+ /tmp/ rw,
+ /tmp/** rwk,
+EOF
diff --git a/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql
new file mode 100755
index 00000000..5d5b4265
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+# The fix to make versions of percona-xtrabackup > v2.2 work with Trove
+# was put into the mysql guestagent code for Mitaka. There are no current
+# plans to backport so we need to make sure the guest generated when the
+# tests are run for Kilo or Liberty get the 2.2 verson of PXB
+if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then
+ PXB_VERSION_OVERRIDE="-22"
+fi
+apt-get -y install percona-toolkit percona-server-common-5.6 percona-server-server-5.6 percona-server-test-5.6 percona-server-client-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE}
diff --git a/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key
new file mode 100755
index 00000000..c2b686c4
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo
+
+set -e
+set -o xtrace
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
+
+#5 add Percona GPG key
+if [ ! -e /home/${GUEST_USERNAME}/.gnupg ]; then
+ mkdir -p /home/${GUEST_USERNAME}/.gnupg
+fi
+
+# sometimes the primary key server is unavailable and we should try an
+# alternate. see
+# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
+# shell errexit so we can interrogate the exit code and take action
+# based on the exit code. We will reenable it later.
+set +e
+apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A
+
+if [ "$?" -ne "0" ];
+then
+ echo "Trying alternate keyserver hkp://keyserver.ubuntu.com"
+ set -e
+ apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A
+fi
+
+set -e
+
+# add Percona repo
+# creates the percona sources list
+cat <<EOL > /etc/apt/sources.list.d/percona.list
+deb http://repo.percona.com/apt $RELEASE main
+deb-src http://repo.percona.com/apt $RELEASE main
+EOL
+
+# force an update
+apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local
new file mode 100755
index 00000000..a3e1dc7c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+#CONTEXT: chroot on host
+#PURPOSE: Allows mysqld to create temporary files when restoring backups
+
+cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
+ /tmp/ rw,
+ /tmp/** rwk,
+EOF
diff --git a/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql b/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql
new file mode 100755
index 00000000..cd853256
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql
@@ -0,0 +1,79 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+
+cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
+# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html'
+# for best practices.
+# It is recommended to disable memory overcommit,
+# but the Python interpreter may require it on smaller flavors.
+# We therefore stick with the heuristic overcommit setting.
+vm.overcommit_memory=0
+vm.nr_hugepages=64
+
+_EOF_
+
+cat > "/etc/rc.local" << _EOF_
+# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html'
+# Postgres 9.4 added support for THP. Using huge pages reduces overhead when
+# using large contiguous chunks of memory, like PostgreSQL does.
+if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
+ echo never > /sys/kernel/mm/transparent_hugepage/defrag
+fi
+if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
+ echo always > /sys/kernel/mm/transparent_hugepage/enabled
+fi
+
+exit \$?
+
+_EOF_
+
+apt-get -y install postgresql-9.4 postgresql-contrib-9.4 postgresql-server-dev-9.4
+
+###########################################
+# Hack alert:
+# For Postgresql 9.4, pg_rewind is not in the main source tree and
+# no packages exist in the repos, so it must be compiled manually
+# and installed on the image until we can move to 9.5
+# See README at
+# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE
+
+tmpdir=/tmp/build
+mkdir -p $tmpdir
+cd $tmpdir
+git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE
+cd postgres/contrib
+git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE
+
+dev_pkgs="libreadline-dev libkrb5-dev libssl-dev libpam-dev libxml2-dev libxslt-dev libedit-dev libselinux1-dev bison flex"
+
+apt-get install $dev_pkgs -y
+
+# Unfortunately, on ubuntu, was not able to get pg_rewind to build
+# outside of the pgsql source tree. Configure and compile postgres
+# but only call make install against the contrib/pg_rewind directory
+# so that support library is accessible to the server
+cd $tmpdir/postgres
+./configure
+make
+cd contrib/pg_rewind
+make install
+
+# Make the pg_rewind binary and the library used by the
+# pg_rewind stored procedures accessible
+ln -s /usr/local/pgsql/bin/pg_rewind /usr/bin/pg_rewind
+ln -s /usr/local/pgsql/lib/pg_rewind_support.so /usr/lib/postgresql/9.4/lib/pg_rewind_support.so
+
+cd
+rm -rf $tmpdir
+apt-get remove -y $dev_pkgs
+
+# End hack
+################################
+
+# Install the native Python client.
+apt-get -y install libpq-dev
+pip install psycopg2
diff --git a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo
new file mode 100755
index 00000000..48a25d1d
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+set -e
+set -o xtrace
+
+cat <<EOL > /etc/apt/sources.list.d/postgresql.list
+deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main
+EOL
+
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
+
+apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql
new file mode 100755
index 00000000..ae658957
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql
@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y install percona-xtradb-cluster-server-5.6 percona-xtradb-cluster-client-5.6 percona-xtrabackup
+
+# Don't auto start mysql (we'll start it up in guest)
+update-rc.d mysql defaults
+update-rc.d mysql disable
diff --git a/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key
new file mode 100755
index 00000000..c2b686c4
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT
+# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo
+
+set -e
+set -o xtrace
+
+[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
+[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal"
+
+#5 add Percona GPG key
+if [ ! -e /home/${GUEST_USERNAME}/.gnupg ]; then
+ mkdir -p /home/${GUEST_USERNAME}/.gnupg
+fi
+
+# sometimes the primary key server is unavailable and we should try an
+# alternate. see
+# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
+# shell errexit so we can interrogate the exit code and take action
+# based on the exit code. We will reenable it later.
+set +e
+apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A
+
+if [ "$?" -ne "0" ];
+then
+ echo "Trying alternate keyserver hkp://keyserver.ubuntu.com"
+ set -e
+ apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A
+fi
+
+set -e
+
+# add Percona repo
+# creates the percona sources list
+cat <<EOL > /etc/apt/sources.list.d/percona.list
+deb http://repo.percona.com/apt $RELEASE main
+deb-src http://repo.percona.com/apt $RELEASE main
+EOL
+
+# force an update
+apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local
new file mode 100755
index 00000000..a3e1dc7c
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -e
+
+#CONTEXT: chroot on host
+#PURPOSE: Allows mysqld to create temporary files when restoring backups
+
+cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
+ /tmp/ rw,
+ /tmp/** rwk,
+EOF
diff --git a/integration/scripts/files/elements/ubuntu-redis/README.md b/integration/scripts/files/elements/ubuntu-redis/README.md
new file mode 100644
index 00000000..426072cf
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-redis/README.md
@@ -0,0 +1 @@
+Sets up a redis server install in the image.
diff --git a/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis b/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis
new file mode 100755
index 00000000..57fcdf90
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis
@@ -0,0 +1,53 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT
+# PURPOSE: Install controller base required packages
+
+set -ex
+
+export DEBIAN_FRONTEND=noninteractive
+
+cat > "/etc/sysctl.d/10-redis-performance.conf" << _EOF_
+# See 'http://redis.io/topics/admin' for best practices.
+# Make sure to set the Linux kernel overcommit memory setting to 1.
+vm.overcommit_memory=1
+
+# Linux kernel will silently truncate 'tcp-backlog' to the value of
+# '/proc/sys/net/core/somaxconn' so make sure to raise both the value of
+# 'somaxconn' and 'tcp_max_syn_backlog' in order to get the desired effect.
+net.ipv4.tcp_max_syn_backlog=1024
+net.core.somaxconn=1024
+
+_EOF_
+
+cat > "/etc/rc.local" << _EOF_
+# Make sure to disable Linux kernel feature transparent huge pages,
+# it will affect greatly both memory usage and latency in a negative way.
+if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
+ echo never > /sys/kernel/mm/transparent_hugepage/defrag
+fi
+if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
+ echo never > /sys/kernel/mm/transparent_hugepage/enabled
+fi
+
+_EOF_
+
+add-apt-repository -y ppa:chris-lea/redis-server
+apt-get -y update
+apt-get install -y redis-server
+
+cat > "/etc/default/redis-server" << _EOF_
+# Call ulimit -n with this argument prior to invoking Redis itself.
+# This may be required for high-concurrency environments. Redis itself cannot
+# alter its limits as it is not being run as root.
+ULIMIT=65536
+
+_EOF_
+
+# Install Python driver for Redis ('redis-py').
+pip install redis
+
+# By default, redis-py will attempt to use the HiredisParser if installed.
+# Using Hiredis can provide up to a 10x speed improvement in parsing responses
+# from the Redis server.
+pip install hiredis
diff --git a/integration/scripts/files/elements/ubuntu-vertica/README.md b/integration/scripts/files/elements/ubuntu-vertica/README.md
new file mode 100644
index 00000000..86202379
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-vertica/README.md
@@ -0,0 +1 @@
+Sets up a Vertica CE 7.1 debian package and other dependencies install in the image.
diff --git a/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb b/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb
new file mode 100755
index 00000000..56c02cda
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
+# PURPOSE: Stages the package installer file from DATASTORE_PKG_LOCATION,
+# so that guest image has the package file.
+
+set -e
+set -o xtrace
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+[ -f "$DATASTORE_PKG_LOCATION" ] || die "Datastore package installer file not found at:" $DATASTORE_PKG_LOCATION
+dd if=${DATASTORE_PKG_LOCATION} of=${TMP_HOOKS_PATH}/vertica.deb
diff --git a/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica b/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica
new file mode 100755
index 00000000..2d50bc6a
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica
@@ -0,0 +1,54 @@
+#!/bin/sh
+
+# CONTEXT: GUEST during CONSTRUCTION as ROOT - install.d
+# PURPOSE: Install controller base required packages
+
+set -e
+set -o xtrace
+export DEBIAN_FRONTEND=noninteractive
+
+
+# Copy the package file to the image,
+# as it needs to be used later during configuration.
+dd if=/tmp/in_target.d/vertica.deb of=/vertica.deb
+
+# Install base packages
+apt-get install -qy build-essential bc iptables
+apt-get install -qy curl sysstat pstack mcelog
+apt-get install -qy python-dev g++ unixODBC unixODBC-dev dialog
+apt-get install -qy dialog libbz2-dev libboost-all-dev libcurl4-gnutls-dev
+apt-get install -qy openjdk-7-jdk
+
+# Install Vertica package
+dpkg -i /vertica.deb
+
+# Creating dbadmin user and verticadba group
+groupadd verticadba
+useradd -g verticadba -d /home/dbadmin -s /bin/bash -m dbadmin
+echo "export PATH=/opt/vertica/bin:\$PATH" >> ~dbadmin/.profile
+echo "export TZ=`date +%Z`" >> ~dbadmin/.profile
+
+# Create base directory for to be used for database creation
+mkdir /var/lib/vertica
+chown dbadmin:verticadba /var/lib/vertica
+
+# Backup /etc/hosts
+cp -p /etc/hosts /etc/hosts.bkp
+
+# Compile the SDK examples - the supplied UDFs can then be loaded
+cd /opt/vertica/sdk/examples
+TMPDIR=/tmp JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 make
+cd
+
+cat > "/etc/rc.local" << _EOF_
+# Vertica requires THP to be turned off
+if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
+ echo never > /sys/kernel/mm/transparent_hugepage/defrag
+fi
+if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
+ echo never > /sys/kernel/mm/transparent_hugepage/enabled
+fi
+
+exit \$?
+
+_EOF_
diff --git a/integration/scripts/files/keys/authorized_keys b/integration/scripts/files/keys/authorized_keys
new file mode 100644
index 00000000..4093a714
--- /dev/null
+++ b/integration/scripts/files/keys/authorized_keys
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmQeA/uyEyFf9DsmwR+OztWb7Hb/uTC+R3xG1QgBvRwhSbpBnyBESGMZZ07bIw5Ib7BUSDzwoeryUqNAhAhir2KLeIYODS39UmTwOIl+rIvhlTxhsIoQHV90pewD2qw0T8KgVMPUDsQ0Bd98E6e5dbxciZp67ihVD0r7srhdSRo8PIc56hJWrD52j5FeiIGEmLXHXiZLOyma1M7j/EmiV81wHAzgql6sihWSZHm3xPZZ712JtXbmHhe3RLFIK13u9PSb3XbuEIdGwkZdzP+vYNE0CsYqwjXjVRrY/APsiEkbSNVzHI5p2W1L7ZMtSOMUqZ1Ve+sytVb+YcIJ9L8y07 trove@devstack \ No newline at end of file
diff --git a/integration/scripts/files/keys/id_rsa b/integration/scripts/files/keys/id_rsa
new file mode 100644
index 00000000..041e61fd
--- /dev/null
+++ b/integration/scripts/files/keys/id_rsa
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5kHgP7shMhX/Q7JsEfjs7Vm+x2/7kwvkd8RtUIAb0cIUm6QZ
+8gREhjGWdO2yMOSG+wVEg88KHq8lKjQIQIYq9ii3iGDg0t/VJk8DiJfqyL4ZU8Yb
+CKEB1fdKXsA9qsNE/CoFTD1A7ENAXffBOnuXW8XImaeu4oVQ9K+7K4XUkaPDyHOe
+oSVqw+do+RXoiBhJi1x14mSzspmtTO4/xJolfNcBwM4KperIoVkmR5t8T2We9dib
+V25h4Xt0SxSCtd7vT0m9127hCHRsJGXcz/r2DRNArGKsI141Ua2PwD7IhJG0jVcx
+yOadltS+2TLUjjFKmdVXvrMrVW/mHCCfS/MtOwIDAQABAoIBAQCTAQHbjmwuHu8e
+2F24x7Tz//UWPT9fbRtK/6RO3ctlCsS/bXCHHARnrGcDdfHq1yv6PS21/UvXtThE
+Dn4qO75X9DzgnAFNgEwELjPyVBM4YG2pF3SQ+MJESaI4hgGY8Rws5eMF/qFhdbo1
+hATggqFqnQZqWy0DP9wkq8ESk1nYNICehj0d6Mo5uW190TDMD6QrfHg4rDYbgITf
+SCmsRdybCASlgOHCrYgjweG9czNoFimhaG8WwP59yfCX0A1TrDJ0toryyopupIre
+A+5HHBM2Dk3KylrtVBAPRsM9eGUo4bmz8p6hRkuw7mr321d416MHdIXcHK38EWR0
+ZvUWM8QJAoGBAPlPe3ggR9xhEbQJQkOohnZ1hkogE/uxZrcRiUI3pnDNcCXy9Ogt
+SYfwxYnHLa5kuynbmCCzSLOtq2DN9QwJ5o+zgRjX9T8DAWub6KFdEnTHhq/ZLF/w
+PWPg3Oe8dYDEEcrPLvN25aetY2LrWKsRPFL8//WLJc1+LFRG6Vc1ATftAoGBAOxv
+hGxNGrcD5c8g0ZcyeKVbCCRGfp0+mwonnQ25mDyIXNH+PEHa7SPs5dVexA5r0/Ky
+lM/jQgs756EvslwA2oukqVz7ehDhJI4RE92OPjpYrAu7HF6eN/fAhUMghs/vAZ6c
+YCM6i9emHYHM4mU6H/yLIr+0e2JNf+479bB5hQTHAoGBAOGnFUQXQ7OukE16C+Yd
+RQc3PIMfIbcwTJ3qW2f54sY6zAUtMIptYx6NyN35z9kHB3jNb5Y5b9ZhnLqT7/Yj
+h/INMQ4BedK8r65sgVR8X1YfukKzuLxlP8uFHa0KIPiZftkoSYDH3vmzsD86cRj1
+ErqykCH4/hBO4WSugkkSirXJAoGARXvQcvOF8lsW4nRGpCSVCCNklSSSeSu47JcP
+tMTiVIfOn3gTxVbNck1gjgA3pfVSaHTK/v1On3aPb/NQe3FUyM0vaMAO8372+zlR
+mT8AUq1Ugm4OvE/LKuhNQZkBhYI7+50BM9k0179d1JOdxRn75IAPSj+EMzOLcTv1
+zFMqIGkCgYBm5xT3Gu8fJh/8ztelzrDkGga6UpYKKYjHGFHpaqmDn9sjjCu8X2pG
+JUGgyUVj1NkJAtHMS45Ud3upQwxpy2aNmMaQbwzHybvX7EYZHHVuCwsSzaXRtwj2
+Q6mG1Ghi0UQ76SPKQr0Vu8Uu+0CAzYAK4IEKeH6BCRjrzHggSpdNzQ==
+-----END RSA PRIVATE KEY----- \ No newline at end of file
diff --git a/integration/scripts/files/keys/id_rsa.pub b/integration/scripts/files/keys/id_rsa.pub
new file mode 100644
index 00000000..4093a714
--- /dev/null
+++ b/integration/scripts/files/keys/id_rsa.pub
@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmQeA/uyEyFf9DsmwR+OztWb7Hb/uTC+R3xG1QgBvRwhSbpBnyBESGMZZ07bIw5Ib7BUSDzwoeryUqNAhAhir2KLeIYODS39UmTwOIl+rIvhlTxhsIoQHV90pewD2qw0T8KgVMPUDsQ0Bd98E6e5dbxciZp67ihVD0r7srhdSRo8PIc56hJWrD52j5FeiIGEmLXHXiZLOyma1M7j/EmiV81wHAzgql6sihWSZHm3xPZZ712JtXbmHhe3RLFIK13u9PSb3XbuEIdGwkZdzP+vYNE0CsYqwjXjVRrY/APsiEkbSNVzHI5p2W1L7ZMtSOMUqZ1Ve+sytVb+YcIJ9L8y07 trove@devstack \ No newline at end of file
diff --git a/integration/scripts/files/requirements/fedora-requirements-default.txt b/integration/scripts/files/requirements/fedora-requirements-default.txt
new file mode 100644
index 00000000..c976bfac
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-default.txt
@@ -0,0 +1,31 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.22.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0
+kombu>=2.5.0
+babel>=1.3
+python-heatclient>=0.3.0
+passlib
+jinja2>=2.6
+PyMySQL>=0.6.2 # MIT License
+python-neutronclient>=2.3.11,<3
+netifaces>=0.10.4
+oslo.config>=1.9.3 # Apache-2.0
+oslo.messaging>=1.8.0 # Apache-2.0
+oslo.i18n>=1.5.0 # Apache-2.0
+oslo.serialization>=1.4.0 # Apache-2.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.4.0 # Apache-2.0
+oslo.log>=1.8.0 # Apache-2.0
+osprofiler>=0.3.0
+oslo.concurrency>=1.8.0 # Apache-2.0
+pexpect>=3.1,!=3.3
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
+xmltodict>=0.10.1 # MIT
diff --git a/integration/scripts/files/requirements/fedora-requirements-juno.txt b/integration/scripts/files/requirements/fedora-requirements-juno.txt
new file mode 100644
index 00000000..a3ddea3a
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-juno.txt
@@ -0,0 +1,19 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools>=0.9.34,!=1.4.0,<=1.5.0
+extras
+python-novaclient>=2.18.0,<=2.20.0
+python-swiftclient>=2.2.0,<=2.3.1
+python-cinderclient>=1.1.0,<=1.1.1
+kombu>=2.5.0,<=3.0.7
+six>=1.7.0,<=1.9.0
+Babel>=1.3,<=1.3
+python-heatclient>=0.2.9,<0.3.0
+passlib<=1.6.2
+Jinja2<=2.7.2
+python-neutronclient>=2.3.6,<2.4.0
+netifaces>=0.10.4,<=0.10.4
+oslo.config>=1.4.0,<=1.6.0 # Apache-2.0
+oslo.utils>=1.4.0,<1.5.0 # Apache-2.0
diff --git a/integration/scripts/files/requirements/fedora-requirements-kilo.txt b/integration/scripts/files/requirements/fedora-requirements-kilo.txt
new file mode 100644
index 00000000..48bda11b
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-kilo.txt
@@ -0,0 +1,24 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools>=0.9.36,!=1.2.0
+extras
+python-novaclient>=2.22.0,<2.24.0
+python-swiftclient>=2.2.0,<2.5.0
+python-cinderclient>=1.1.0,<1.2.0
+kombu>=2.5.0
+six>=1.9.0
+Babel>=1.3
+python-heatclient>=0.3.0,<0.5.0
+passlib
+Jinja2>=2.6 # BSD License (3 clause)
+python-neutronclient>=2.3.11,<2.5.0
+netifaces>=0.10.4
+oslo.config>=1.9.3,<1.10.0 # Apache-2.0
+oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0
+oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0
+oslo.utils>=1.4.0,<1.5.0 # Apache-2.0
+oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0
+oslo.messaging>=1.8.0,<1.9.0 # Apache-2.0
+osprofiler>=0.3.0 # Apache-2.0
diff --git a/integration/scripts/files/requirements/fedora-requirements-liberty.txt b/integration/scripts/files/requirements/fedora-requirements-liberty.txt
new file mode 100644
index 00000000..f47cbada
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-liberty.txt
@@ -0,0 +1,27 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.22.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+kombu>=2.5.0
+babel>=1.3
+python-heatclient>=0.3.0
+passlib
+jinja2>=2.6
+python-neutronclient>=2.3.11,<3
+netifaces>=0.10.4
+oslo.context>=0.2.0,<=1.0.0
+oslo.config>=1.9.3,<1.10.0 # Apache-2.0
+oslo.messaging>=1.8.0 # Apache-2.0
+oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0
+oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.4.0,<1.5.0 # Apache-2.0
+oslo.log>=1.8.0 # Apache-2.0
+osprofiler>=0.3.0
+oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0
+pexpect>=3.1,!=3.3
diff --git a/integration/scripts/files/requirements/fedora-requirements-mitaka.txt b/integration/scripts/files/requirements/fedora-requirements-mitaka.txt
new file mode 100644
index 00000000..6f8c478c
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-mitaka.txt
@@ -0,0 +1,28 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.22.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+kombu>=2.5.0
+babel>=1.3
+python-heatclient>=0.3.0
+passlib
+jinja2>=2.6
+python-neutronclient>=2.3.11,<3
+netifaces>=0.10.4
+oslo.config>=1.9.3 # Apache-2.0
+oslo.messaging>=1.8.0 # Apache-2.0
+oslo.i18n>=1.5.0 # Apache-2.0
+oslo.serialization>=1.4.0 # Apache-2.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.4.0 # Apache-2.0
+oslo.log>=1.8.0 # Apache-2.0
+osprofiler>=0.3.0
+oslo.concurrency>=1.8.0 # Apache-2.0
+pexpect>=3.1,!=3.3
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
diff --git a/integration/scripts/files/requirements/fedora-requirements-newton.txt b/integration/scripts/files/requirements/fedora-requirements-newton.txt
new file mode 100644
index 00000000..c976bfac
--- /dev/null
+++ b/integration/scripts/files/requirements/fedora-requirements-newton.txt
@@ -0,0 +1,31 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.22.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0
+kombu>=2.5.0
+babel>=1.3
+python-heatclient>=0.3.0
+passlib
+jinja2>=2.6
+PyMySQL>=0.6.2 # MIT License
+python-neutronclient>=2.3.11,<3
+netifaces>=0.10.4
+oslo.config>=1.9.3 # Apache-2.0
+oslo.messaging>=1.8.0 # Apache-2.0
+oslo.i18n>=1.5.0 # Apache-2.0
+oslo.serialization>=1.4.0 # Apache-2.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.4.0 # Apache-2.0
+oslo.log>=1.8.0 # Apache-2.0
+osprofiler>=0.3.0
+oslo.concurrency>=1.8.0 # Apache-2.0
+pexpect>=3.1,!=3.3
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
+xmltodict>=0.10.1 # MIT
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-default.txt b/integration/scripts/files/requirements/ubuntu-requirements-default.txt
new file mode 100644
index 00000000..9607060b
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-default.txt
@@ -0,0 +1,30 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.18.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0
+kombu>=2.5.0
+six>=1.7.0
+babel
+python-heatclient>=0.2.9
+passlib
+jinja2
+PyMySQL>=0.6.2 # MIT License
+python-neutronclient>=2.3.6,<3
+netifaces>=0.10.4
+oslo.config>=1.4.0 # Apache-2.0
+oslo.messaging>=1.4.0,!=1.5.0
+oslo.i18n>=1.0.0
+oslo.serialization>=1.0.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.1.0
+osprofiler>=0.3.0
+oslo.concurrency>=0.3.0
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
+xmltodict>=0.10.1 # MIT
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-juno.txt b/integration/scripts/files/requirements/ubuntu-requirements-juno.txt
new file mode 100644
index 00000000..a3ddea3a
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-juno.txt
@@ -0,0 +1,19 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools>=0.9.34,!=1.4.0,<=1.5.0
+extras
+python-novaclient>=2.18.0,<=2.20.0
+python-swiftclient>=2.2.0,<=2.3.1
+python-cinderclient>=1.1.0,<=1.1.1
+kombu>=2.5.0,<=3.0.7
+six>=1.7.0,<=1.9.0
+Babel>=1.3,<=1.3
+python-heatclient>=0.2.9,<0.3.0
+passlib<=1.6.2
+Jinja2<=2.7.2
+python-neutronclient>=2.3.6,<2.4.0
+netifaces>=0.10.4,<=0.10.4
+oslo.config>=1.4.0,<=1.6.0 # Apache-2.0
+oslo.utils>=1.4.0,<1.5.0 # Apache-2.0
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt b/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt
new file mode 100644
index 00000000..48bda11b
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt
@@ -0,0 +1,24 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools>=0.9.36,!=1.2.0
+extras
+python-novaclient>=2.22.0,<2.24.0
+python-swiftclient>=2.2.0,<2.5.0
+python-cinderclient>=1.1.0,<1.2.0
+kombu>=2.5.0
+six>=1.9.0
+Babel>=1.3
+python-heatclient>=0.3.0,<0.5.0
+passlib
+Jinja2>=2.6 # BSD License (3 clause)
+python-neutronclient>=2.3.11,<2.5.0
+netifaces>=0.10.4
+oslo.config>=1.9.3,<1.10.0 # Apache-2.0
+oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0
+oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0
+oslo.utils>=1.4.0,<1.5.0 # Apache-2.0
+oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0
+oslo.messaging>=1.8.0,<1.9.0 # Apache-2.0
+osprofiler>=0.3.0 # Apache-2.0
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt b/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt
new file mode 100644
index 00000000..1f3d0d7b
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt
@@ -0,0 +1,26 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.18.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+kombu>=2.5.0
+six>=1.7.0
+babel
+python-heatclient>=0.2.9
+passlib
+jinja2
+python-neutronclient>=2.3.6,<3
+netifaces>=0.10.4
+oslo.context>=0.2.0,<=1.0.0
+oslo.config>=1.4.0 # Apache-2.0
+oslo.messaging>=1.4.0,!=1.5.0
+oslo.i18n>=1.0.0
+oslo.serialization>=1.0.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.1.0
+osprofiler>=0.3.0
+oslo.concurrency>=0.3.0
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt b/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt
new file mode 100644
index 00000000..8724d163
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt
@@ -0,0 +1,27 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.18.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+kombu>=2.5.0
+six>=1.7.0
+babel
+python-heatclient>=0.2.9
+passlib
+jinja2
+python-neutronclient>=2.3.6,<3
+netifaces>=0.10.4
+oslo.config>=1.4.0 # Apache-2.0
+oslo.messaging>=1.4.0,!=1.5.0
+oslo.i18n>=1.0.0
+oslo.serialization>=1.0.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.1.0
+osprofiler>=0.3.0
+oslo.concurrency>=0.3.0
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
diff --git a/integration/scripts/files/requirements/ubuntu-requirements-newton.txt b/integration/scripts/files/requirements/ubuntu-requirements-newton.txt
new file mode 100644
index 00000000..9607060b
--- /dev/null
+++ b/integration/scripts/files/requirements/ubuntu-requirements-newton.txt
@@ -0,0 +1,30 @@
+# The order of packages is significant, because pip processes them in the order
+# of appearance. Changing the order has an impact on the overall integration
+# process, which may cause wedges in the gate later.
+unittest2
+testtools
+extras
+python-novaclient>=2.18.0
+python-swiftclient>=2.2.0
+python-cinderclient>=1.1.0
+python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0
+kombu>=2.5.0
+six>=1.7.0
+babel
+python-heatclient>=0.2.9
+passlib
+jinja2
+PyMySQL>=0.6.2 # MIT License
+python-neutronclient>=2.3.6,<3
+netifaces>=0.10.4
+oslo.config>=1.4.0 # Apache-2.0
+oslo.messaging>=1.4.0,!=1.5.0
+oslo.i18n>=1.0.0
+oslo.serialization>=1.0.0
+oslo.service>=0.1.0 # Apache-2.0
+oslo.utils>=1.1.0
+osprofiler>=0.3.0
+oslo.concurrency>=0.3.0
+enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD
+pycrypto>=2.6 # Public Domain
+xmltodict>=0.10.1 # MIT
diff --git a/integration/scripts/files/trove-guest.systemd.conf b/integration/scripts/files/trove-guest.systemd.conf
new file mode 100644
index 00000000..47bc2cf8
--- /dev/null
+++ b/integration/scripts/files/trove-guest.systemd.conf
@@ -0,0 +1,32 @@
+[Unit]
+Description=Trove Guest
+After=syslog.target
+After=network.target
+
+[Service]
+Type=simple
+User=GUEST_USERNAME
+Group=GUEST_USERNAME
+
+ExecStartPre=/bin/bash -c "sudo mkdir -p GUEST_LOGDIR ; sudo chown GUEST_USERNAME:root GUEST_LOGDIR"
+
+# If ~/trove-installed does not exist, copy the trove source from
+# the user's development environment, then touch the sentinel file
+ExecStartPre=/bin/bash -c "test -d /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
+
+# If /etc/trove does not exist, create it and then copy the trove-guestagent.conf
+# from /etc/trove on the user's development environment,
+ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf"
+
+ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove"
+
+ExecStart=/home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d
+
+# Give a reasonable amount of time for the server to start up/shut down
+TimeoutSec=300
+
+# PgSql doesn't play nice with PrivateTmp
+PrivateTmp=false
+
+[Install]
+WantedBy=multi-user.target
diff --git a/integration/scripts/files/trove-guest.upstart.conf b/integration/scripts/files/trove-guest.upstart.conf
new file mode 100644
index 00000000..2e20d6cf
--- /dev/null
+++ b/integration/scripts/files/trove-guest.upstart.conf
@@ -0,0 +1,40 @@
+description "Trove Guest"
+author "Auto-Gen"
+
+start on (filesystem and net-device-up IFACE!=lo)
+stop on runlevel [016]
+chdir /var/run
+pre-start script
+ mkdir -p /var/run/trove
+ chown GUEST_USERNAME:root /var/run/trove/
+
+ mkdir -p /var/lock/trove
+ chown GUEST_USERNAME:root /var/lock/trove/
+
+ mkdir -p GUEST_LOGDIR
+ chown GUEST_USERNAME:root GUEST_LOGDIR
+
+ # Copy the trove source from the user's development environment
+ if [ ! -d /home/GUEST_USERNAME/trove ]; then
+ sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove
+ fi
+
+ # Ensure conf dir exists and is readable
+ mkdir -p /etc/trove/conf.d
+ chmod -R +r /etc/trove
+end script
+
+script
+ # For backwards compatibility until https://review.openstack.org/#/c/100381 merges
+ TROVE_CONFIG="--config-dir=/etc/trove/conf.d"
+ if [ ! -f /etc/trove/conf.d/guest_info ] && [ ! -f /etc/trove/conf.d/trove-guestagent.conf ]; then
+
+ chmod +r /etc/guest_info
+ sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/
+ mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/trove-guestagent.conf
+ TROVE_CONFIG="--config-file=/etc/guest_info --config-file=/etc/trove/trove-guestagent.conf"
+
+ fi
+
+ exec su -c "/home/GUEST_USERNAME/trove/contrib/trove-guestagent $TROVE_CONFIG" GUEST_USERNAME
+end script
diff --git a/integration/scripts/functions b/integration/scripts/functions
new file mode 100644
index 00000000..1a09f552
--- /dev/null
+++ b/integration/scripts/functions
@@ -0,0 +1,324 @@
+#!/bin/bash
+# This file format was stolen from devstack <3
+
+# This method was stolen from devstack
+# git clone only if directory doesn't exist already. Since ``DEST`` might not
+# be owned by the installation user, we create the directory and change the
+# ownership to the proper user.
+# Set global RECLONE=yes to simulate a clone when dest-dir exists
+# git_clone remote dest-dir branch
+function git_clone {
+ [[ "$OFFLINE" = "True" ]] && return
+
+ GIT_REMOTE=$1
+ GIT_DEST=$2
+ GIT_BRANCH=$3
+
+ if echo $GIT_BRANCH | egrep -q "^refs"; then
+ # If our branch name is a gerrit style refs/changes/...
+ if [[ ! -d $GIT_DEST ]]; then
+ git_timed clone $GIT_REMOTE $GIT_DEST
+ fi
+ cd $GIT_DEST
+ git_timed fetch $GIT_REMOTE $GIT_BRANCH && git_timed checkout FETCH_HEAD
+ else
+ # do a full clone only if the directory doesn't exist
+ if [[ ! -d $GIT_DEST ]]; then
+ git_timed clone $GIT_REMOTE $GIT_DEST
+ cd $GIT_DEST
+ # This checkout syntax works for both branches and tags
+ git_timed checkout $GIT_BRANCH
+ elif [[ "$RECLONE" == "yes" ]]; then
+ # if it does exist then simulate what clone does if asked to RECLONE
+ cd $GIT_DEST
+ # set the url to pull from and fetch
+ git_timed remote set-url origin $GIT_REMOTE
+ git_timed fetch origin
+ # remove the existing ignored files (like pyc) as they cause breakage
+ # (due to the py files having older timestamps than our pyc, so python
+ # thinks the pyc files are correct using them)
+ find $GIT_DEST -name '*.pyc' -delete
+ git_timed checkout -f origin/$GIT_BRANCH
+ # a local branch might not exist
+ git_timed branch -D $GIT_BRANCH || true
+ git_timed checkout -b $GIT_BRANCH
+ fi
+ fi
+}
+
+# Determinate is the given option present in the INI file
+# ini_has_option config-file section option
+function ini_has_option() {
+ local file=$1
+ local section=$2
+ local option=$3
+ local line
+ line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file")
+ [ -n "$line" ]
+}
+
+# Get an option from an INI file
+# iniget config-file section option
+function iniget() {
+ local file=$1
+ local section=$2
+ local option=$3
+ local line
+ line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file")
+ echo ${line#*=}
+}
+
+# Set an option in an INI file
+# iniset config-file section option value
+function iniset() {
+ local file=$1
+ local section=$2
+ local option=$3
+ local value=$4
+ if ! grep -q "^\[$section\]" "$file"; then
+ # Add section at the end
+ echo -e "\n[$section]" >>"$file"
+ fi
+ if ! ini_has_option "$file" "$section" "$option"; then
+ # Add it
+ sed -i -e "/^\[$section\]/ a\\
+$option = $value
+" "$file"
+ else
+ # Replace it
+ sed -i -e "/^\[$section\]/,/^\[.*\]/ s|^\($option[ \t]*=[ \t]*\).*$|\1$value|" "$file"
+ fi
+}
+
+# Determine OS Vendor, Release and Update
+# Tested with OS/X, Ubuntu, RedHat, CentOS, Fedora
+# Returns results in global variables:
+# os_VENDOR - vendor name
+# os_RELEASE - release
+# os_UPDATE - update
+# os_PACKAGE - package type
+# os_CODENAME - vendor's codename for release
+# GetOSVersion
+GetOSVersion() {
+ # Figure out which vendor we are
+ if [[ -x "`which sw_vers 2>/dev/null`" ]]; then
+ # OS/X
+ os_VENDOR=`sw_vers -productName`
+ os_RELEASE=`sw_vers -productVersion`
+ os_UPDATE=${os_RELEASE##*.}
+ os_RELEASE=${os_RELEASE%.*}
+ os_PACKAGE=""
+ if [[ "$os_RELEASE" =~ "10.7" ]]; then
+ os_CODENAME="lion"
+ elif [[ "$os_RELEASE" =~ "10.6" ]]; then
+ os_CODENAME="snow leopard"
+ elif [[ "$os_RELEASE" =~ "10.5" ]]; then
+ os_CODENAME="leopard"
+ elif [[ "$os_RELEASE" =~ "10.4" ]]; then
+ os_CODENAME="tiger"
+ elif [[ "$os_RELEASE" =~ "10.3" ]]; then
+ os_CODENAME="panther"
+ else
+ os_CODENAME=""
+ fi
+ elif [[ -x $(which lsb_release 2>/dev/null) ]]; then
+ os_VENDOR=$(lsb_release -i -s)
+ os_RELEASE=$(lsb_release -r -s)
+ os_UPDATE=""
+ os_PACKAGE="rpm"
+ if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then
+ os_PACKAGE="deb"
+ elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then
+ lsb_release -d -s | grep -q openSUSE
+ if [[ $? -eq 0 ]]; then
+ os_VENDOR="openSUSE"
+ fi
+ elif [[ $os_VENDOR == "openSUSE project" ]]; then
+ os_VENDOR="openSUSE"
+ elif [[ $os_VENDOR =~ Red.*Hat ]]; then
+ os_VENDOR="Red Hat"
+ fi
+ os_CODENAME=$(lsb_release -c -s)
+ elif [[ -r /etc/redhat-release ]]; then
+ # Red Hat Enterprise Linux Server release 5.5 (Tikanga)
+ # Red Hat Enterprise Linux Server release 7.0 Beta (Maipo)
+ # CentOS release 5.5 (Final)
+ # CentOS Linux release 6.0 (Final)
+ # Fedora release 16 (Verne)
+ # XenServer release 6.2.0-70446c (xenenterprise)
+ os_CODENAME=""
+ for r in "Red Hat" CentOS Fedora XenServer; do
+ os_VENDOR=$r
+ if [[ -n "`grep \"$r\" /etc/redhat-release`" ]]; then
+ ver=`sed -e 's/^.* \([0-9].*\) (\(.*\)).*$/\1\|\2/' /etc/redhat-release`
+ os_CODENAME=${ver#*|}
+ os_RELEASE=${ver%|*}
+ os_UPDATE=${os_RELEASE##*.}
+ os_RELEASE=${os_RELEASE%.*}
+ break
+ fi
+ os_VENDOR=""
+ done
+ os_PACKAGE="rpm"
+ elif [[ -r /etc/SuSE-release ]]; then
+ for r in openSUSE "SUSE Linux"; do
+ if [[ "$r" = "SUSE Linux" ]]; then
+ os_VENDOR="SUSE LINUX"
+ else
+ os_VENDOR=$r
+ fi
+
+ if [[ -n "`grep \"$r\" /etc/SuSE-release`" ]]; then
+ os_CODENAME=`grep "CODENAME = " /etc/SuSE-release | sed 's:.* = ::g'`
+ os_RELEASE=`grep "VERSION = " /etc/SuSE-release | sed 's:.* = ::g'`
+ os_UPDATE=`grep "PATCHLEVEL = " /etc/SuSE-release | sed 's:.* = ::g'`
+ break
+ fi
+ os_VENDOR=""
+ done
+ os_PACKAGE="rpm"
+ # If lsb_release is not installed, we should be able to detect Debian OS
+ elif [[ -f /etc/debian_version ]] && [[ $(cat /proc/version) =~ "Debian" ]]; then
+ os_VENDOR="Debian"
+ os_PACKAGE="deb"
+ os_CODENAME=$(awk '/VERSION=/' /etc/os-release | sed 's/VERSION=//' | sed -r 's/\"|\(|\)//g' | awk '{print $2}')
+ os_RELEASE=$(awk '/VERSION_ID=/' /etc/os-release | sed 's/VERSION_ID=//' | sed 's/\"//g')
+ fi
+ export os_VENDOR os_RELEASE os_UPDATE os_PACKAGE os_CODENAME
+}
+
+
+# Translate the OS version values into common nomenclature
+# Sets ``DISTRO`` from the ``os_*`` values
+function GetDistro() {
+ GetOSVersion
+ if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then
+ # 'Everyone' refers to Ubuntu / Debian releases by the code name adjective
+ DISTRO=$os_CODENAME
+ elif [[ "$os_VENDOR" =~ (Fedora) ]]; then
+ # For Fedora, just use 'f' and the release
+ DISTRO="f$os_RELEASE"
+ elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then
+ DISTRO="opensuse-$os_RELEASE"
+ elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then
+ # For SLE, also use the service pack
+ if [[ -z "$os_UPDATE" ]]; then
+ DISTRO="sle${os_RELEASE}"
+ else
+ DISTRO="sle${os_RELEASE}sp${os_UPDATE}"
+ fi
+ elif [[ "$os_VENDOR" =~ (Red Hat) || "$os_VENDOR" =~ (CentOS) ]]; then
+ # Drop the . release as we assume it's compatible
+ DISTRO="rhel${os_RELEASE::1}"
+ elif [[ "$os_VENDOR" =~ (XenServer) ]]; then
+ DISTRO="xs$os_RELEASE"
+ else
+ # Catch-all for now is Vendor + Release + Update
+ DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE"
+ fi
+ export DISTRO
+}
+
+
+# Determine if current distribution is a Fedora-based distribution
+# (Fedora, RHEL, CentOS, etc).
+# is_fedora
+function is_fedora {
+ if [[ -z "$os_VENDOR" ]]; then
+ GetOSVersion
+ fi
+
+ [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || [ "$os_VENDOR" = "CentOS" ]
+}
+
+
+# Determine if current distribution is a SUSE-based distribution
+# (openSUSE, SLE).
+# is_suse
+function is_suse {
+ if [[ -z "$os_VENDOR" ]]; then
+ GetOSVersion
+ fi
+
+ [ "$os_VENDOR" = "openSUSE" ] || [ "$os_VENDOR" = "SUSE LINUX" ]
+}
+
+# Get the path to the direcotry where python executables are installed.
+# get_python_exec_prefix
+function get_python_exec_prefix() {
+ if is_fedora || is_suse; then
+ echo "/usr/bin"
+ else
+ echo "/usr/local/bin"
+ fi
+}
+
+# Returns 'true' if 'true', 'yes', 'on' or '1' - false, otherwise.
+# Converts values to lower case first.
+# If no default is provided, assumes false.
+function get_bool() {
+ local VARIABLE="$1"
+ local DEFAULT=${2:-false}
+
+ VALUE=${!VARIABLE:-$DEFAULT}
+
+ VALUE=$(eval echo "$VALUE" | tr '[:upper:]' '[:lower:]')
+ if [[ "1 yes true on" =~ "$VALUE" ]]; then
+ VALUE=true
+ else
+ VALUE=false
+ fi
+
+ echo $VALUE
+}
+
+# Get the project branch to switch to. Uses PROJECT_BRANCH_NAME,
+# then BRANCH_DEFAULT, then 'master'
+function get_project_branch() {
+ local PROJECT_BRANCH_NAME=$1
+ local BRANCH_DEFAULT=${2:-master}
+
+ PROJECT_BRANCH_OVERRIDE=${!PROJECT_BRANCH_NAME}
+
+ BRANCH=${PROJECT_BRANCH_OVERRIDE:-$BRANCH_DEFAULT}
+
+ echo "$BRANCH"
+}
+
+# Try to switch to a branch or commit in a repo
+# Fails if the branch/commit doesn't exist
+function git_checkout() {
+ local PROJECT=$1
+ local REPO_DIR=$2
+ local REPO_BRANCH=$3
+ local REPO_BRANCH_VAR_NAME=$4
+
+ if [ -n "$REPO_BRANCH" ]; then
+ pushd "$REPO_DIR"
+ if [ $PROJECT == "diskimage-builder" ] || [ $PROJECT == "tripleo-image-elements" ]; then
+ REPO_BRANCH=master
+ fi
+ CURRENT_BRANCH=$(git branch | grep "\*" | awk '{print $2}')
+ GIT_STATUS=$(git checkout "$REPO_BRANCH" &> /dev/null || echo "failed")
+ if [[ "$GIT_STATUS" = "failed" ]]; then
+ exclaim "${COLOR_RED}Could not switch to branch/commit '$REPO_BRANCH' in $PROJECT, exiting${COLOR_NONE}"
+ echo "Please set '$REPO_BRANCH_VAR_NAME' to a valid branch/commit and try again."
+ if [[ "$CURRENT_BRANCH" != "master" ]]; then
+ echo "Note: This repo is currently on branch ${CURRENT_BRANCH} - if this is correct,"
+ echo "you should set $REPO_BRANCH_VAR_NAME=${CURRENT_BRANCH} and re-run your command."
+ else
+ echo "Note: This error may also mean that there are modified files in $PROJECT."
+ echo " If that is the case, please stash them and re-run your command."
+ fi
+ exit 1
+ else
+ if [[ "$REPO_BRANCH" != "$CURRENT_BRANCH" ]]; then
+ exclaim "${COLOR_BLUE}Switched to $PROJECT branch '$REPO_BRANCH'${COLOR_NONE}"
+ else
+ echo "Using $PROJECT branch '$REPO_BRANCH'"
+ fi
+ fi
+ popd
+ fi
+}
+
diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu
new file mode 100644
index 00000000..62875eec
--- /dev/null
+++ b/integration/scripts/functions_qemu
@@ -0,0 +1,174 @@
+#!/bin/bash
+#
+# Additional functions that would mostly just pertain to a Ubuntu + Qemu setup
+#
+
+function build_vm() {
+ exclaim "Actually building the image, this can take up to 15 minutes"
+
+ # set variables here and ensure they are not changed during the duration of this script
+ readonly HOMEDIR=$1
+ readonly HOST_USERNAME=$2
+ GUEST_USERNAME=${GUEST_USERNAME:-$2}
+ HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$2}
+ VM=$3
+ DISTRO=$4
+ SERVICE_TYPE=$5
+
+ readonly SSH_DIR=${KEY_DIR:-${HOMEDIR}/.ssh}
+ manage_ssh_keys
+
+ if [ $DISTRO == 'ubuntu' ]; then
+ export RELEASE=trusty
+ export DIB_RELEASE=$RELEASE
+ export DIB_CLOUD_IMAGES=cloud-images.ubuntu.com
+ # Use the apt sources.list on the build host, its almost always preferred
+ if [ -f /etc/apt/sources.list ]; then
+ export DIB_APT_SOURCES=/etc/apt/sources.list
+ EXTRA_ELEMENTS="apt-sources apt-conf-dir"
+ fi
+ fi
+ if [ $DISTRO == 'fedora' ]; then
+ EXTRA_ELEMENTS=selinux-permissive
+ fi
+
+ export HOST_USERNAME
+ export HOST_SCP_USERNAME
+ export GUEST_USERNAME
+ export CONTROLLER_IP
+ export REDSTACK_SCRIPTS
+ export SERVICE_TYPE
+ export PATH_TROVE
+ export ESCAPED_PATH_TROVE
+ export SSH_DIR
+ export GUEST_LOGDIR
+ export ESCAPED_GUEST_LOGDIR
+ export ELEMENTS_PATH=$REDSTACK_SCRIPTS/files/elements
+ export ELEMENTS_PATH+=:$PATH_DISKIMAGEBUILDER/elements
+ export ELEMENTS_PATH+=:$PATH_TRIPLEO_ELEMENTS/elements
+ export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive"
+ export DATASTORE_PKG_LOCATION
+ export BRANCH_OVERRIDE
+ export DIB_APT_CONF_DIR=/etc/apt/apt.conf.d
+ export DIB_CLOUD_INIT_ETC_HOSTS=true
+ local QEMU_IMG_OPTIONS=$(! $(qemu-img | grep -q 'version 1') && echo "--qemu-img-options compat=0.10")
+ ${PATH_DISKIMAGEBUILDER}/bin/disk-image-create -a amd64 -o "${VM}" \
+ -x ${QEMU_IMG_OPTIONS} ${DISTRO} ${EXTRA_ELEMENTS} vm heat-cfntools \
+ cloud-init-datasources ${DISTRO}-guest ${DISTRO}-${SERVICE_TYPE}
+}
+
+function build_guest_image() {
+ exclaim "Building an image for use with development and integration tests."
+ if [ -z "$1" ]
+ then
+ echo "You must pass an image type to build, like mysql"
+ exit 1
+ fi
+ SERVICE_TYPE=$1
+ VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
+ if ! [[ " $VALID_SERVICES " =~ " $SERVICE_TYPE " ]]; then
+ exclaim "You did not pass in a valid image type. Valid types are:" $VALID_SERVICES
+ exit 1
+ fi
+
+ GUEST_LOGDIR=$(iniget $PATH_TROVE/etc/trove/trove-guestagent.conf.sample DEFAULT log_dir)
+ GUEST_LOGFILE=$(iniget $PATH_TROVE/etc/trove/trove-guestagent.conf.sample DEFAULT log_file)
+
+ if [ -z $GUEST_LOGDIR ] || [ -z $GUEST_LOGFILE ]
+ then
+ exclaim "error: log_dir and log_file are required in: " $PATH_TROVE/etc/trove/trove-guestagent.conf.sample
+ exit 1
+ fi
+
+ ESCAPED_GUEST_LOGDIR=`echo $GUEST_LOGDIR | sed 's/\//\\\\\//g'`
+
+ USERNAME=`whoami`
+ # To change the distro, edit the redstack.rc file
+ readonly IMAGENAME=${DISTRO}_${SERVICE_TYPE}
+ readonly VM_PATH=$USERHOME/images/${IMAGENAME}
+ readonly VM_PATH_NAME=${VM_PATH}/${IMAGENAME}
+ mkdir -p $VM_PATH
+
+ # If the path does not exist, build it, otherwise just upload it
+ # (unless we're explicitly told to rebuild it)
+ REBUILD_IMAGE=$(echo "${REBUILD_IMAGE}" | tr '[:upper:]' '[:lower:]')
+ if [ "${REBUILD_IMAGE}" = "true" ] || [ ! -d $VM_PATH ] || [ `ls -1 $VM_PATH | wc -l` -eq '0' ]
+ then
+ if [ "${REBUILD_IMAGE}" = "true" ]
+ then
+ exclaim "Rebuilding image"
+ rm -rf "${VM_PATH}"
+ fi
+ build_vm $USERHOME $USERNAME $VM_PATH_NAME $DISTRO $SERVICE_TYPE
+ touch -c "${VM_PATH}"
+ else
+ exclaim "Found image in $VM_PATH - using the qcow2 image found here..."
+ ELEMENTS_DIR="files/elements/${DISTRO}-${SERVICE_TYPE}"
+ ELEMENTS_DIR_GUEST="files/elements/${DISTRO}-guest"
+ # Print out a warning on all the elements files that are newer than the image.
+ # Directories are not excluded as that is the only way to determine if a file
+ # has been removed.
+ # The rebuild is not automatically triggered as there are valid reasons for a
+ # new file to be present (rollback of change, inadvertent .log files present,
+ # feature half implemented, etc.).
+ IMAGE_OLD=
+ while IFS= read -r -d '' ELEMENT_FILE
+ do
+ if [ "${ELEMENT_FILE}" -nt "${VM_PATH}" ]
+ then
+ IMAGE_OLD=true
+ exclaim "${COLOR_RED}WARNING: Element file '${ELEMENT_FILE}' is newer than cached image${COLOR_NONE}"
+ fi
+ done < <(find "${ELEMENTS_DIR}" "${ELEMENTS_DIR_GUEST}" -depth -print0)
+ if [ "${IMAGE_OLD}" = "true" ]
+ then
+ exclaim "${COLOR_RED}Use ${COLOR_NONE}REBUILD_IMAGE=True${COLOR_RED} to rebuild image${COLOR_NONE}"
+ fi
+
+ fi
+}
+
+function clean_instances() {
+ LIST=`virsh -q list|awk '{print $1}'`
+ for i in $LIST; do sudo virsh destroy $i; done
+}
+
+function manage_ssh_keys() {
+ if [ -e ${SSH_DIR} ]; then
+ echo "${SSH_DIR} already exists"
+ else
+ echo "Creating ${SSH_DIR} for ${HOST_USERNAME}"
+ sudo -Hiu ${HOST_USERNAME} mkdir -m go-w -p ${SSH_DIR}
+ fi
+
+ if [ ! -f ${SSH_DIR}/id_rsa.pub ]; then
+ sudo ${PKG_MGR} ${PKG_GET_ARGS} install expect
+ generate_empty_passphrase_ssh_key ${HOST_USERNAME}
+ fi
+
+ add_host_key_to_authorizedkeys
+}
+
+function generate_empty_passphrase_ssh_key () {
+ echo "generating a empty passphrase DEV ONLY rsa key"
+ expect -c "
+spawn sudo -Hiu ${HOST_USERNAME} /usr/bin/ssh-keygen -f ${SSH_DIR}/id_rsa -q
+expect \"empty for no passphrase\"
+send \n
+expect assphrase
+send \n
+expect eof"
+}
+
+function add_host_key_to_authorizedkeys () {
+ # test to see if the host key is already in its own authorized_keys file - if not then add it. This is then later copied
+ # to the guest image
+ is_in_keyfile=`cat ${SSH_DIR}/id_rsa.pub | grep -f - ${SSH_DIR}/authorized_keys | wc -l`
+ if [ $is_in_keyfile == 0 ]; then
+ echo "Adding keyfile to authorized_keys, it does not yet exist"
+ cat ${SSH_DIR}/id_rsa.pub >> ${SSH_DIR}/authorized_keys
+ chmod 600 ${SSH_DIR}/authorized_keys
+ else
+ echo "Keyfile already exists in authorized_keys - skipping"
+ fi
+}
diff --git a/integration/scripts/image-projects-list b/integration/scripts/image-projects-list
new file mode 100644
index 00000000..76d0da45
--- /dev/null
+++ b/integration/scripts/image-projects-list
@@ -0,0 +1,3 @@
+diskimage-builder
+tripleo-image-elements
+trove
diff --git a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc
new file mode 100644
index 00000000..9b80b6c6
--- /dev/null
+++ b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc
@@ -0,0 +1,3 @@
+[[post-config|\$CINDER_CONF]]
+[DEFAULT]
+notification_driver = messagingv2
diff --git a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc
new file mode 100644
index 00000000..fe48b022
--- /dev/null
+++ b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc
@@ -0,0 +1,3 @@
+[[post-config|\$NOVA_CONF]]
+[DEFAULT]
+instance_usage_audit = True
diff --git a/integration/scripts/local.conf.d/ceilometer_services.conf.rc b/integration/scripts/local.conf.d/ceilometer_services.conf.rc
new file mode 100644
index 00000000..ce33948e
--- /dev/null
+++ b/integration/scripts/local.conf.d/ceilometer_services.conf.rc
@@ -0,0 +1,3 @@
+[[post-config|\$CEILOMETER_CONF]]
+[notification]
+store_events = True
diff --git a/integration/scripts/local.conf.d/sample.rc b/integration/scripts/local.conf.d/sample.rc
new file mode 100644
index 00000000..78753bc7
--- /dev/null
+++ b/integration/scripts/local.conf.d/sample.rc
@@ -0,0 +1,42 @@
+#
+# Files in this directory are automatically added to the devstack
+# local.conf file, between a specific set of tags.
+#
+# Filenames must end with '.rc' to be recognized; sample.rc is
+# ignored.
+#
+# A '\' is required in front of any devstack variables since all
+# .rc files are parsed first (using eval).
+#
+# Meta section headings must be included in each file, such as:
+# [[local|localrc]]
+# as the order of inserting the files is not guaranteed.
+#
+# All files are inherently included by default - to exclude a file,
+# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in redstack.rc
+# For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
+#
+# Symbolic links are followed, so additional files can be loaded
+# by placing them in an external directory and linking it in
+# local.conf.d (this should allow complete flexibility in setting
+# up testing options).
+# For Example:
+# cd /path/to/trove-integration/scripts/local.conf.d
+# ln -s $HOME/local.conf.d local.conf.d
+# cp /path/to/my_conf.rc $HOME/local.conf.d
+
+
+[[local|localrc]]
+# Put regular devstack variables under this meta section heading.
+# This section is written out to a file and sourced by devstack,
+# so it can contain logic as well.
+
+# The following section types should only contain ini file style
+# section headings and name=value pairs
+[[post-config|\$TROVE_CONF]]
+
+[[post-config|\$TROVE_TASKMANAGER_CONF]]
+
+[[post-config|\$TROVE_CONDUCTOR_CONF]]
+
+[[post-config|\$TROVE_API_PASTE_INI]]
diff --git a/integration/scripts/local.conf.d/trove_services.conf.rc b/integration/scripts/local.conf.d/trove_services.conf.rc
new file mode 100644
index 00000000..6eedc1c3
--- /dev/null
+++ b/integration/scripts/local.conf.d/trove_services.conf.rc
@@ -0,0 +1,24 @@
+[[post-config|\$TROVE_CONF]]
+[profiler]
+enabled = $ENABLE_PROFILER
+trace_sqlalchemy = $PROFILER_TRACE_SQL
+
+[[post-config|\$TROVE_TASKMANAGER_CONF]]
+[profiler]
+enabled = $ENABLE_PROFILER
+trace_sqlalchemy = $PROFILER_TRACE_SQL
+
+[[post-config|\$TROVE_CONDUCTOR_CONF]]
+[profiler]
+enabled = $ENABLE_PROFILER
+trace_sqlalchemy = $PROFILER_TRACE_SQL
+
+[[post-config|\$TROVE_GUESTAGENT_CONF]]
+[profiler]
+enabled = $ENABLE_PROFILER
+trace_sqlalchemy = $PROFILER_TRACE_SQL
+
+[[post-config|\$TROVE_API_PASTE_INI]]
+[filter:osprofiler]
+enabled = $ENABLE_PROFILER
+hmac_keys = $PROFILER_HMAC_KEYS
diff --git a/integration/scripts/local.conf.d/use_kvm.rc b/integration/scripts/local.conf.d/use_kvm.rc
new file mode 100644
index 00000000..06bc2ebc
--- /dev/null
+++ b/integration/scripts/local.conf.d/use_kvm.rc
@@ -0,0 +1,4 @@
+[[local|localrc]]
+
+# force kvm as the libvirt type.
+LIBVIRT_TYPE=kvm
diff --git a/integration/scripts/local.conf.d/use_uuid_token.rc b/integration/scripts/local.conf.d/use_uuid_token.rc
new file mode 100644
index 00000000..587a4064
--- /dev/null
+++ b/integration/scripts/local.conf.d/use_uuid_token.rc
@@ -0,0 +1,3 @@
+[[local|localrc]]
+
+KEYSTONE_TOKEN_FORMAT=UUID
diff --git a/integration/scripts/local.conf.d/using_vagrant.rc b/integration/scripts/local.conf.d/using_vagrant.rc
new file mode 100644
index 00000000..7333cd46
--- /dev/null
+++ b/integration/scripts/local.conf.d/using_vagrant.rc
@@ -0,0 +1,9 @@
+[[local|localrc]]
+
+# This is similar to code found at
+# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile
+# and seems to make instances ping'able in VirtualBox.
+FLAT_INTERFACE=eth1
+PUBLIC_INTERFACE=eth1
+FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28
+HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4`
diff --git a/integration/scripts/local.conf.rc b/integration/scripts/local.conf.rc
new file mode 100644
index 00000000..6b9eb0ad
--- /dev/null
+++ b/integration/scripts/local.conf.rc
@@ -0,0 +1,37 @@
+$TROVE_PRESENT_TAG
+# Set some arguments for devstack.
+#
+# Note: This file contains autogenerated parts.
+# All lines are removed from between the tag/end of tag
+# markers (lines with '$MARKER_TOKEN' at beginning and end) and
+# are replaced by trove-integration.
+# Edits to these sections will not persist.
+#
+# See the '$USER_OPTS_TAG' section
+# for ways to insert user args into this file.
+#
+
+#
+# This section is for things that belong in localrc
+# It comes from $DEFAULT_LOCALRC
+#
+[[local|localrc]]
+
+$LOCALRC_OPTS_TAG
+$LOCALRC_OPTS_TAG_END
+
+#
+# User options here were inserted from the file USER_LOCAL_CONF
+# (defaults to $USERHOME/.$LOCAL_CONF)
+#
+
+$USER_OPTS_TAG
+$USER_OPTS_TAG_END
+
+#
+# Additional options here were inserted by trove-integration
+# automatically from files in $LOCAL_CONF_D
+#
+
+$ADD_OPTS_TAG
+$ADD_OPTS_TAG_END
diff --git a/integration/scripts/localrc.rc b/integration/scripts/localrc.rc
new file mode 100644
index 00000000..94c598ee
--- /dev/null
+++ b/integration/scripts/localrc.rc
@@ -0,0 +1,100 @@
+# These passwords originally come from redstack.rc.
+MYSQL_PASSWORD=$MYSQL_PASSWORD
+RABBIT_PASSWORD=$RABBIT_PASSWORD
+SERVICE_TOKEN=$SERVICE_TOKEN
+ADMIN_PASSWORD=$ADMIN_PASSWORD
+SERVICE_PASSWORD=$SERVICE_PASSWORD
+
+PUBLIC_INTERFACE=eth0
+TROVE_LOGDIR=$TROVE_LOGDIR
+TROVE_AUTH_CACHE_DIR=$TROVE_AUTH_CACHE_DIR
+
+# Enable the Trove plugin for devstack
+if [[ $USE_DEVSTACK_TROVE_PLUGIN = true ]]; then
+ enable_plugin trove $TROVE_REPO $TROVE_BRANCH
+fi
+
+# Enable Trove, Swift, and Heat
+ENABLED_SERVICES+=,trove,tr-api,tr-tmgr,tr-cond
+ENABLED_SERVICES+=,s-proxy,s-object,s-container,s-account
+ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng
+ENABLED_SERVICES+=,-n-novnc,-n-xvnc
+if [[ $ENABLE_NEUTRON = true ]]; then
+ ENABLED_SERVICES+=,neutron,q-svc,q-agt,q-dhcp,q-l3,q-meta
+ disable_service n-net
+else
+ enable_service n-net
+ disable_service neutron q-svc q-agt q-dhcp q-l3 q-meta
+fi
+# OSprofiler depends on Ceilometer
+if [[ $ENABLE_CEILOMETER = true ]] || [[ $ENABLE_PROFILER = true ]]; then
+ CEILOMETER_BACKEND=mysql
+ CEILOMETER_NOTIFICATION_TOPICS=notifications,profiler
+ PROJ_BRANCH=$(get_project_branch CEILOMETER_BRANCH $PROJECT_BRANCH)
+ enable_plugin ceilometer ${GIT_OPENSTACK}/ceilometer.git $PROJ_BRANCH
+fi
+
+# Enable Mistral, if configured
+if [[ $ENABLE_MISTRAL = true ]]; then
+ PROJ_BRANCH=$(get_project_branch MISTRAL_BRANCH $PROJECT_BRANCH)
+ enable_plugin mistral ${GIT_OPENSTACK}/mistral.git $PROJ_BRANCH
+fi
+
+# Use Git repositories for client components
+LIBS_FROM_GIT=python-troveclient
+if [[ $LIBS_FROM_GIT_ALL_CLIENTS = true ]]; then
+ LIBS_FROM_GIT+=,python-ceilometerclient
+ LIBS_FROM_GIT+=,python-cinderclient
+ LIBS_FROM_GIT+=,python-glanceclient
+ LIBS_FROM_GIT+=,python-heatclient
+ LIBS_FROM_GIT+=,python-keystoneclient
+ LIBS_FROM_GIT+=,python-mistralclient
+ LIBS_FROM_GIT+=,python-neutronclient
+ LIBS_FROM_GIT+=,python-novaclient
+ LIBS_FROM_GIT+=,python-openstackclient
+ LIBS_FROM_GIT+=,python-swiftclient
+else
+ if [[ $ENABLE_NEUTRON = true ]]; then
+ LIBS_FROM_GIT+=,python-neutronclient
+ fi
+ if [[ $ENABLE_MISTRAL = true ]]; then
+ LIBS_FROM_GIT+=,python-mistralclient
+ fi
+ if [[ $ENABLE_CEILOMETER = true ]]; then
+ LIBS_FROM_GIT+=,python-ceilometerclient
+ fi
+fi
+# Add Git repositories for libraries
+if [[ $LIBS_FROM_GIT_ALL_OSLO = true ]]; then
+ LIBS_FROM_GIT+=,cliff,futurist
+ LIBS_FROM_GIT+=,debtcollector,automaton
+ LIBS_FROM_GIT+=,oslo.cache,oslo.concurrency
+ LIBS_FROM_GIT+=,oslo.config,oslo.context
+ LIBS_FROM_GIT+=,oslo.db,oslo.i18n
+ LIBS_FROM_GIT+=,oslo.log,oslo.messaging
+ LIBS_FROM_GIT+=,oslo.middleware,oslo.policy
+ LIBS_FROM_GIT+=,oslo.reports,oslo.rootwrap
+ LIBS_FROM_GIT+=,oslo.serialization,oslo.service
+ LIBS_FROM_GIT+=,oslo.utils,oslo.versionedobjects
+ LIBS_FROM_GIT+=,oslo.vmware
+ LIBS_FROM_GIT+=,pycadf,stevedore
+ LIBS_FROM_GIT+=,taskflow,tooz
+ LIBS_FROM_GIT+=,pbr
+fi
+
+NOVNC_FROM_PACKAGE=false
+SWIFT_HASH=$SWIFT_HASH
+# Set Cinder Volume from Redstack so that later Redstack can help manage
+# reconnecting Volume Group to Backing File
+DEST=$DEST
+DATA_DIR=$DATA_DIR
+SERVICE_DIR=$SERVICE_DIR
+VOLUME_GROUP=${VOLUME_GROUP}
+VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE}
+VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE}
+# The lock_path is by default /opt/stack/nova; if this path is a shared
+# folder in VirtualBox things seem to break. We fix it by setting EXTRA_OPS
+# to force lock_path to /tmp.
+EXTRA_OPTS=(lock_path=$USERHOME/nova_locks rescan_timeout=180 resizefs_timeout=240 force_dhcp_release=False)
+UNDO_REQUIREMENTS=False
+
diff --git a/integration/scripts/projects-list b/integration/scripts/projects-list
new file mode 100644
index 00000000..eacd4d8e
--- /dev/null
+++ b/integration/scripts/projects-list
@@ -0,0 +1,12 @@
+keystone
+nova
+glance
+horizon
+swift
+neutron
+heat
+python-openstackclient
+python-novaclient
+python-troveclient
+python-neutronclient
+python-heatclient
diff --git a/integration/scripts/redstack b/integration/scripts/redstack
new file mode 100755
index 00000000..105aceb9
--- /dev/null
+++ b/integration/scripts/redstack
@@ -0,0 +1,1433 @@
+#!/usr/bin/env bash
+###############################################################################
+# RedStack, the Trove Dev Machine Controller #
+###############################################################################
+# #
+# This script provides all the functionality to run all the steps from #
+# setting up the environment, resetting the nova database to running the #
+# test. #
+# #
+###############################################################################
+
+REDSTACK_SCRIPTS=${REDSTACK_SCRIPTS:-`pwd`}
+REDSTACK_TESTS=$REDSTACK_SCRIPTS/../tests/
+
+DEFAULT_LOCAL_CONF=local.conf.rc
+DEFAULT_LOCALRC=localrc.rc
+LOCAL_CONF=local.conf
+LOCALRC=localrc
+LOCALRC_AUTO=.localrc.auto
+USER_LOCAL_CONF_NAME=.devstack.$LOCAL_CONF
+
+USERHOME=$HOME
+# Load options not checked into VCS.
+if [ -f $USERHOME/.redstack.options.rc ]; then
+ . $USERHOME/.redstack.options.rc
+fi
+if [ -f $REDSTACK_SCRIPTS/options.rc ]; then
+ . $REDSTACK_SCRIPTS/options.rc
+fi
+
+# NOTE(mriedem): The gate-trove-functional-dsvm-* job config in project-config
+# sets this value for Jenkins runs.
+BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default}
+if [[ $BRANCH_OVERRIDE == "default" && $OVERRIDE_ZUUL_BRANCH != "master" ]]; then
+ BRANCH_OVERRIDE=$OVERRIDE_ZUUL_BRANCH
+fi
+
+# Bail on errors.
+set -e
+
+# Get default host ip from interface
+function get_default_host_ip() {
+ host_iface=$(ip route | sed -n '/^default/{ s/.*dev \(\w\+\)\s\+.*/\1/; p; }' | head -1)
+ echo `LC_ALL=C ip -f inet addr show ${host_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}' | head -1`
+}
+
+# Load functions devstack style
+. $REDSTACK_SCRIPTS/functions
+. $REDSTACK_SCRIPTS/functions_qemu
+
+# Load global configuration variables.
+. $REDSTACK_SCRIPTS/redstack.rc
+. $REDSTACK_SCRIPTS/reviews.rc
+
+# allow overrides from devstack if already set
+[[ -f $PATH_DEVSTACK_SRC/functions-common ]] && source $PATH_DEVSTACK_SRC/functions-common
+[[ -f $PATH_DEVSTACK_SRC/functions ]] && source $PATH_DEVSTACK_SRC/functions
+
+# Source the old-style localrc, or new-style .local.auto - only one should exist.
+# Note: The devstack localrc's have references to 'enable_plugin' which causes
+# errors when sourcing them in the stable/juno and stable/kilo branches.
+# These errors are safe to ignore when sourcing these files.
+set +e
+[[ -f $PATH_DEVSTACK_SRC/$LOCALRC ]] && source $PATH_DEVSTACK_SRC/$LOCALRC
+[[ -f $PATH_DEVSTACK_SRC/$LOCALRC_AUTO ]] && source $PATH_DEVSTACK_SRC/$LOCALRC_AUTO
+set -e
+
+# Set up variables for the CONF files - this has to happen after loading redstack.rc, since
+# TROVE_CONF_DIR is defined there - these will be used by devstack too
+export TROVE_CONF=$TROVE_CONF_DIR/trove.conf
+export TROVE_TASKMANAGER_CONF=$TROVE_CONF_DIR/trove-taskmanager.conf
+export TROVE_CONDUCTOR_CONF=$TROVE_CONF_DIR/trove-conductor.conf
+export TROVE_GUESTAGENT_CONF=$TROVE_CONF_DIR/trove-guestagent.conf
+export TROVE_API_PASTE_INI=$TROVE_CONF_DIR/api-paste.ini
+export TEST_CONF=$TROVE_CONF_DIR/test.conf
+
+# Public facing bits
+SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http}
+NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0}
+NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24}
+NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
+BRIDGE_IP=${BRIDGE_IP:-172.24.4.1}
+KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
+KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
+KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
+GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292}
+GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-http}
+TROVE_AUTH_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0
+
+# The following depends on whether neutron is used or nova-network
+# neutron uses a bridge, nova-network does not
+[[ $ENABLE_NEUTRON = true ]] && CONTROLLER_IP=$BRIDGE_IP || CONTROLLER_IP=$NETWORK_GATEWAY
+
+# PATH_TROVE more than likely has file separators, which sed does not like
+# This will escape them
+ESCAPED_PATH_TROVE=$(echo $PATH_TROVE | sed 's/\//\\\//g')
+ESCAPED_REDSTACK_SCRIPTS=$(echo $REDSTACK_SCRIPTS | sed 's/\//\\\//g')
+TROVE_AUTH_CACHE_DIR=${TROVE_AUTH_CACHE_DIR:-/var/cache/trove}
+TROVE_LOGDIR=${TROVE_LOGDIR:-$DEST/logs}
+# DATASTORE_PKG_LOCATION defines the location from where the datastore packages
+# can be accessed by the DIB elements. This is applicable only for datastores
+# that do not have a public repository from where their packages can be accessed.
+# This can either be a url to a private repository or a location on the local
+# filesystem that contains the datastore packages.
+DATASTORE_PKG_LOCATION=${DATASTORE_PKG_LOCATION:-}
+
+# Support entry points installation of console scripts
+if [[ -d $PATH_TROVE/bin ]]; then
+ TROVE_BIN_DIR=$PATH_TROVE/bin
+else
+ TROVE_BIN_DIR=$(get_python_exec_prefix)
+fi
+
+# Allow sourcing config values from env.rc for overrides
+if [ -f /tmp/integration/env.rc ]; then
+ source /tmp/integration/env.rc
+fi
+
+# set up respective package managers
+if is_fedora; then
+ PKG_INSTALL_OPTS=""
+ PKG_MGR=dnf
+ PKG_GET_ARGS="-y"
+else
+ PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive"
+ PKG_MGR=apt-get
+ PKG_GET_ARGS="-y --allow-unauthenticated --force-yes"
+fi
+PKG_INSTALL_ARG="install"
+PKG_UPDATE_ARG="update"
+
+###############################################################################
+# Utility functions
+###############################################################################
+
+# Colors that can be used in 'exclaim'
+COLOR_RED='\033[0;31m'
+COLOR_GREEN='\033[0;32m'
+COLOR_BLUE='\033[0;34m'
+COLOR_NONE='\033[0m'
+
+function exclaim () {
+ echo "*******************************************************************************"
+ echo -e "$@"
+ echo "*******************************************************************************"
+}
+
+# Set the location of the Trove setup commands file for devstack - either in
+# the devstack repo, or the Trove one. Also sets the flag which activates the
+# devstack trove plugin, if required. We'll use the devstack version if it
+# exists, otherwise we assume the plugin method.
+function set_trove_plugin_vars () {
+ FAIL_IF_MISSING=${1:-true}
+
+ TROVE_SETUP_CMD_FILE="$PATH_DEVSTACK_SRC/lib/trove"
+ if [ -f "$TROVE_SETUP_CMD_FILE" ]; then
+ USE_DEVSTACK_TROVE_PLUGIN=false
+ else
+ TROVE_SETUP_CMD_FILE="$DEST/trove/devstack/plugin.sh"
+ USE_DEVSTACK_TROVE_PLUGIN=true
+ fi
+ # Only complain if we have a devstack directory and are told to
+ if [ "$FAIL_IF_MISSING" = "true" ] && [ -d "$PATH_DEVSTACK_SRC" ] && [ ! -f "$TROVE_SETUP_CMD_FILE" ]; then
+ exclaim "${COLOR_RED}Trove setup file '${TROVE_SETUP_CMD_FILE}' not found!${COLOR_NONE}"
+ exit 1
+ fi
+}
+
+function pkg_install () {
+ echo Installing $@...
+ sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_INSTALL_ARG $@
+}
+
+function pkg_update () {
+ echo Updating $@...
+ sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_UPDATE_ARG $@
+}
+
+function set_home_dir() {
+ exclaim "set_home_dir has been nooped."
+ exit 1
+}
+
+function set_http_proxy() {
+ if [ ! "${http_proxy}" = '' ]; then
+ HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy"
+ fi
+}
+
+function get_ip_for_device() {
+ /sbin/ifconfig $1 | awk '/inet addr/{gsub(/addr:/,"");print $2}'
+}
+
+function ip_chunk() {
+ # Given 1-4 returns a bit of where the ip range starts.
+ # Full IP= `ip_chunk 1`.`ip_chunk 2`.`ip_chunk 3`.`ip_chunk 4`
+ get_ip_for_device $1 | cut -d. -f$2
+}
+
+# Add a flavor and a corresponding flavor.resize
+# (flavor.resize adds 16 to the memory and one more vcpu)
+function add_flavor() {
+ local FLAVOR_NAME=$1
+ local FLAVOR_ID=$2
+ local FLAVOR_MEMORY_MB=$3
+ local FLAVOR_ROOT_GB=$4
+ local FLAVOR_VCPUS=$5
+
+ credentials="--os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT"
+ if [[ -z "$FLAVOR_LIST_FOR_ADD" ]]; then
+ FLAVOR_LIST_FOR_ADD=$(nova $credentials flavor-list | cut -d'|' -f3 | sed -e's/ /,/g')
+ fi
+
+ base_id=${FLAVOR_ID}
+ base_name_prefix=test
+ ephemeral_name_prefix=${base_name_prefix}.eph
+ for name_prefix in $base_name_prefix $ephemeral_name_prefix; do
+ reg_name=${name_prefix}.${FLAVOR_NAME}-${FLAVOR_ROOT_GB}
+ resize_name=${reg_name}.resize
+ ephemeral=0
+ if [[ $name_prefix == $ephemeral_name_prefix ]]; then
+ ephemeral=1
+ fi
+ for name in ${reg_name} ${resize_name}; do
+ id=$base_id
+ memory=${FLAVOR_MEMORY_MB}
+ vcpus=${FLAVOR_VCPUS}
+ if [[ $ephemeral != 0 ]]; then
+ if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
+ id=1${id}
+ else
+ id=${id}e
+ fi
+ fi
+ if [[ $name == ${resize_name} ]]; then
+ if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
+ id=2${id}
+ else
+ id=${id}r
+ fi
+ memory=$((${FLAVOR_MEMORY_MB} + 16))
+ vcpus=$((${FLAVOR_VCPUS} + 1))
+ fi
+ if [[ $FLAVOR_LIST_FOR_ADD != *",$name,"* ]]; then
+ nova $credentials flavor-create $name $id $memory $FLAVOR_ROOT_GB $vcpus --ephemeral $ephemeral
+ fi
+ done
+ done
+}
+
+function get_attribute_id() {
+ openstack --os-auth-url=$TROVE_AUTH_ENDPOINT --os-username=admin --os-password=$ADMIN_PASSWORD --os-project-name=admin $1 list | grep " $2" | get_field $3
+}
+
+
+###############################################################################
+# Install all the required dependencies
+###############################################################################
+
+function install_prep_packages() {
+ # Called before devstack
+ exclaim 'Updating dependencies (part 1a)...'
+ pkg_update
+ exclaim 'Installing dependencies (part 1b)...'
+ pkg_install python-pip
+ if is_fedora; then
+ pkg_install git gettext
+ else
+ pkg_install git-core kvm-ipxe gettext
+ fi
+ sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
+}
+
+function install_devstack_code() {
+ exclaim "Installing devstack..."
+ # Installs devstack (if needed).
+ if [ ! -d $PATH_DEVSTACK_SRC ]; then
+ echo "DevStack not in a shared folder, cloning from git."
+ mkdir -p $PATH_DEVSTACK_SRC
+ git clone $DEVSTACK_REPO $PATH_DEVSTACK_SRC
+ fi
+
+ source $PATH_DEVSTACK_SRC/functions-common
+ source $PATH_DEVSTACK_SRC/functions
+
+ # Switch to a branch if specified. The order the variables are checked is:
+ # DEVSTACK_BRANCH then PROJECT_BRANCH
+ BRANCH_SPECIFIED=$(test -z "${DEVSTACK_BRANCH}${PROJECT_BRANCH}" || echo 'True')
+ if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then
+ PROJ_BRANCH=$(get_project_branch DEVSTACK_BRANCH $PROJECT_BRANCH)
+ ENV_VARS="DEVSTACK_BRANCH' or 'PROJECT_BRANCH"
+ git_checkout "devstack" "$PATH_DEVSTACK_SRC" "$PROJ_BRANCH" "$ENV_VARS"
+ fi
+
+ exclaim "Installing devstack projects..."
+ # Ensures present user can get to the devstack dirs
+ sudo mkdir -p $PATH_DEVSTACK_OUTPUT
+ if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then
+ sudo chown `whoami` $PATH_DEVSTACK_OUTPUT
+ fi
+ # Clones all of the code to where devstack expects it to be
+ pushd $PATH_DEVSTACK_OUTPUT
+ cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/projects-list \
+ $REDSTACK_SCRIPTS/image-projects-list
+ popd
+}
+
+function install_reviews_on_top_of_devstack() {
+ exclaim "Putting gerrit review code on top of the existing devstack code"
+ run_review_for nova $PATH_NOVA $REVIEW_NOVA
+ run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT
+ run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE
+ run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT
+ run_review_for python-openstackclient $PATH_OPENSTACKCLIENT $REVIEW_PYTHON_OPENSTACKCLIENT
+ run_review_for glance $PATH_GLANCE $REVIEW_GLANCE
+ run_review_for swift $PATH_SWIFT $REVIEW_SWIFT
+ run_review_for python-swiftclient $PATH_PYTHON_SWIFTCLIENT $REVIEW_PYTHON_SWIFTCLIENT
+ run_review_for trove $PATH_TROVE $REVIEW_TROVE
+ run_review_for python-troveclient $PATH_PYTHON_TROVECLIENT $REVIEW_PYTHON_TROVECLIENT
+}
+
+function run_review_for() {
+ # Splits based on colon in the REVIEW_ARG and pulls from
+ GIT_NAME=$1
+ PATH_ARG=$2
+ REVIEW_ARG=$3
+ for review in `echo $REVIEW_ARG| tr ":" "\n"`
+ do
+ # This should be the ref spec for what we pull
+ pushd $PATH_ARG
+ git_timed pull https://review.openstack.org/p/openstack/$GIT_NAME refs/changes/$review
+ popd
+ done
+}
+
+function fixup_broken_devstack() {
+ # Nothing to do here, devstack is working
+ :
+}
+
+# Delete all the lines from FILE_NAME between START_TAG and END_TAG
+# Tags must appear at the beginning of a line
+function clear_file_lines() {
+ local FILE_NAME=$1
+ local START_TAG=$2
+ local END_TAG=$3
+
+ sed -i "/^$START_TAG$/,/^$END_TAG$/{/^$START_TAG/!{/^$END_TAG/!d;}}" "$FILE_NAME"
+}
+
+# Checks to see if a variable with the same name as FILE_NAME exists.
+# Returns 'true' if no varable exists or if the value of the variable
+# is set to 'true' - returns the VAR_NAME to set otherwise.
+# FILE_NAME is first converted to uppercase, the extension is removed
+# and all remaining '.' and spaces are replaced with '_'
+function check_filename_var() {
+ local FILE_NAME=$1
+
+ DEREF_VALUE=false
+ if [ -f "$FILE_NAME" ]; then
+ VAR_NAME=$(basename "$FILE_NAME" ".rc" | tr '[:lower:][:blank:][:punct:]' '[:upper:]__')
+ DEREF_VALUE=$(get_bool "$VAR_NAME" "true")
+ if [ "$DEREF_VALUE" != "true" ]; then
+ DEREF_VALUE=$VAR_NAME
+ fi
+ fi
+ echo "$DEREF_VALUE"
+}
+
+# Add the contents of one file to another, after the given tag
+# Run through 'eval' if PARSE_FILE is true (defaults to true)
+# Start with a blank line if BLANK_LINE_TO_START is true (defaults to false)
+function add_file_contents() {
+ local FILE_NAME=$1
+ local FILE_TO_ADD=$2
+ local TAG=$3
+ local PARSE_FILE=${4:-true}
+ local BLANK_LINE_TO_START=${5:-false}
+
+ TEMP_FILE=".redstack.$$"
+ rm -f "$TEMP_FILE"
+ if [ "$BLANK_LINE_TO_START" = "true" ]; then
+ echo "" > "$TEMP_FILE"
+ fi
+ if [ -f "$FILE_TO_ADD" ]; then
+ echo "Adding $FILE_TO_ADD to $FILE_NAME"
+ echo "# Contents from $FILE_TO_ADD" >> "$TEMP_FILE"
+ if [ "$PARSE_FILE" = "true" ]; then
+ eval echo "\"$(cat "$FILE_TO_ADD")\"" >> "$TEMP_FILE"
+ else
+ cat "$FILE_TO_ADD" >> "$TEMP_FILE"
+ fi
+ echo "# End Of Contents from $FILE_TO_ADD" >> "$TEMP_FILE"
+ fi
+ echo "" >> "$TEMP_FILE"
+ sed -i "/^$TAG/r $TEMP_FILE" "$FILE_NAME"
+ rm -f "$TEMP_FILE"
+}
+
+function run_devstack() {
+ exclaim "Running devstack..."
+
+ # (Re)Creating this lock directory seems sure-fire.
+ rm -rf "$USERHOME/nova_locks"
+ mkdir -p "$USERHOME/nova_locks"
+
+ TROVE_PRESENT_TAG="# Trove-integration"
+ LOCAL_CONF_D=local.conf.d
+ CONF_MATCH="*.rc"
+ MARKER_TOKEN="#####"
+ USER_LOCAL_CONF=$(readlink -f "${USER_LOCAL_CONF:-$USERHOME/$USER_LOCAL_CONF_NAME}")
+ LOCALRC_OPTS_TAG="$MARKER_TOKEN Redstack Localrc Options $MARKER_TOKEN"
+ LOCALRC_OPTS_TAG_END="$MARKER_TOKEN End Of Redstack Localrc Options $MARKER_TOKEN"
+ USER_OPTS_TAG="$MARKER_TOKEN User Specified Options $MARKER_TOKEN"
+ USER_OPTS_TAG_END="$MARKER_TOKEN End Of User Specified Options $MARKER_TOKEN"
+ ADD_OPTS_TAG="$MARKER_TOKEN Additional Options $MARKER_TOKEN"
+ ADD_OPTS_TAG_END="$MARKER_TOKEN End Of Additional Options $MARKER_TOKEN"
+ set_trove_plugin_vars false
+
+ pushd "$PATH_DEVSTACK_SRC"
+ DEVSTACK_LOCAL_CONF=$LOCAL_CONF
+ # remain backwards compatible with existing localrc files
+ if [ -f "$LOCALRC" ]; then
+ DEVSTACK_LOCAL_CONF=$LOCALRC
+ echo "Old-style devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF found."
+ echo "Consider removing to generate the preferred-sytle config file $LOCAL_CONF."
+ fi
+ if [ -f "$DEVSTACK_LOCAL_CONF" ]; then
+ # Check if we have already configured the devstack config file
+ already_in_conf=$(grep "$TROVE_PRESENT_TAG" "$DEVSTACK_LOCAL_CONF" | wc -l)
+ if [ "$already_in_conf" == 0 ]; then
+ # We can no longer append to an existing old-style localrc file
+ if [ "$DEVSTACK_LOCAL_CONF" == "$LOCALRC" ]; then
+ echo "The devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF is too old to append to."
+ echo "Please remove and try again."
+ exit 1
+ fi
+ # Otherwise append the redstack version to the existing file
+ eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC")\"" >> "$DEVSTACK_LOCAL_CONF"
+ fi
+ else
+ # If a devstack config file doesn't exist, create it
+ eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCAL_CONF")\"" > "$DEVSTACK_LOCAL_CONF"
+ fi
+
+ # We can only replace sections from the LOCAL_CONF style files
+ if [ "$DEVSTACK_LOCAL_CONF" == "$LOCAL_CONF" ]; then
+ # Clear out all the options
+ clear_file_lines "$DEVSTACK_LOCAL_CONF" "$LOCALRC_OPTS_TAG" "$LOCALRC_OPTS_TAG_END"
+ clear_file_lines "$DEVSTACK_LOCAL_CONF" "$USER_OPTS_TAG" "$USER_OPTS_TAG_END"
+ clear_file_lines "$DEVSTACK_LOCAL_CONF" "$ADD_OPTS_TAG" "$ADD_OPTS_TAG_END"
+
+ # Add the main localrc file
+ PARSE_FILE="true"
+ BLANK_LINE_TO_START="true"
+ if [ -f "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" ]; then
+ add_file_contents "$DEVSTACK_LOCAL_CONF" "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" "$LOCALRC_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
+ fi
+
+ # Add any user options
+ PARSE_FILE="false"
+ BLANK_LINE_TO_START="true"
+ if [ -f "$USER_LOCAL_CONF" ]; then
+ add_file_contents "$DEVSTACK_LOCAL_CONF" "$USER_LOCAL_CONF" "$USER_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
+ fi
+
+ # Add all the files in the LOCAL_CONF_D directory that match CONF_MATCH (except for sample files)
+ # and that aren't excluded. Files are excluded by having a variable
+ # 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in redstack.rc
+ # For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
+ PARSE_FILE="true"
+ BLANK_LINE_TO_START="false"
+ while IFS= read -r -d '' CONF_FILE
+ do
+ FILE_NAME_VAR=$(check_filename_var "$CONF_FILE")
+ if [ "$FILE_NAME_VAR" = "true" ]; then
+ add_file_contents "$DEVSTACK_LOCAL_CONF" "$CONF_FILE" "$ADD_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START"
+ else
+ echo "Skipping $CONF_FILE"
+ echo "Use $FILE_NAME_VAR=true to include"
+ fi
+ done < <(find "$REDSTACK_SCRIPTS/${LOCAL_CONF_D}" -name "${CONF_MATCH}" -follow -not -name "sample*.rc" -type f -print0)
+ # this is to add a blank line for readability
+ add_file_contents "$DEVSTACK_LOCAL_CONF" "" "$ADD_OPTS_TAG"
+ fi
+ ./stack.sh
+ popd
+}
+
+
+function cmd_install() {
+ sudo mkdir -p $TROVE_LOGDIR # Creates TROVE_LOGDIR if it does not exist
+ if [ ! -w $TROVE_LOGDIR ]; then
+ sudo chown `whoami` $TROVE_LOGDIR
+ fi
+ install_prep_packages
+ install_devstack_code
+ install_reviews_on_top_of_devstack
+ fixup_broken_devstack
+ run_devstack
+ exclaim "${COLOR_GREEN}FINISHED INSTALL${COLOR_NONE}"
+}
+
+
+###############################################################################
+# Build the image
+# see functions_qemu
+###############################################################################
+
+# Grab a numbered field from python prettytable output
+# Fields are numbered starting with 1
+# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc.
+# get_field field-number
+function get_field() {
+ while read data; do
+ if [ "$1" -lt 0 ]; then
+ field="(\$(NF$1))"
+ else
+ field="\$$(($1 + 1))"
+ fi
+ echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}"
+ done
+}
+
+function get_glance_id () {
+ echo `$@ | grep ' id ' | get_field 2`
+}
+
+function set_bin_path() {
+ if is_fedora; then
+ sed -i "s|%bin_path%|/usr/bin|g" $TEST_CONF
+ else
+ sed -i "s|%bin_path%|/usr/local/bin|g" $TEST_CONF
+ fi
+}
+
+function set_mysql_pkg() {
+ if is_fedora; then
+ MYSQL_PKG="mysql-community-server"
+ else
+ MYSQL_PKG="mysql-server-5.6"
+ fi
+}
+
+
+function cmd_set_datastore() {
+ IMAGEID=$1
+ DATASTORE_TYPE=$2
+ # rd_manage datastore_update <datastore_name> <default_version>
+ rd_manage datastore_update "$DATASTORE_TYPE" ""
+ PACKAGES=${PACKAGES:-""}
+ if [ "$DATASTORE_TYPE" == "mysql" ]; then
+ set_mysql_pkg
+ PACKAGES=${PACKAGES:-$MYSQL_PKG}
+ VERSION="5.6"
+ elif [ "$DATASTORE_TYPE" == "percona" ]; then
+ PACKAGES=${PACKAGES:-"percona-server-server-5.6"}
+ VERSION="5.6"
+ elif [ "$DATASTORE_TYPE" == "pxc" ]; then
+ PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"}
+ VERSION="5.6"
+ elif [ "$DATASTORE_TYPE" == "mariadb" ]; then
+ PACKAGES=${PACKAGES:-"mariadb-server"}
+ VERSION="10.1"
+ elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
+ PACKAGES=${PACKAGES:-"mongodb-org"}
+ VERSION="3.2"
+ elif [ "$DATASTORE_TYPE" == "redis" ]; then
+ PACKAGES=${PACKAGES:-"redis-server"}
+ VERSION="3.0"
+ elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
+ PACKAGES=${PACKAGES:-"cassandra"}
+ VERSION="2.1.0"
+ elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
+ PACKAGES=${PACKAGES:-"couchbase-server"}
+ VERSION="2.2.0"
+ elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
+ PACKAGES=${PACKAGES:-"postgresql-9.4"}
+ VERSION="9.4"
+ elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
+ PACKAGES=${PACKAGES:-"couchdb"}
+ VERSION="1.6.1"
+ elif [ "$DATASTORE_TYPE" == "vertica" ]; then
+ PACKAGES=${PACKAGES:-"vertica"}
+ VERSION="7.1"
+ elif [ "$DATASTORE_TYPE" == "db2" ]; then
+ PACKAGES=${PACKAGES:-""}
+ VERSION="10.5"
+ else
+ echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
+ exit 1
+ fi
+
+ sed -i "s/%datastore_type%/$DATASTORE_TYPE/g" $TEST_CONF
+ sed -i "s/%datastore_version%/$VERSION/g" $TEST_CONF
+
+ #rd_manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
+ rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
+ rd_manage datastore_version_update "$DATASTORE_TYPE" "inactive_version" "manager1" $IMAGEID "" 0
+ rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
+ rd_manage datastore_update Test_Datastore_1 ""
+
+ if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
+ # add the configuration parameters to the database for the kick-start datastore
+ rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
+ fi
+
+ cmd_stop
+ iniset $TROVE_CONF DEFAULT default_datastore "$DATASTORE_TYPE"
+ sleep 1.5
+ cmd_start
+}
+
+###############################################################################
+# Run Unit Tests
+###############################################################################
+
+function cmd_unit_tests() {
+ exclaim "Running Trove Unit Tests..."
+ $PATH_TROVE/run_tests.sh -N
+}
+
+###############################################################################
+# Start various OpenStack daemons interactively in a screen session
+###############################################################################
+
+function cmd_start_deps() {
+ if ! sudo vgs $VOLUME_GROUP; then
+ exclaim "Reconnecting Volume Group to Backing File"
+ sudo losetup -f --show ${VOLUME_BACKING_FILE}
+ fi
+
+ if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then
+ exclaim "Re-mounting Swift Disk Image"
+ sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1 || true
+ fi
+
+ if [[ -e $PATH_DEVSTACK_SRC/stack-screenrc ]]; then
+ screen -dmS stack -c $PATH_DEVSTACK_SRC/stack-screenrc
+ fi
+}
+
+function cmd_stop_deps() {
+ if screen -ls | grep -q stack; then
+ screen -S stack -X quit
+ rm -f $DEST/status/stack/*
+ fi
+}
+
+
+###############################################################################
+# Initialize Trove
+###############################################################################
+
+function rd_manage() {
+ pushd $PATH_TROVE
+ $TROVE_BIN_DIR/trove-manage --config-file=$TROVE_CONF "$@"
+ popd
+}
+
+function install_test_packages() {
+ sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
+}
+
+function mod_confs() {
+ DATASTORE_TYPE=$1
+
+ sudo install -b --mode 0664 $REDSTACK_SCRIPTS/conf/test_begin.conf $TEST_CONF
+ TROVE_REPORT_DIR=$REDSTACK_SCRIPTS/../report/
+ EXTRA_CONF=$REDSTACK_SCRIPTS/conf/test.extra.conf
+ if [ -e $EXTRA_CONF ]; then
+ cat $EXTRA_CONF >> $TEST_CONF
+ fi
+ # Append datastore specific configuration file
+ DATASTORE_CONF=$REDSTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf
+ if [ ! -f $DATASTORE_CONF ]; then
+ exclaim "Datastore configuration file ${DATASTORE_CONF} not found"
+ exit 1
+ fi
+ cat $DATASTORE_CONF | sudo tee -a $TEST_CONF > /dev/null
+ cat $REDSTACK_SCRIPTS/conf/test_end.conf | sudo tee -a $TEST_CONF > /dev/null
+
+ #Add the paths to the test conf
+ sed -i "s,%report_directory%,$TROVE_REPORT_DIR,g" $TEST_CONF
+ sed -i "s,%keystone_path%,$PATH_KEYSTONE,g" $TEST_CONF
+ sed -i "s,%nova_path%,$PATH_NOVA,g" $TEST_CONF
+ sed -i "s,%glance_path%,$PATH_GLANCE,g" $TEST_CONF
+ sed -i "s,%trove_path%,$PATH_TROVE,g" $TEST_CONF
+ sed -i "s,%service_host%,$SERVICE_HOST,g" $TEST_CONF
+ sed -i "s,%swifth_path%,$PATH_SWIFT,g" $TEST_CONF
+
+ # Add the region name into test.conf
+ sed -i "s/%region_name%/${REGION_NAME}/g" $TEST_CONF
+
+ # Add the tenant id's into test.conf
+ sed -i "s/%admin_tenant_id%/$(get_attribute_id project admin 1)/g" $TEST_CONF
+ sed -i "s/%alt_demo_tenant_id%/$(get_attribute_id project alt_demo 1)/g" $TEST_CONF
+ sed -i "s/%demo_tenant_id%/$(get_attribute_id project demo 1)/g" $TEST_CONF
+ sed -i "s/%admin_password%/$ADMIN_PASSWORD/g" $TEST_CONF
+
+ # Enable neutron tests if needed
+ sed -i "s/%neutron_enabled%/$ENABLE_NEUTRON/g" $TEST_CONF
+
+ # If neutron is enabled, we create a shared network and write this info to the
+ # confs so that the integration tests can use it.
+ if [[ $ENABLE_NEUTRON = true ]]; then
+ management_network_id=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/ alt-private / {print $2}')
+ management_subnet=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT subnet-list | awk '/ alt-private-subnet / {print $2}')
+ echo "Using neutron network $management_network_id and subnet $management_subnet"
+ sed -i "s,%shared_network%,$management_network_id,g" $TEST_CONF
+ sed -i "s,%shared_network_subnet%,$management_subnet,g" $TEST_CONF
+ else
+ # do not leave invalid keys in the configuration when using Nova for networking
+ sed -i "/%shared_network%/d" $TEST_CONF
+ sed -i "/%shared_network_subnet%/d" $TEST_CONF
+ fi
+
+ if [ "$DATASTORE_TYPE" = "vertica" ]; then
+ # Vertica needs more time than mysql for its boot/start/stop operations.
+ setup_cluster_configs cluster_member_count 3
+ elif [ "$DATASTORE_TYPE" = "pxc" ]; then
+ setup_cluster_configs min_cluster_member_count 2
+ elif [ "$DATASTORE_TYPE" = "cassandra" ]; then
+ setup_cluster_configs cluster_member_count 2
+ elif [ "$DATASTORE_TYPE" = "mongodb" ]; then
+ setup_cluster_configs cluster_member_count 2
+ # Decrease the number of required config servers per cluster to save resources.
+ iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1
+ fi
+
+ set_bin_path
+
+}
+
+function setup_cluster_configs() {
+ # Setting cluster_member_count to 2 to decrease cluster spawn time.
+ iniset $TROVE_CONF $DATASTORE_TYPE $1 $2
+}
+
+# Add useful flavors for testing (with corresponding *.resize flavors)
+function add_test_flavors() {
+ # name id ram root_vol vcpu
+ # the ram and vcpu for name.resize are automatically calculated
+ # eph and non-eph flavors are created for each entry
+ add_flavor 'tiny' 10 512 3 1
+
+ add_flavor 'small' 15 768 3 1
+ add_flavor 'small' 16 768 4 1
+ add_flavor 'small' 17 768 5 1
+
+ add_flavor 'medium' 20 1024 4 1
+ add_flavor 'medium' 21 1024 5 1
+
+ add_flavor 'large' 25 2048 5 1
+ add_flavor 'large' 26 2048 10 1
+ add_flavor 'large' 27 2048 15 1
+
+ # This will allow Nova to create an instance, but not enough disk to boot the image
+ add_flavor 'fault_1' 30 512 1 1
+ # This should be enough memory to cause Nova to fail entirely due to too much allocation
+ add_flavor 'fault_2' 31 131072 5 1
+}
+
+function cmd_test_init() {
+ exclaim 'Initializing Configuration for Running Tests...'
+
+ exclaim "Installing python test packages."
+ install_test_packages
+
+ exclaim "Modifying test.conf and guest.conf with appropriate values."
+ mod_confs $1
+
+ exclaim "Creating Test Flavors."
+ add_test_flavors
+
+ if [[ -n $KEY_DIR ]]; then
+ exclaim "Installing the SSH key from $KEY_DIR to the test environment."
+ mkdir -m 700 -p $USERHOME/.ssh
+ install -b --mode 0400 $KEY_DIR/id_rsa $USERHOME/.ssh
+ cat $KEY_DIR/authorized_keys >> $USERHOME/.ssh/authorized_keys
+ chmod 600 $USERHOME/.ssh/authorized_keys
+ fi
+}
+
+function cmd_build_image() {
+ IMAGE_DATASTORE_TYPE=${1:-'mysql'}
+ ESCAPED_PATH_TROVE=${2:-'\/opt\/stack\/trove'}
+ HOST_SCP_USERNAME=${3:-'ubuntu'}
+ GUEST_USERNAME=${4:-'ubuntu'}
+
+ exclaim "Ensuring we have all packages needed to build image."
+ sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
+ sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu
+ sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
+
+ install_devstack_code
+
+ cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/image-projects-list
+
+ exclaim "Use tripleo-diskimagebuilder to actually build the Trove Guest Agent Image."
+ build_guest_image $IMAGE_DATASTORE_TYPE
+}
+
+function cmd_build_and_upload_image() {
+ local IMAGE_URL=""
+ # Use /tmp as file_cache
+ FILES=/tmp
+ if [[ -n $IMAGE_DOWNLOAD_URL ]]; then
+ exclaim "Downloading and using cached image"
+ IMAGE_URL=$IMAGE_DOWNLOAD_URL
+ else
+ exclaim "Trying to build image"
+ build_guest_image $1
+ QCOW_IMAGE=`find $VM_PATH -name '*.qcow2'`
+ IMAGE_URL="file://$QCOW_IMAGE"
+ fi
+
+ # The devstack openrc has references to 'enable_plugin' which causes errors
+ # in the stable/juno and stable/kilo branches. These are safe to ignore.
+ set +e; source $PATH_DEVSTACK_SRC/openrc admin admin; set -e
+ TOKEN=$(openstack token issue | grep ' id ' | get_field 2)
+ GLANCE_IMAGEIDS=$(glance image-list | grep $(basename $IMAGE_URL .qcow2) | get_field 1)
+ if [[ -n $GLANCE_IMAGEIDS ]]; then
+ glance image-delete $GLANCE_IMAGEIDS
+ fi
+ GLANCE_IMAGEID=`get_glance_id upload_image $IMAGE_URL $TOKEN`
+ set +e; source $PATH_DEVSTACK_SRC/openrc demo demo; set -e
+ [[ -z "$GLANCE_IMAGEID" ]] && echo "Glance upload failed!" && exit 1
+ echo "IMAGE ID: $GLANCE_IMAGEID"
+
+ exclaim "Updating Datastores"
+ cmd_set_datastore $GLANCE_IMAGEID $1
+}
+
+
+function cmd_initialize() {
+ exclaim '(Re)Initializing Trove...'
+ pushd $PATH_DEVSTACK_SRC
+ ./unstack.sh
+ ./stack.sh
+ popd
+}
+
+
+###############################################################################
+# Start Trove specific daemons interactively in a screen session
+###############################################################################
+
+function tr_screen_it {
+ if screen -ls | grep -q stack; then
+ echo "Starting $@..."
+ screen -S stack -p $1 -X stuff "$2"$'\015'
+ fi
+}
+
+function init_fake_mode() {
+ # Create a test conf which, unlike the conf which runs on a user's machine,
+ # takes advantage of the running keystone service we have in our VM.
+ # You could think of this fake mode, which runs in the VM as being
+ # slightly less fake than the default one which runs outside of it.
+ CONF_FILE=/tmp/trove.conf.test
+ cp $PATH_TROVE/etc/trove/trove.conf.test $CONF_FILE
+ # Switch keystone from the fake class to the real one.
+ sed -i \
+ "s/trove.tests.fakes.keystone/keystone.middleware.auth_token/g" \
+ $CONF_FILE
+ sed -i "s/log_file = rdtest.log//g" $CONF_FILE
+ sed -i "s/use_stderr = False/use_stderr = True/g" $CONF_FILE
+ cd $PATH_TROVE
+ set -e
+ rm -f trove_test.sqlite
+ set +e
+ $TROVE_BIN_DIR/trove-manage --config-file=$CONF_FILE db_sync
+ sqlite3 trove_test.sqlite \
+ "INSERT INTO datastores VALUES ('a00000a0-00a0-0a00-00a0-000a000000aa', \
+ 'mysql', 'b00000b0-00b0-0b00-00b0-000b000000bb'); \
+ INSERT INTO datastores values ('e00000e0-00e0-0e00-00e0-000e000000ee', \
+ 'Test_Datastore_1', ''); \
+ INSERT INTO datastore_versions VALUES ('b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 'a00000a0-00a0-0a00-00a0-000a000000aa', '5.6', \
+ 'c00000c0-00c0-0c00-00c0-000c000000cc', $MYSQL_PKG, 1, 'mysql'); \
+ INSERT INTO datastore_versions VALUES ('d00000d0-00d0-0d00-00d0-000d000000dd', \
+ 'a00000a0-00a0-0a00-00a0-000a000000aa', 'inactive_version', \
+ '', '', 0, 'manager1'); \
+ INSERT INTO datastore_configuration_parameters VALUES \
+ ('00000000-0000-0000-0000-000000000001', \
+ 'key_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 0, 4294967296, 0, 'integer', 0, NULL); \
+ INSERT INTO datastore_configuration_parameters VALUES \
+ ('00000000-0000-0000-0000-000000000002', \
+ 'connect_timeout', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 0, 65535, 1, 'integer', 0, NULL); \
+ INSERT INTO datastore_configuration_parameters VALUES \
+ ('00000000-0000-0000-0000-000000000003', \
+ 'join_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 0, 4294967296, 0, 'integer', 0, NULL); \
+ INSERT INTO datastore_configuration_parameters VALUES \
+ ('00000000-0000-0000-0000-000000000004', \
+ 'local_infile', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 0, 1, 0, 'integer', 0, NULL); \
+ INSERT INTO datastore_configuration_parameters VALUES \
+ ('00000000-0000-0000-0000-000000000005', \
+ 'collation_server', 'b00000b0-00b0-0b00-00b0-000b000000bb', \
+ 0, NULL, NULL, 'string', 0, NULL); \
+ "
+}
+
+function cmd_start() {
+ if screen -ls | grep -q stack; then
+ set_trove_plugin_vars
+ USE_SCREEN=True
+ TOP_DIR=$PATH_DEVSTACK_SRC
+ LOGDIR=$TROVE_LOGDIR
+ RUNNING=$(screen -S stack -Q windows)
+ if [[ "$RUNNING" =~ " tr-" ]]; then
+ exclaim "${COLOR_RED}WARNING: Trove services appear to be running. Please run 'stop' or 'restart'${COLOR_NONE}"
+ else
+ source /dev/stdin < <(sed -n '/^function start_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE")
+ start_trove
+ fi
+ else
+ echo "WARNING: Could not start Trove services because there is no stack screen session running"
+ fi
+}
+
+function cmd_start_fake() {
+ init_fake_mode
+ CONF_FILE=/tmp/trove.conf.test
+ tr_screen_it tr-fake "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@"
+}
+
+function cmd_run() {
+ cd $PATH_TROVE; $TROVE_BIN_DIR/trove-api \
+ --config-file=$TROVE_CONF $@
+}
+
+function cmd_run_fake() {
+ init_fake_mode
+ CONF_FILE=/tmp/trove.conf.test
+ $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@
+}
+
+###############################################################################
+# Stop any active Trove screen session
+###############################################################################
+
+function cmd_stop() {
+ if screen -ls | grep -q stack; then
+ set_trove_plugin_vars
+ rm -f $DEST/status/stack/tr-*
+ USE_SCREEN=True
+ source /dev/stdin < <(sed -n '/^function stop_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE")
+ MAX_RETRY=5
+ COUNT=1
+ while true; do
+ RUNNING=$(screen -S stack -Q windows)
+ if [[ "$RUNNING" =~ " tr-" ]]; then
+ stop_trove
+ else
+ break
+ fi
+ ((COUNT++))
+ if [ "$COUNT" -gt "$MAX_RETRY" ]; then
+ exclaim "${COLOR_RED}WARNING: Could not stop Trove services after ${MAX_RETRY} attempts${COLOR_NONE}"
+ break
+ fi
+ done
+ else
+ echo "WARNING: Could not stop Trove services because there is no stack screen session running"
+ fi
+}
+
+
+###############################################################################
+# Run Integration Tests
+###############################################################################
+
+function cmd_int_tests() {
+ exclaim "Running Trove Integration Tests..."
+ if [ ! $USAGE_ENDPOINT ]; then
+ export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
+ fi
+ cd $REDSTACK_SCRIPTS
+ if [ $# -lt 1 ]; then
+ args="--group=blackbox"
+ else
+ args="$@"
+ fi
+
+ # -- verbose makes it prettier.
+ # -- logging-clear-handlers keeps the novaclient and other things from
+ # spewing logs to stdout.
+ args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
+ echo "python $args"
+ python $args
+}
+
+function cmd_int_tests_simple() {
+ exclaim "Running Trove Simple Integration Tests..."
+ cd $REDSTACK_SCRIPTS
+ if [ $# -lt 1 ]; then
+ args="--group=simple_blackbox"
+ else
+ args="$@"
+ fi
+
+ # -- verbose makes it prettier.
+ # -- logging-clear-handlers keeps the novaclient and other things from
+ # spewing logs to stdout.
+ args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args"
+ echo "python $args"
+ python $args
+}
+
+function cmd_int_tests_white_box() {
+ export PYTHONPATH=$PYTHONPATH:$PATH_TROVE
+ export PYTHONPATH=$PYTHONPATH:$PATH_NOVA
+ cmd_int_tests --test-config white_box=True \
+ --config-file=$TROVE_CONF \
+ --nova-flags=/etc/nova/nova.conf $@
+}
+
+function cmd_example_tests() {
+ set +e
+ cmd_stop
+ set -e
+ cmd_start_fake
+ sleep 3
+ echo "
+{
+ \"directory\": \"$REDSTACK_TESTS/../apidocs/src/resources/samples/\",
+ \"auth_url\":\"http://$KEYSTONE_AUTH_HOST:35357/v2.0/tokens\",
+ \"api_url\":\"http://$SERVICE_HOST:8779\",
+ \"replace_host\":\"https://ord.databases.api.rackspacecloud.com\",
+ \"replace_dns_hostname\": \"e09ad9a3f73309469cf1f43d11e79549caf9acf2.rackspaceclouddb.com\",
+ \"username\":\"examples\",
+ \"password\":\"examples\",
+ \"tenant\":\"trove\"
+}" > /tmp/example-tests.conf
+ python $REDSTACK_TESTS/examples/examples/example_generation.py /tmp/example-tests.conf
+ pushd $REDSTACK_TESTS/../apidocs
+ mvn clean
+ mvn generate-sources
+ popd
+ cmd_stop
+}
+
+
+###############################################################################
+# Misc. tools
+###############################################################################
+
+function mysql_nova() {
+ echo mysql nova --execute "$@"
+ mysql -u root -p$MYSQL_PASSWORD nova --execute "$@"
+}
+
+function mysql_trove() {
+ echo mysql trove --execute "$@"
+ mysql -u root -p$MYSQL_PASSWORD trove --execute "$@"
+}
+
+function cmd_wipe_logs() {
+ for file in `ls $TROVE_LOGDIR/*.log`
+ do
+ echo "Reseting log file $file..."
+ echo "Reset at `date`" > $file
+ done
+}
+
+function cmd_rd_sql() {
+ mysql -u root -p$MYSQL_PASSWORD trove
+}
+
+function cmd_fake_sql() {
+ pushd $PATH_TROVE
+ sqlite3 trove_test.sqlite $@
+ popd
+}
+
+function cmd_vagrant_ssh() {
+ # Runs a command on a vagrant VM from the host machine.
+ VHOST=`vagrant ssh_config host | awk '/HostName/{print $2}'`
+ VUSER=`vagrant ssh_config host | awk '/User /{print $2}'`
+ VPORT=`vagrant ssh_config host | awk '/Port/{print $2}'`
+ VIDFILE=`vagrant ssh_config host | awk '/IdentityFile/{print $2}'`
+ echo ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
+ ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@"
+}
+
+
+function cmd_run_ci() {
+ exclaim "Running CI suite..."
+ set +e
+ cmd_stop_deps
+ cmd_stop
+ set -e
+ cmd_install
+ cmd_test_init $1
+ # The arg will be the image type
+ cmd_build_and_upload_image $1
+
+ # Test in fake mode.
+ exclaim "Testing in fake mode."
+ cmd_start_fake
+ FAKE_MODE=True cmd_int_tests
+ cmd_stop
+
+ # Test in real mode.
+ exclaim "Testing in real mode."
+ cmd_start
+ FAKE_MODE=False cmd_int_tests
+}
+
+function cmd_wipe_queues() {
+ # Obliterate rabbit.
+ for i in stop_app reset start_app "change_password guest $RABBIT_PASSWORD"; \
+ do sudo rabbitmqctl $i; done
+}
+
+function cmd_clear() {
+ cmd_int_tests --group=dbaas.api.instances.delete
+ clean_instances
+ mysql_nova "DELETE FROM instance_info_caches;"
+ mysql_nova "DELETE FROM instances;"
+ mysql_trove "DELETE FROM instances;"
+ mysql_trove "DELETE FROM service_statuses;"
+ cmd_wipe_queues
+}
+
+function exec_cmd_on_output() {
+ local output_cmd=$1
+ local exec_cmd=$2
+ local delete_sleep_time=${3:-0}
+ local skip_pattern=${4:-""}
+
+ echo "Cleaning up objects from '${output_cmd}'"
+ local skip_cmd="cat"
+ if [[ -n "${skip_pattern}" ]]; then
+ local temp_skip_cmd=(grep -v "${skip_pattern}")
+ skip_cmd=${temp_skip_cmd[*]}
+ fi
+ local max_retry=10
+ local count=1
+ local again=
+ while true; do
+ ids=$($output_cmd | ${skip_cmd} | grep -v -e'---' | grep -iv ' id ' | cut -d'|' -f2)
+ if [[ -n $ids ]]; then
+ for id in $ids; do
+ echo -e "Executing: ${exec_cmd} ${id} ${again}"
+ # don't stop if we get an error executing the delete, and don't print
+ # out anything from stderr
+ set +e
+ ${exec_cmd} "${id}" &> /dev/null
+ set -e
+ done
+ sleep "${delete_sleep_time}"
+ else
+ break
+ fi
+ ((count++))
+ if [[ "$count" -gt "$max_retry" ]]; then
+ exclaim "${COLOR_RED}WARNING: '$output_cmd' still returning output after ${max_retry} delete attempts${COLOR_NONE}"
+ break
+ fi
+ again="${COLOR_BLUE}(again)${COLOR_NONE}"
+ done
+}
+
+function cmd_clean() {
+ echo "Cleaning up project '${OS_PROJECT_NAME}'"
+ # reset any stuck backups
+ mysql_trove "update backups set state='COMPLETED'"
+ # clear out any DS version metadata
+ mysql_trove "delete from datastore_version_metadata"
+ # reset any stuck instances, and clear all replicas
+ mysql_trove "update instances set task_id=2, slave_of_id=null"
+ # reset any stuck clusters
+ mysql_trove "update clusters set task_id=1"
+ # get rid of any extraneous quota usage
+ mysql_trove "delete from quota_usages"
+ # mark all instance modules as deleted
+ mysql_trove "update instance_modules set deleted=1"
+
+ source "${PATH_DEVSTACK_SRC}"/openrc admin "${OS_PROJECT_NAME}"
+ # delete any trove clusters
+ exec_cmd_on_output "trove cluster-list" "trove cluster-delete" 20
+ # delete any trove instances
+ exec_cmd_on_output "trove list" "trove delete" 10
+ # delete any backups
+ exec_cmd_on_output "trove backup-list" "trove backup-delete"
+ # clean up any remaining nova instances or cinder volumes
+ exec_cmd_on_output "nova list" "nova delete" 5
+ exec_cmd_on_output "cinder list" "cinder delete" 1
+ # delete any config groups since all instances should be gone now
+ exec_cmd_on_output "trove configuration-list" "trove configuration-delete"
+ # delete any modules too
+ exec_cmd_on_output "trove module-list" "trove module-delete"
+ # make sure that security groups are also gone, except the default
+ exec_cmd_on_output "openstack security group list" "nova security group delete" 0 "default"
+ # delete server groups
+ exec_cmd_on_output "nova server-group-list" "nova server-group-delete"
+}
+
+function cmd_kick_start() {
+ cmd_test_init $1
+ cmd_build_and_upload_image $1
+}
+
+function cmd_dsvm_gate_tests() {
+ DATASTORE_TYPE=${1:-'mysql'}
+ TEST_GROUP=${2:-${DATASTORE_TYPE}}
+ HOST_SCP_USERNAME=${3:-'jenkins'}
+ GUEST_USERNAME=${4:-'ubuntu'}
+ CONTROLLER_IP=${5:-'10.1.0.1'}
+ ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'}
+
+ if [[ $BRANCH_OVERRIDE == "stable/liberty" ]]; then
+ # Devstack in liberty doesn't copy the clouds.yaml file to /etc so we need to
+ # ensure we have access to the clouds.yaml file set up by devstack-gate
+ sudo mkdir -p ~/.config/openstack
+ sudo ln -s $DEST/.config/openstack/clouds.yaml ~/.config/openstack/clouds.yaml
+ sudo chown -R $(whoami) ~/.config
+ fi
+
+ # Devstack vm-gate runs as the jenkins user, but needs to connect to the guest image as ubuntu
+ echo "User=ubuntu" >> /home/jenkins/.ssh/config
+
+ # Fix iptables rules that prevent amqp connections from the devstack box to the guests
+ sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited || true
+
+ sudo chown -R $(whoami) /etc/trove
+ sudo chown -R $(whoami) $DEST/trove-integration
+ iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP
+ iniset $TROVE_GUESTAGENT_CONF oslo_messaging_rabbit rabbit_hosts $CONTROLLER_IP
+ cd $DEST/trove-integration/scripts
+ sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
+
+ cmd_kick_start $DATASTORE_TYPE
+
+ # Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1
+ source $DEST/devstack/accrc/admin/admin
+ # NOTE(mriedem): We have to treat stable branches before liberty special
+ # due to constraints with older versions of python-openstackclient.
+ if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
+ SWIFT_ENDPOINT=$(openstack endpoint list | grep 'swift' | get_field 1)
+ openstack endpoint create swift --region RegionOne --publicurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' \
+ --internalurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --adminurl 'http://'$CONTROLLER_IP':8080'
+ openstack endpoint delete $SWIFT_ENDPOINT
+ else
+ OS_CLIENT_ARGS="--os-auth-type v3password --os-auth-url $KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:5000/v3 --os-identity-api-version=3"
+ SWIFT_ENDPOINTS=$(openstack endpoint list $OS_CLIENT_ARGS --service swift -c ID -f value)
+ openstack endpoint create $OS_CLIENT_ARGS swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
+ openstack endpoint create $OS_CLIENT_ARGS swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
+ openstack endpoint create $OS_CLIENT_ARGS swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne
+ echo $SWIFT_ENDPOINTS | xargs -n 1 openstack endpoint delete $OS_CLIENT_ARGS
+ fi
+
+ cmd_int_tests --group=$TEST_GROUP
+}
+
+function cmd_reset_task() {
+ mysql_trove "UPDATE instances SET task_id=1 WHERE id='$1'"
+}
+
+function cmd_clone_projects() {
+ UPDATE_PROJECTS=$1
+ PROJECT_LIST_FILES=${@:2}
+ for project in $(cat $PROJECT_LIST_FILES); do
+ if [ ! -d $PATH_DEVSTACK_OUTPUT/$project ]; then
+ echo "Creating a new clone of $project..."
+ git_clone $GIT_OPENSTACK/"$project".git ${PATH_DEVSTACK_OUTPUT}/$project master
+ else
+ if [ $UPDATE_PROJECTS != "force_update" ]; then
+ echo "$project was already cloned or exists in a shared folder. Ignoring..."
+ else
+ echo "$project was already cloned. Pulling changes to update."
+ cd $PATH_DEVSTACK_OUTPUT/$project
+ git pull
+ fi
+ fi
+ # Switch to a branch if specified. The order the variables are checked is:
+ # <PROJECT>_BRANCH then PROJECT_CLIENT_BRANCH (if a client) then PROJECT_BRANCH
+ # Note: For the Trove project, only TROVE_BRANCH and PYTHON_TROVECLIENT_BRANCH are used
+ PROJECT_BRANCH_NAME=$(eval echo "${project}_BRANCH" | tr '[:lower:]-' '[:upper:]_')
+ PROJECT_BRANCH_VALUE=${!PROJECT_BRANCH_NAME}
+ # TROVE_BRANCH is defaulted to master if not set, so use the original value here
+ if [[ "$project" = "trove" ]]; then
+ PROJECT_BRANCH_VALUE=${TROVE_BRANCH_ORIG}
+ fi
+ BRANCH_SPECIFIED=$(test -z "${PROJECT_BRANCH_VALUE}${PROJECT_CLIENT_BRANCH}${PROJECT_BRANCH}" || echo 'True')
+ if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then
+ # Set up the default branch and env var names for the project
+ DEFAULT_BRANCH="$PROJECT_BRANCH"
+ ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_BRANCH"
+ # Don't use 'PROJECT_BRANCH' or 'PROJECT_CLIENT_BRANCH' for the Trove project
+ if [[ "$project" =~ "trove" ]]; then
+ DEFAULT_BRANCH=master
+ ENV_VARS="$PROJECT_BRANCH_NAME"
+ # Use 'PROJECT_CLIENT_BRANCH' first for clients
+ elif [[ "$project" =~ "client" ]]; then
+ DEFAULT_BRANCH="${PROJECT_CLIENT_BRANCH:-$PROJECT_BRANCH}"
+ ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_CLIENT_BRANCH' or 'PROJECT_BRANCH"
+ fi
+ PROJ_BRANCH=$(get_project_branch $PROJECT_BRANCH_NAME $DEFAULT_BRANCH)
+ git_checkout "$project" "$PATH_DEVSTACK_OUTPUT/$project" "$PROJ_BRANCH" "$ENV_VARS"
+ fi
+ done
+}
+
+function cmd_repl() {
+ INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@
+}
+
+
+###############################################################################
+# Process the user provided command and run the appropriate command
+###############################################################################
+
+# Let's not run this as the root user
+if [ $EUID -eq 0 ]; then
+ echo "You are running this script as root. You need to run as a regular user"
+ exit 1
+fi
+
+# Set this to exit immediately on error
+set -o errexit
+
+# set_home_dir
+set_http_proxy
+set_trove_plugin_vars false
+
+function print_usage() {
+ echo "Usage: $0 [command]"
+ echo "
+ Commands :
+ --setup environment--
+ install - Install all the required dependencies and bring up tr-api and tr-tmgr
+ - devstack config can be altered by using a USER_LOCAL_CONF file
+ which will be copied into devstack/local.conf on each 'install' run
+ (defaults to \$HOME/$USER_LOCAL_CONF_NAME)
+ - Set DEVSTACK_BRANCH to switch the branch/commit of devstack
+ (i.e. 'stable/kilo' or '7ef2462')
+ test-init - Configure the test configuration files and add keystone test users
+ build-image - Builds the vm image for the trove guest
+ initialize - Reinitialize the trove database, users, services, and test config
+
+ --helper for environment--
+ kick-start - kick start the setup of trove.
+ (redstack test-init/build-image in one step)
+ [mysql no-clean] no clean avoids rebuilding packages from scratch
+ - Set REBUILD_IMAGE=True to force rebuild (won't use cached image)
+
+ --trove dependency services--
+ start-deps - Start or resume daemons Trove depends on.
+ stop-deps - Kill daemons Trove depends on.
+
+ --trove services--
+ start - Start or resume daemons Trove depends on.
+ stop - Kill daemons Trove depends on.
+ restart - Runs stop then start for Trove services.
+
+ --tests--
+ unit-tests - Run the unit tests.dependencies
+ int-tests - Runs the integration tests (requires all daemons).
+ simple-tests - Runs the simple integration tests (requires all daemons).
+ dsvm-gate-tests - Configures and runs the int-tests in a devstack vm-gate environment.
+
+ --tools--
+ debug - Debug this script (shows all commands).
+ wipe-logs - Resets all log files.
+ rd-sql - Opens the Trove MySQL database.
+ vagrant-ssh - Runs a command from the host on the server.
+ clear - Destroy instances and rabbit queues.
+ clean - Clean up resources created by a failed test run.
+ run - Starts RD but not in a screen.
+ run-fake - Runs the server in fake mode.
+ update-projects - Git pull on all the daemons trove dependencies.
+ reset-task - Sets an instance task to NONE.
+ wipe-queues - Resets RabbitMQ queues.
+ "
+ exit 1
+}
+
+function run_command() {
+ # Print the available commands
+ if [ $# -lt 1 ]; then
+ print_usage
+ fi
+
+ case "$1" in
+ "install" ) cmd_install;;
+ "test-init" ) cmd_test_init $@;;
+ "build-image" ) shift; cmd_build_image $@;;
+ "initialize" ) cmd_initialize;;
+ "unit-tests" ) cmd_unit_tests;;
+ "start-deps" ) cmd_start_deps;;
+ "stop-deps" ) cmd_stop_deps;;
+ "start" ) cmd_start;;
+ "int-tests" ) shift; cmd_int_tests $@;;
+ "int-tests-wb" ) shift; cmd_int_tests_white_box $@;;
+ "simple-tests") shift; cmd_int_tests_simple $@;;
+ "stop" ) cmd_stop;;
+ "restart" ) cmd_stop; cmd_start;;
+ "wipe-logs" ) cmd_wipe_logs;;
+ "rd-sql" ) shift; cmd_rd_sql $@;;
+ "fake-sql" ) shift; cmd_fake_sql $@;;
+ "run-ci" ) shift; cmd_run_ci $@;;
+ "vagrant-ssh" ) shift; cmd_vagrant_ssh $@;;
+ "debug" ) shift; echo "Enabling debugging."; \
+ set -o xtrace; run_command $@;;
+ "clear" ) shift; cmd_clear $@;;
+ "clean" ) shift; cmd_clean $@;;
+ "run" ) shift; cmd_run $@;;
+ "kick-start" ) shift; cmd_kick_start $@;;
+ "dsvm-gate-tests" ) shift; cmd_dsvm_gate_tests $@;;
+ "run-fake" ) shift; cmd_run_fake $@;;
+ "start-fake" ) shift; cmd_start_fake $@;;
+ "update-projects" ) cmd_clone_projects force_update \
+ $REDSTACK_SCRIPTS/projects-list \
+ $REDSTACK_SCRIPTS/image-projects-list;;
+ "reset-task" ) shift; cmd_reset_task $@;;
+ "wipe-queues" ) shift; cmd_wipe_queues $@;;
+ "example-tests" ) shift; cmd_example_tests $@;;
+ "repl" ) shift; cmd_repl $@;;
+ "help" ) print_usage;;
+ * )
+ echo "'$1' not a valid command"
+ exit 1
+ esac
+}
+
+run_command $@
diff --git a/integration/scripts/redstack.rc b/integration/scripts/redstack.rc
new file mode 100644
index 00000000..b4036ba1
--- /dev/null
+++ b/integration/scripts/redstack.rc
@@ -0,0 +1,120 @@
+# Paths inside the VM.
+[ -z $SERVICE_HOST ] && SERVICE_HOST=`get_default_host_ip`
+[ -z $DEST ] && DEST=/opt/stack
+[ -z $BRIDGE_IP ] && BRIDGE_IP=172.24.4.1
+[ -z $PATH_DEVSTACK_SRC ] && PATH_DEVSTACK_SRC=~/devstack
+[ -z $TROVE_CONF_DIR ] && TROVE_CONF_DIR=/etc/trove
+[ -z $MYSQL_HOST ] && MYSQL_HOST=$SERVICE_HOST
+
+# Set up the region name
+# Try REGION_NAME then OS_REGION_NAME then RegionOne (the devstack default)
+REGION_NAME=${REGION_NAME:-${OS_REGION_NAME:-RegionOne}}
+
+# Enable neutron instead of nova-network
+# Note: Until a few key changesets land, we can't enable Neutron properly.
+# See: https://review.openstack.org/#/c/356026
+# https://review.openstack.org/#/c/356763
+# https://review.openstack.org/#/c/356701
+# NEUTRON_DEFAULT=true
+NEUTRON_DEFAULT=false
+if [[ $BRANCH_OVERRIDE == "stable/liberty" || $BRANCH_OVERRIDE == "stable/mitaka" ]]; then
+ NEUTRON_DEFAULT=false
+fi
+ENABLE_NEUTRON=$(get_bool ENABLE_NEUTRON $NEUTRON_DEFAULT)
+
+# Enable osprofiler - note: Enables Ceilometer as well
+ENABLE_PROFILER=$(get_bool ENABLE_PROFILER false)
+PROFILER_TRACE_SQL=$(get_bool PROFILER_TRACE_SQL false)
+[ -z $PROFILER_HMAC_KEYS ] && PROFILER_HMAC_KEYS=SECRET_KEY
+
+# Enable ceilometer
+ENABLE_CEILOMETER=$(get_bool ENABLE_CEILOMETER $ENABLE_PROFILER)
+
+# Enable Mistral
+ENABLE_MISTRAL=$(get_bool ENABLE_MISTRAL false)
+
+# Enable LIBS_FROM_GIT
+LIBS_FROM_GIT_ALL_CLIENTS=$(get_bool LIBS_FROM_GIT_ALL_CLIENTS false)
+LIBS_FROM_GIT_ALL_OSLO=$(get_bool LIBS_FROM_GIT_ALL_OSLO false)
+
+# Don't include certain .rc files in local.conf.d by default
+USING_VAGRANT=$(get_bool USING_VAGRANT false)
+USE_KVM=$(get_bool USE_KVM false)
+USE_UUID_TOKEN=$(get_bool USE_UUID_TOKEN false)
+
+# Specify configuration for Ceilometer
+CEILOMETER_SERVICES_CONF=$(get_bool CEILOMETER_SERVICES_CONF $ENABLE_CEILOMETER)
+CEILOMETER_CINDER_CONF=$(get_bool CEILOMETER_CINDER_CONF false)
+CEILOMETER_NOVA_CONF=$(get_bool CEILOMETER_NOVA_CONF false)
+
+# Paths for various OpenStack components
+PATH_DEVSTACK_OUTPUT=$DEST
+PATH_NOVA=$DEST/nova
+PATH_KEYSTONE=$DEST/keystone
+PATH_GLANCE=$DEST/glance
+PATH_SWIFT=$DEST/swift
+PATH_TROVE=$DEST/trove
+PATH_PYTHON_NOVACLIENT=$DEST/python-novaclient
+PATH_KEYSTONECLIENT=$DEST/python-keystoneclient
+PATH_OPENSTACKCLIENT=$DEST/python-openstackclient
+PATH_PYTHON_SWIFTCLIENT=$DEST/python-swiftclient
+PATH_PYTHON_TROVECLIENT=$DEST/python-troveclient
+PATH_TROVE_DASHBOARD=$DEST/trove-dashboard
+PATH_DISKIMAGEBUILDER=$DEST/diskimage-builder
+PATH_TRIPLEO_ELEMENTS=$DEST/tripleo-image-elements
+
+# Save the state of TROVE_BRANCH first, since it's used in redstack
+TROVE_BRANCH_ORIG=${TROVE_BRANCH}
+# Devstack and OpenStack git repo source paths, etc.
+GIT_BASE=${GIT_BASE:-git://git.openstack.org}
+GIT_OPENSTACK=${GIT_OPENSTACK:-${GIT_BASE}/openstack}
+DEVSTACK_REPO=${DEVSTACK_REPO:-${GIT_BASE}/openstack-dev/devstack.git}
+TROVE_REPO=${TROVE_REPO:-${GIT_OPENSTACK}/trove.git}
+TROVE_DIR=${TROVE_DIR:-${PATH_TROVE}}
+TROVE_BRANCH=${TROVE_BRANCH:-master}
+TROVE_CLIENT_REPO=${TROVE_CLIENT_REPO:-${TROVECLIENT_REPO:-${GIT_OPENSTACK}/python-troveclient.git}}
+TROVE_CLIENT_DIR=${TROVE_CLIENT_DIR:-${TROVECLIENT_DIR:-${PATH_PYTHON_TROVECLIENT}}}
+TROVE_CLIENT_BRANCH=${TROVE_CLIENT_BRANCH:-${TROVECLIENT_BRANCH:-master}}
+TROVE_DASHBOARD_REPO=${TROVE_DASHBOARD_REPO:-${TROVEDASHBOARD_REPO:-${GIT_OPENSTACK}/trove-dashboard.git}}
+TROVE_DASHBOARD_DIR=${TROVE_DASHBOARD_DIR:-${TROVEDASHBOARD_DIR:-${PATH_TROVE_DASHBOARD}}}
+TROVE_DASHBOARD_BRANCH=${TROVE_DASHBOARD_BRANCH:-${TROVEDASHBOARD_BRANCH:-master}}
+
+# Destination for working data
+DATA_DIR=${DEST}/data
+# Destination for status files
+SERVICE_DIR=${DEST}/status
+
+# Cinder Volume Group Name
+VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes}
+VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-${DATA_DIR}/${VOLUME_GROUP}-backing-file}
+VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-51200M}
+
+# Passwords used by devstack.
+MYSQL_PASSWORD=e1a2c042c828d3566d0a
+RABBIT_PASSWORD=f7999d1955c5014aa32c
+SERVICE_TOKEN=be19c524ddc92109a224
+ADMIN_PASSWORD=3de4922d8b6ac5a1aad9
+SERVICE_PASSWORD=7de4162d826bc5a11ad9
+
+# Swift hash used by devstack.
+SWIFT_HASH=12go358snjw24501
+
+# Swift Disk Image
+SWIFT_DATA_DIR=${DATA_DIR}/swift
+SWIFT_DISK_IMAGE=${SWIFT_DATA_DIR}/drives/images/swift.img
+
+DISTRO=${DISTRO:-ubuntu}
+#DISTRO=fedora
+
+# The following values can be used to tweak how devstack sets
+# up Trove. If not explicitly set, the defaults in the code are used.
+# To make changes without modifying the repo, add these variables
+# to options.rc or ~/redstack.options.rc
+#export TROVE_MAX_ACCEPTED_VOLUME_SIZE=10
+#export TROVE_MAX_INSTANCES_PER_TENANT=10
+#export TROVE_MAX_VOLUMES_PER_TENANT=40
+#export TROVE_AGENT_CALL_LOW_TIMEOUT=15
+#export TROVE_AGENT_CALL_HIGH_TIMEOUT=300
+#export TROVE_RESIZE_TIME_OUT=900
+#export TROVE_USAGE_TIMEOUT=1500
+#export TROVE_STATE_CHANGE_WAIT_TIME=180
diff --git a/integration/scripts/reviews.rc b/integration/scripts/reviews.rc
new file mode 100644
index 00000000..accccdcd
--- /dev/null
+++ b/integration/scripts/reviews.rc
@@ -0,0 +1,5 @@
+# This file will contain variables such as below, uncommented.
+# There will be a : separator between multiple inflight reviews
+# The path comes from the gerrit review system, and is the only
+# unique portion of each gerrit review.
+#REVIEW_PYTHON_NOVACLIENT=18/5018/2