diff options
author | Amrith Kumar <amrith@tesora.com> | 2016-10-09 07:13:29 -0400 |
---|---|---|
committer | Amrith Kumar <amrith@tesora.com> | 2016-10-09 07:14:25 -0400 |
commit | b120be1772f2416730ea1066782bf0813ca2dd1a (patch) | |
tree | e86dddeed295fdb97489f66170d6965a58474cd9 /integration/scripts | |
parent | 473d360b906dba80f0f914e88b461221c1bfd5f3 (diff) | |
download | trove-b120be1772f2416730ea1066782bf0813ca2dd1a.tar.gz |
Merge trove-integration into trove
This commit will merge into trove, the trove-integration tree as of
commit 9f92ca853f8aa2f72921e54682c918941a8f0919. This is in
preparation for making trove-integration go away.
In addition, it supresses any consideration of the integration
directory in the trove tox tests as it is understandably a small pile
of pooh and in need of much cleanup.
Change-Id: Ib7f2655c4c5ed86b5454708c04371ee55e37ec2d
Partially-Implements-Blueprint: eliminate-trove-integration-and-redstack
Diffstat (limited to 'integration/scripts')
128 files changed, 4687 insertions, 0 deletions
diff --git a/integration/scripts/conf.json.example b/integration/scripts/conf.json.example new file mode 100644 index 00000000..ee6bc7bf --- /dev/null +++ b/integration/scripts/conf.json.example @@ -0,0 +1,12 @@ +{ + "devstack":null, + "glance":null, + "horizon":null, + "keystone":null, + "nova":null, + "python_openstackclient":null, + "python_novaclient":null, + "trove":null, + "python_troveclient":null, + "tempest":null +} diff --git a/integration/scripts/conf/cassandra.conf b/integration/scripts/conf/cassandra.conf new file mode 100644 index 00000000..5a2f0acc --- /dev/null +++ b/integration/scripts/conf/cassandra.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.large-5", + "instance_bigger_flavor_name": "test.large-5.resize", + "instance_eph_flavor_name": "test.eph.large-5", + "instance_bigger_eph_flavor_name": "test.eph.large-5.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/couchbase.conf b/integration/scripts/conf/couchbase.conf new file mode 100644 index 00000000..5a2f0acc --- /dev/null +++ b/integration/scripts/conf/couchbase.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.large-5", + "instance_bigger_flavor_name": "test.large-5.resize", + "instance_eph_flavor_name": "test.eph.large-5", + "instance_bigger_eph_flavor_name": "test.eph.large-5.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/couchdb.conf b/integration/scripts/conf/couchdb.conf new file mode 100644 index 00000000..65bd382c --- /dev/null +++ b/integration/scripts/conf/couchdb.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.tiny-3", + "instance_bigger_flavor_name": "test.tiny-3.resize", + "instance_eph_flavor_name": "test.eph.tiny-3", + "instance_bigger_eph_flavor_name": "test.eph.tiny-3.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/db2.conf b/integration/scripts/conf/db2.conf new file mode 100644 index 00000000..a842ea02 --- /dev/null +++ b/integration/scripts/conf/db2.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.small-5", + "instance_bigger_flavor_name": "test.small-5.resize", + "instance_eph_flavor_name": "test.eph.small-5", + "instance_bigger_eph_flavor_name": "test.eph.small-5.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/mariadb.conf b/integration/scripts/conf/mariadb.conf new file mode 100644 index 00000000..0e29d114 --- /dev/null +++ b/integration/scripts/conf/mariadb.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.small-4", + "instance_bigger_flavor_name": "test.small-4.resize", + "instance_eph_flavor_name": "test.eph.small-4", + "instance_bigger_eph_flavor_name": "test.eph.small-4.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/mongodb.conf b/integration/scripts/conf/mongodb.conf new file mode 100644 index 00000000..0e0aae02 --- /dev/null +++ b/integration/scripts/conf/mongodb.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.large-5", + "instance_bigger_flavor_name": "test.large-5.resize", + "instance_eph_flavor_name": "test.eph.large-5", + "instance_bigger_eph_flavor_name": "test.eph.large-5.resize", + "trove_volume_support": true, + "trove_volume_size": 5, diff --git a/integration/scripts/conf/mysql.conf b/integration/scripts/conf/mysql.conf new file mode 100644 index 00000000..eac3c579 --- /dev/null +++ b/integration/scripts/conf/mysql.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.small-3", + "instance_bigger_flavor_name": "test.small-3.resize", + "instance_eph_flavor_name": "test.eph.small-3", + "instance_bigger_eph_flavor_name": "test.eph.small-3.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/percona.conf b/integration/scripts/conf/percona.conf new file mode 100644 index 00000000..eac3c579 --- /dev/null +++ b/integration/scripts/conf/percona.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.small-3", + "instance_bigger_flavor_name": "test.small-3.resize", + "instance_eph_flavor_name": "test.eph.small-3", + "instance_bigger_eph_flavor_name": "test.eph.small-3.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/postgresql.conf b/integration/scripts/conf/postgresql.conf new file mode 100644 index 00000000..8033f582 --- /dev/null +++ b/integration/scripts/conf/postgresql.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.medium-4", + "instance_bigger_flavor_name": "test.medium-4.resize", + "instance_eph_flavor_name": "test.eph.medium-4", + "instance_bigger_eph_flavor_name": "test.eph.medium-4.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/pxc.conf b/integration/scripts/conf/pxc.conf new file mode 100644 index 00000000..eac3c579 --- /dev/null +++ b/integration/scripts/conf/pxc.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.small-3", + "instance_bigger_flavor_name": "test.small-3.resize", + "instance_eph_flavor_name": "test.eph.small-3", + "instance_bigger_eph_flavor_name": "test.eph.small-3.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/redis.conf b/integration/scripts/conf/redis.conf new file mode 100644 index 00000000..65bd382c --- /dev/null +++ b/integration/scripts/conf/redis.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.tiny-3", + "instance_bigger_flavor_name": "test.tiny-3.resize", + "instance_eph_flavor_name": "test.eph.tiny-3", + "instance_bigger_eph_flavor_name": "test.eph.tiny-3.resize", + "trove_volume_support": true, + "trove_volume_size": 1, diff --git a/integration/scripts/conf/test_begin.conf b/integration/scripts/conf/test_begin.conf new file mode 100644 index 00000000..eff96dea --- /dev/null +++ b/integration/scripts/conf/test_begin.conf @@ -0,0 +1,101 @@ +{ + "dbaas_url":"http://%service_host%:8779/v1.0", + "version_url":"http://%service_host%:8779", + "trove_auth_url":"http://%service_host%:35357/v2.0/tokens", + "trove_client_insecure":false, + "auth_strategy":null, + "trove_client_region_name": "%region_name%", + + "nova_client": { + "url":"http://%service_host%:8774/v1.1", + "auth_url":"http://%service_host%:35357/v2.0", + "nova_service_type":"compute", + "volume_service_type":"volume" + }, + + "flavors": null, + + "white_box":false, + "start_services": true, + "test_mgmt":false, + "use_local_ovz":false, + "use_venv":false, + "glance_code_root":"%glance_path%", + "glance_api_conf":"/vagrant/conf/glance-api.conf", + "glance_reg_conf":"/vagrant/conf/glance-reg.conf", + "glance_images_directory": "/glance_images", + "glance_image": "debian-squeeze-x86_64-openvz.tar.gz", + "report_directory":"%report_directory%", + "usr_bin_dir":"%bin_path%", + "nova_code_root":"%nova_path%", + "nova_conf":"/home/vagrant/nova.conf", + "keystone_code_root":"%keystone_path%", + "keystone_conf":"/etc/keystone/keystone.conf", + "keystone_use_combined":true, + "trove_code_root":"%trove_path%", + "trove_conf":"/tmp/trove.conf", + "trove_version":"v1.0", + "trove_api_updated":"2012-08-01T00:00:00Z", + "trove_max_accepted_volume_size": 1000, + "trove_max_instances_per_user": 55, + "trove_max_volumes_per_user": 100, + "use_reaper":false, + "users": [ + { "auth_user":"admin", + "auth_key":"%admin_password%", + "tenant":"admin", + "tenant_id":"%admin_tenant_id%", + "requirements": { + "is_admin":true, + "services": ["trove"] + } + }, + { "auth_user":"alt_demo", + "auth_key":"%admin_password%", + "tenant":"alt_demo", + "tenant_id":"%alt_demo_tenant_id%", + "requirements": { + "is_admin":false, + "services": ["trove"] + } + }, + { "auth_user":"admin_alt_demo", + "auth_key":"%admin_password%", + "tenant":"alt_demo", + "tenant_id":"%alt_demo_tenant_id%", + "requirements": { + "is_admin":true, + "services": ["swift"] + } + }, + { "auth_user":"demo", + "auth_key":"%admin_password%", + "tenant":"demo", + "tenant_id":"%demo_tenant_id%", + "requirements": { + "is_admin":false, + "services": ["nova", "trove"] + } + } + ], + "root_removed_from_instance_api": true, + "root_timestamp_disabled": false, + "openvz_disabled": true, + "management_api_disabled": true, + "dbaas_image": 1, + "dns_driver":"trove.dns.rsdns.driver.RsDnsDriver", + "dns_instance_entry_factory":"trove.dns.rsdns.driver.RsDnsInstanceEntryFactory", + "trove_dns_support": false, + "databases_page_size": 20, + "instances_page_size": 20, + "users_page_size": 20, + "rabbit_runs_locally": true, + "dbaas_datastore": "%datastore_type%", + "dbaas_datastore_version": "%datastore_version%", + "neutron_enabled": %neutron_enabled%, + "shared_network": "%shared_network%", + "shared_network_subnet": "%shared_network_subnet%", + "instance_fault_1_flavor_name": "test.fault_1-1", + "instance_fault_1_eph_flavor_name": "test.eph.fault_1-1", + "instance_fault_2_flavor_name": "test.fault_2-5", + "instance_fault_2_eph_flavor_name": "test.eph.fault_2-5", diff --git a/integration/scripts/conf/test_end.conf b/integration/scripts/conf/test_end.conf new file mode 100644 index 00000000..10fc2de1 --- /dev/null +++ b/integration/scripts/conf/test_end.conf @@ -0,0 +1,2 @@ + "sentinel": null +} diff --git a/integration/scripts/conf/vertica.conf b/integration/scripts/conf/vertica.conf new file mode 100644 index 00000000..242d7fa1 --- /dev/null +++ b/integration/scripts/conf/vertica.conf @@ -0,0 +1,6 @@ + "instance_flavor_name": "test.large-10", + "instance_bigger_flavor_name": "test.large-10.resize", + "instance_eph_flavor_name": "test.eph.large-10", + "instance_bigger_eph_flavor_name": "test.eph.large-10.resize", + "trove_volume_support": true, + "trove_volume_size": 5, diff --git a/integration/scripts/create_vm b/integration/scripts/create_vm new file mode 100755 index 00000000..7b1b3767 --- /dev/null +++ b/integration/scripts/create_vm @@ -0,0 +1,84 @@ +#!/usr/bin/env python +""" +Sets up a VM with hardcoded paths to multiple source trees. +Creates a script for use with VMWare or a Vagrantfile. + +Uses a configuration file (in JSON format) that stores the paths to checked-out +copies of OpenStack projects on the host machine. If the path is None then +it lets devstack download them. +""" + +import json + + +class Config(object): + """ + Very simple configuration file thats just some JSON. + """ + + vm_paths = { + 'devstack':"/devstack", + 'glance': '/opt/stack/glance', + 'horizon': '/opt/stack/horizon', + 'keystone': "/opt/stack/keystone", + 'nova': "/opt/stack/nova", + 'python_openstackclient': "/opt/stack/python-openstackclient", + 'python_novaclient': "/opt/stack/python-novaclient", + 'trove':"/opt/stack/trove", + 'python_troveclient':"/opt/stack/python-troveclient", + 'tempest':"/opt/stack/tempest" + } + + def __init__(self, **kwargs): + for name in Config.vm_paths.keys(): + if name not in kwargs: + raise RuntimeError('Missing configuration value "%s".' % name) + value = kwargs[name] + if value is not None and type(value) is not str \ + and type(value) is not unicode: + raise RuntimeError('Path "%s" must be a string or None but is ' + 'of type %s.' % (name, type(value))) + setattr(self, name, kwargs[name]) + self.vagrant_path = kwargs.get("vagrant_path", "Vagrantfile") + + @staticmethod + def load(file_path): + file_contents = open(file_path, "r").read() + dict = json.loads(file_contents); + return Config(**dict) + + def write_vagrant_file(self): + with open(self.vagrant_path, 'w') as file: + file.write(""" +Vagrant::Config.run do |global_config| + # Host config + global_config.vm.define :host do |config| + + config.vm.network "33.33.44.11" + + config.vm.box = "precise" + config.vm.host_name = "host" + + config.ssh.timeout = 3600 + config.vm.customize do |vm| + vm.memory_size = 2048 + end + + config.vm.share_folder "integration", "/integration", "../" + +""") + for key in Config.vm_paths.keys(): + local_path = getattr(self, key) + vm_path = Config.vm_paths[key] + if local_path is not None: + file.write('\tconfig.vm.share_folder "%s", "%s", "%s" \n' + % (key, vm_path, local_path)) + file.write(""" + + end +end + """) + +if __name__=="__main__": + conf = Config.load("conf.json") + conf.write_vagrant_file() diff --git a/integration/scripts/files/elements/apt-conf-dir/README.rst b/integration/scripts/files/elements/apt-conf-dir/README.rst new file mode 100644 index 00000000..c94e00ea --- /dev/null +++ b/integration/scripts/files/elements/apt-conf-dir/README.rst @@ -0,0 +1,16 @@ +============ +apt-conf-dir +============ + +This element overrides the default apt.conf.d directory for APT based systems. + +Environment Variables +--------------------- + +DIB_APT_CONF_DIR: + :Required: No + :Default: None + :Description: To override `DIB_APT_CONF_DIR`, set it to the path to your + apt.conf.d. The new apt.conf.d will take effect at build time + and run time. + :Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf`` diff --git a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd new file mode 100755 index 00000000..e286d684 --- /dev/null +++ b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd @@ -0,0 +1,21 @@ +#!/bin/bash +# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +# exit directly if DIB_APT_CONF_DIR is not defined properly +if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then + echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in" + exit 0 +elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then + echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory." + echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR" + exit 1 +fi + +# copy the apt.conf to cloudimg +sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep b/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep new file mode 100755 index 00000000..97a5e438 --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/15-reddwarf-dep @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep + +source $_LIB/die + +BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default} +ADD_BRANCH=$(basename ${BRANCH_OVERRIDE}) +REQUIREMENTS_FILE=${REDSTACK_SCRIPTS}/files/requirements/fedora-requirements-${ADD_BRANCH}.txt + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" +[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found" +[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set" + +sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt + +# Grab the upper constraints file, but don't fail if we can't find it. +# If we are running in the CI environment, $DEST will be set and stackrc +# will use $DEST/requirements as the location for the requirements repo. +# Use that as it will help us chain a job with something that is changing UC. + +UC_FILE=upper-constraints.txt + +if [ -f "${DEST}/requirements/${UC_FILE}" ]; then + echo "Found ${DEST}/requirements/${UC_FILE}, using that" + sudo -Hiu ${HOST_USERNAME} dd if="${DEST}/requirements/${UC_FILE}" \ + of="${TMP_HOOKS_PATH}/${UC_FILE}" +else + UC_DIR=$(pwd) + UC_BRANCH=${BRANCH_OVERRIDE} + if [ "${ADD_BRANCH}" == "default" ]; then + UC_BRANCH=master + fi + + set +e + curl -o "${UC_DIR}/${UC_FILE}" \ + https://git.openstack.org/cgit/openstack/requirements/plain/${UC_FILE}?h=${UC_BRANCH} + set -e + + if [ -f "${UC_DIR}/${UC_FILE}" ]; then + sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE} + rm -f "${UC_DIR}/${UC_FILE}" + fi +fi diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd b/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd new file mode 100755 index 00000000..960342e3 --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly +# configured + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance" +[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller" +[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host" +[ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir" +[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir" + +sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service diff --git a/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key new file mode 100755 index 00000000..63453a75 --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/extra-data.d/62-ssh-key @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where +# they will be duplicated in the guest VM. +# This process allows the host to log into the guest but more importantly the guest phones home to get the trove +# source + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" + +[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host" + +if [ `whoami` = "root" ]; then + die "This should not be run as root" +fi + +# copy files over the "staging" area for the guest image (they'll later be put in the correct location by the guest user +# not these keys should not be overridden otherwise a) you won't be able to ssh in and b) the guest won't be able to +# rsync the files +if [ -e ${SSH_DIR}/authorized_keys ]; then + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/authorized_keys of=${TMP_HOOKS_PATH}/ssh-authorized-keys + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub +else + die "SSH Authorized Keys file must exist along with pub and private key" +fi diff --git a/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep b/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep new file mode 100755 index 00000000..98fb24ba --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/install.d/15-reddwarf-dep @@ -0,0 +1,30 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu + +set -e +set -o xtrace + +dnf install -y python-devel libxml2-devel libxslt-devel python-setuptools \ + python-sqlalchemy python-lxml \ + python-routes python-eventlet python-webob \ + python-kombu python-paste-deploy python-paste python-netaddr \ + python-httplib2 python-iso8601 python-pip python-mysql \ + python-migrate python-anyjson gcc python-pexpect + +# pick up the requirements file left for us by +# extra-data.d/15-reddwarf-dep + +TMP_HOOKS_DIR="/tmp/in_target.d" + +UPPER_CONSTRAINTS= +if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then + UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt" +fi + +pip install -q --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS} + +echo "diagnostic pip freeze output follows" +pip freeze +echo "diagnostic pip freeze output above" diff --git a/integration/scripts/files/elements/fedora-guest/install.d/20-etc b/integration/scripts/files/elements/fedora-guest/install.d/20-etc new file mode 100755 index 00000000..bec065ef --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/install.d/20-etc @@ -0,0 +1,8 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image + +dd if=/tmp/in_target.d/trove-guest.service of=/usr/lib/systemd/system/trove-guest.service + +systemctl enable trove-guest.service diff --git a/integration/scripts/files/elements/fedora-guest/install.d/50-user b/integration/scripts/files/elements/fedora-guest/install.d/50-user new file mode 100755 index 00000000..a4b666bf --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/install.d/50-user @@ -0,0 +1,17 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist + +set -e +set -o xtrace + +# Difference from apt, -G admin option +if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then + echo "Adding ${GUEST_USERNAME} user" + useradd -m ${GUEST_USERNAME} -s /bin/bash + passwd ${GUEST_USERNAME} <<_EOF_ +${GUEST_USERNAME} +${GUEST_USERNAME} +_EOF_ +fi
\ No newline at end of file diff --git a/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key b/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key new file mode 100755 index 00000000..80c1e65c --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/install.d/62-ssh-key @@ -0,0 +1,29 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory + +set -e +set -o xtrace + +SSH_DIR="/home/${GUEST_USERNAME}/.ssh" +TMP_HOOKS_DIR="/tmp/in_target.d" + +if [ -e "${TMP_HOOKS_DIR}/ssh-authorized-keys" ]; then + if [ ! -e ${SSH_DIR} ]; then + # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME} + mkdir ${SSH_DIR} + chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR} + fi + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/authorized_keys conv=notrunc if=${TMP_HOOKS_DIR}/ssh-authorized-keys + sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/authorized_keys + if [ ! -e "${SSH_DIR}/id_rsa" ]; then + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa + # perms have to be right on this file for ssh to work + sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub + fi +else + echo "SSH Keys were not staged by host" + exit -1 +fi diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding new file mode 100755 index 00000000..4824cfcf --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/post-install.d/05-ipforwarding @@ -0,0 +1,5 @@ +#!/bin/bash +set -e +set -o xtrace +sed -i -r 's/^\s*#(net\.ipv4\.ip_forward=1.*)/\1/' /etc/sysctl.conf +echo 1 > /proc/sys/net/ipv4/ip_forward diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers new file mode 100755 index 00000000..0581fd2b --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/post-install.d/62-trove-guest-sudoers @@ -0,0 +1,15 @@ +#!/bin/bash +set -e +set -o xtrace + +# CONTEXT: HOST after IMAGE BUILD as SCRIPT USER +# PURPOSE: add the guest user account to the /etc/sudoers files with NOPASSWD + +# Adds user to the sudoers file so they can do everything w/o a pass +# Some binaries might be under /sbin or /usr/sbin, so make sure sudo will +# see them by forcing PATH +TEMPFILE=`mktemp` +echo "${GUEST_USERNAME} ALL=(ALL) NOPASSWD:ALL" > $TEMPFILE +chmod 0440 $TEMPFILE +sudo chown root:root $TEMPFILE +sudo mv $TEMPFILE /etc/sudoers.d/60_trove_guest diff --git a/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update b/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update new file mode 100755 index 00000000..cd2992c1 --- /dev/null +++ b/integration/scripts/files/elements/fedora-guest/post-install.d/90-yum-update @@ -0,0 +1,9 @@ +#!/bin/bash + +# CONTEXT: GUEST after packages installed +# PURPOSE: do dnf update to save each instance having to do all the work + +set -e +set -o xtrace + +dnf -y update diff --git a/integration/scripts/files/elements/fedora-mariadb/README.md b/integration/scripts/files/elements/fedora-mariadb/README.md new file mode 100644 index 00000000..757f00b8 --- /dev/null +++ b/integration/scripts/files/elements/fedora-mariadb/README.md @@ -0,0 +1,3 @@ +Sets up a MariaDB server install in the image. + +TODO: auto-tune settings based on host resources or metadata service. diff --git a/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb b/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb new file mode 100755 index 00000000..a5cc2c0e --- /dev/null +++ b/integration/scripts/files/elements/fedora-mariadb/install.d/10-mariadb @@ -0,0 +1,9 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +dnf -y install mariadb-server percona-xtrabackup diff --git a/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr b/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr new file mode 100755 index 00000000..bcc55205 --- /dev/null +++ b/integration/scripts/files/elements/fedora-mariadb/pre-install.d/10-percona-copr @@ -0,0 +1,10 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup COPR Percona repository + +set -e +set -o xtrace + +# install from Fedora repos +dnf -y install percona-xtrabackup diff --git a/integration/scripts/files/elements/fedora-mongodb/README.md b/integration/scripts/files/elements/fedora-mongodb/README.md new file mode 100644 index 00000000..2518abf2 --- /dev/null +++ b/integration/scripts/files/elements/fedora-mongodb/README.md @@ -0,0 +1 @@ +Sets up a MongoDB install in the image.
\ No newline at end of file diff --git a/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb b/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb new file mode 100755 index 00000000..3b53a18b --- /dev/null +++ b/integration/scripts/files/elements/fedora-mongodb/install.d/10-mongodb @@ -0,0 +1,24 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +cat > "/etc/rc.local" << _EOF_ +# Make sure to disable Linux kernel feature transparent huge pages, +# it will affect greatly both memory usage and latency in a negative way. +# See: http://docs.mongodb.org/manual/tutorial/transparent-huge-pages/ +if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then + echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag +fi +if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then + echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled +fi + +exit \$? + +_EOF_ + +dnf -y install mongodb-server diff --git a/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep b/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep new file mode 100755 index 00000000..a3e2633b --- /dev/null +++ b/integration/scripts/files/elements/fedora-mongodb/install.d/25-trove-mongo-dep @@ -0,0 +1,9 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu + +set -e +set -o xtrace + +pip install pymongo>=3.0.2,!=3.1 diff --git a/integration/scripts/files/elements/fedora-mysql/README.md b/integration/scripts/files/elements/fedora-mysql/README.md new file mode 100644 index 00000000..39a6ab8c --- /dev/null +++ b/integration/scripts/files/elements/fedora-mysql/README.md @@ -0,0 +1,3 @@ +Sets up a MySQL server install in the image. + +TODO: auto-tune settings based on host resources or metadata service. diff --git a/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql b/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql new file mode 100755 index 00000000..ff7efa36 --- /dev/null +++ b/integration/scripts/files/elements/fedora-mysql/install.d/10-mysql @@ -0,0 +1,16 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +dnf -y install https://repo.mysql.com/mysql-community-release-fc22.rpm +dnf -y install mysql-community-server + +# move the config dir for now but leave /etc/my.cnf alone +# ln -s creates problems for the systemd script +mkdir /etc/mysql +mv /etc/my.cnf.d /etc/mysql/conf.d +chown -R mysql:mysql /etc/mysql diff --git a/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup b/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup new file mode 100755 index 00000000..9c9709ca --- /dev/null +++ b/integration/scripts/files/elements/fedora-mysql/install.d/40-xtrabackup @@ -0,0 +1,10 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +dnf -y install percona-xtrabackup + diff --git a/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service b/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service new file mode 100644 index 00000000..a7db5d92 --- /dev/null +++ b/integration/scripts/files/elements/fedora-mysql/post-install.d/30-register-mysql-service @@ -0,0 +1,6 @@ +#!/bin/bash + +set -e +set -o xtrace + +# DO NOT enable or start mysqld for systemd, let the guestagent coordinate startup diff --git a/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server b/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server new file mode 100755 index 00000000..9c43c6ef --- /dev/null +++ b/integration/scripts/files/elements/fedora-percona/install.d/05-percona-server @@ -0,0 +1,17 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo + +set -e +set -o xtrace + +curl -o /etc/pki/rpm-gpg/RPM-GPG-KEY-percona http://www.percona.com/downloads/RPM-GPG-KEY-percona +cat <<EOL > /etc/yum.repos.d/Percona.repo +[percona] +name = CentOS \$releasever - Percona +baseurl=http://repo.percona.com/centos/latest/os/\$basearch/ +enabled = 1 +gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-percona +gpgcheck = 1 +EOL
\ No newline at end of file diff --git a/integration/scripts/files/elements/fedora-percona/install.d/10-mysql b/integration/scripts/files/elements/fedora-percona/install.d/10-mysql new file mode 100755 index 00000000..284e81ac --- /dev/null +++ b/integration/scripts/files/elements/fedora-percona/install.d/10-mysql @@ -0,0 +1,16 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +# The fix to make versions of percona-xtrabackup > v2.2 work with Trove +# was put into the mysql guestagent code for Mitaka. There are no current +# plans to backport so we need to make sure the guest generated when the +# tests are run for Kilo or Liberty get the 2.2 verson of PXB +if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then + PXB_VERSION_OVERRIDE="-22" +fi +dnf -y install percona-toolkit Percona-Server-shared-55 Percona-Server-server-55 Percona-Server-test-55 Percona-Server-client-55 percona-xtrabackup${PXB_VERSION_OVERRIDE} diff --git a/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql b/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql new file mode 100755 index 00000000..a2ee4633 --- /dev/null +++ b/integration/scripts/files/elements/fedora-postgresql/install.d/10-postgresql @@ -0,0 +1,83 @@ +#!/bin/sh + +set -e +set -o xtrace + +cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_ +# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html' +# for best practices. +# It is recommended to disable memory overcommit, +# but the Python interpreter may require it on smaller flavors. +# We therefore stick with the heuristic overcommit setting. +vm.overcommit_memory=0 + +_EOF_ + +cat > "/etc/rc.local" << _EOF_ +# See 'http://www.postgresql.org/docs/9.3/static/kernel-resources.html' +# Disable Linux kernel transparent huge pages. This feature is not supported by +# by Postgres 9.3 and may negatively impact performance of the database. +if test -f /sys/kernel/mm/redhat_transparent_hugepage/defrag; then + echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag +fi +if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then + echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled +fi + +exit \$? + +_EOF_ + +dnf install -y http://yum.postgresql.org/9.4/fedora/fedora-22-x86_64/pgdg-fedora94-9.4-4.noarch.rpm + +dnf install -y postgresql94-server postgresql94-contrib postgresql94-devel gcc + +########################################### +# Hack alert: +# For Postgresql 9.4, pg_rewind is not in the main source tree and +# no packages exist in the repos, so it must be compiled manually +# and installed on the image until we can move to 9.5 +# See README at +# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE + +dev_pkgs="readline-devel zlib-devel krb5-devel openssl-devel pam-devel libxml2-devel libxslt-devel" + +yum install -y $dev_pkgs + +# We need pg_config to be accessible on the path +mkdir -p /tmp/build +cd /tmp/build +git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE +git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE + +ln -s /usr/pgsql-9.4/bin/pg_config /usr/bin/pg_config + +cd pg_rewind +make USE_PGXS=1 top_srcdir=/tmp/build/postgres install +ln -s /usr/pgsql-9.4/bin/pg_rewind /usr/bin/pg_rewind + +# Cleanup + +cd +rm -rf /tmp/build +yum remove -y $dev_pkgs + + +# Though /var/lib/pgsql is the preferred directory, need to move it as +# this is where the volume will be mounted +su - postgres -c "/usr/pgsql-9.4/bin/initdb /var/lib/pgsql/9.4/data" +mv /var/lib/pgsql /var/lib/postgresql + +mv /lib/systemd/system/postgresql-9.4.service /lib/systemd/system/postgresql.service + +sed -i 's/PGDATA=\/var\/lib\/pgsql\/9.4\/data/PGDATA=\/var\/lib\/postgresql\/9.4\/data/' /lib/systemd/system/postgresql.service + +# Create a volatile directory for runtime files. +echo "d /var/run/postgresql/ 0755 postgres postgres" > /lib/tmpfiles.d/postgresql.conf + +# Install the native Python client. +dnf install -y postgresql-devel python-devel +pip install psycopg2 + +systemctl enable postgresql.service +systemctl start postgresql.service diff --git a/integration/scripts/files/elements/fedora-redis/README.md b/integration/scripts/files/elements/fedora-redis/README.md new file mode 100644 index 00000000..426072cf --- /dev/null +++ b/integration/scripts/files/elements/fedora-redis/README.md @@ -0,0 +1 @@ +Sets up a redis server install in the image. diff --git a/integration/scripts/files/elements/fedora-redis/install.d/10-redis b/integration/scripts/files/elements/fedora-redis/install.d/10-redis new file mode 100755 index 00000000..8c23b5f6 --- /dev/null +++ b/integration/scripts/files/elements/fedora-redis/install.d/10-redis @@ -0,0 +1,9 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -ex + +dnf -y install redis +service redis start diff --git a/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra b/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra new file mode 100755 index 00000000..2bd40181 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-cassandra/install.d/10-cassandra @@ -0,0 +1,25 @@ +#!/bin/bash + +set -ex +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get install -qy curl +echo "deb http://debian.datastax.com/community stable main" >> /etc/apt/sources.list.d/cassandra.sources.list +curl -L http://debian.datastax.com/debian/repo_key | apt-key add - +apt-get update +apt-get install -qy openjdk-7-jdk expect python-dev +apt-get install -qy libxml2-dev ntp mc +apt-get install -qy libxslt1-dev python-pexpect +apt-get install -qy python-migrate build-essential + +apt-get install dsc21=2.1.* cassandra=2.1.* -qy + +# The Python Driver 2.0 for Apache Cassandra. +pip install cassandra-driver +# Sorted sets support for the Python driver. +pip install blist + +service cassandra stop +rm -rf /var/lib/cassandra/data/system/* +service cassandra start diff --git a/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase b/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase new file mode 100755 index 00000000..0dc67d8b --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-couchbase/install.d/10-couchbase @@ -0,0 +1,8 @@ +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get install -qy curl +apt-get install -qy libssl0.9.8 +curl -O http://packages.couchbase.com/releases/2.2.0/couchbase-server-community_2.2.0_x86_64.deb +INSTALL_DONT_START_SERVER=1 dpkg -i couchbase-server-community_2.2.0_x86_64.deb diff --git a/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb b/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb new file mode 100755 index 00000000..77871d3f --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-couchdb/install.d/10-couchdb @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +# install the ppa-finding tool for ubuntu 12.0.4 release +apt-get install -y python-software-properties +add-apt-repository -y ppa:couchdb/stable +# update cached list of packages +apt-get update -y +# remove any existing couchdb binaries +apt-get remove -yf couchdb couchdb-bin couchdb-common +# install couchdb +apt-get install -yV couchdb +# install curl to provide a way to interact with CouchDB +# over HTTP REST API +apt-get install -qy curl diff --git a/integration/scripts/files/elements/ubuntu-db2/README.md b/integration/scripts/files/elements/ubuntu-db2/README.md new file mode 100644 index 00000000..b488f2db --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-db2/README.md @@ -0,0 +1,36 @@ + Creates an image for DB2 Express-C v10.5 + + The packages for DB2 Express-C can be downloaded from: + http://www-01.ibm.com/software/data/db2/express-c/download.html + and click on the link "DB2 Express-C for Linux 64-bit". + New users can either get an IBM ID or click on the "Proceed without an + IBM ID". User is provided with a registration form which needs to be + completed in order to proceed further to download the DB2 Express-C + packages. After accepting the license agreement, user can download the + the DB2 Express-C package (.tar.gz file). + + There are 2 options for making the DB2 Express-C package accessible to + the Trove disk-image building process: + - place the package in a private repository and set the environment + variable DATASTORE_PKG_LOCATION with the url to this private + repository. + e.g. export DATASTORE_PKG_LOCATION="http://www.foo.com/db2/v10.5_linuxx64_expc.tar.gz" + + - download the package and place it in any directory on the local + filesystem that the trove-integration scripts can access. Set the + environment variable DATASTORE_PKG_LOCATION with the full path to + the downloaded package. + e.g. export DATASTORE_PKG_LOCATION="/home/stack/db2/v10.5_linuxx64_expc.tar.gz" + + The environment variables used are as follows: + + DATASTORE_PKG_LOCATION - is the place where user stores the DB2 + Express-C package after registration. This can either be a + url to a private repository or the full path to the + downloaded package on a local filesystem. + DATASTORE_DOWNLOAD_OPTS - defines any wget options user wants to specify + like user,password, etc. This is an optional variable and is + needed only if specifying a private repository to download + the packages from. + e.g. export DATASTORE_DOWNLOAD_OPTS="--user=foo --password='secret'" + diff --git a/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs b/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs new file mode 100755 index 00000000..f82fd32a --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-db2/extra-data.d/20-copy-db2-pkgs @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: Download the DB2 Express-C v10.5 packages to a directory on the local filesystem or +# to a private repository. The download location is specified using the env variable: +# DATASTORE_PKG_LOCATION + +[ -n "${TMP_HOOKS_PATH}" ] || die "Temp hook path not set" +[ -n "${DATASTORE_PKG_LOCATION}" ] || die "DATASTORE_PKG_LOCATION not set" + +# First check if the package is available on the local filesystem. +if [ -f "${DATASTORE_PKG_LOCATION}" ]; then + echo "Found the DB2 Express-C packages in ${DATASTORE_PKG_LOCATION}." + dd if="${DATASTORE_PKG_LOCATION}" of=${TMP_HOOKS_PATH}/db2.tar.gz +# else, check if the package is available for download in a private repository. +elif wget ${DATASTORE_DOWNLOAD_OPTS} "${DATASTORE_PKG_LOCATION}" -O ${TMP_HOOKS_PATH}/db2.tar.gz; then + echo "Downloaded the DB2 Express-C package from the private repository" +else + echo "Unable to find the DB2 package at ${DATASTORE_PKG_LOCATION}" + echo "Please register and download the DB2 Express-C packages to a private repository or local filesystem." + exit -1 +fi diff --git a/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2 b/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2 new file mode 100755 index 00000000..63bc3bf7 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-db2/install.d/10-db2 @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Uncompress the DB2 packages and install and configure DB2 on Ubuntu. + +# DB2_PKG_LOCATION points to the directory where the DB2 packages +# are located to install. +DB2_PKG_LOCATION="/db2" +mkdir ${DB2_PKG_LOCATION} +cd ${DB2_PKG_LOCATION} + +# DB2 install requires the hostname to be resolved correctly +host_name=`hostname` +echo "127.0.0.1 ${host_name}" >> /etc/hosts + +tar -xvzf /tmp/in_target.d/db2.tar.gz + +# installing dependencies +apt-get install libaio1 +apt-get install libstdc++6 + +# start the installation process. Accepts the default installation directory '/opt/ibm/db2/V10.5' +${DB2_PKG_LOCATION}/expc/db2_install -b /opt/ibm/db2/V10.5 -f sysreq -l ${DB2_PKG_LOCATION}/db2_install.log + +# create the DB2 users. +# DB2 instance owner - db2inst1 +# DB2 fence user - db2fenc1 +# DB2 admin user - db2das1 +useradd -m db2inst1 +useradd -m db2fenc1 +useradd -m db2das1 + +# Create the DB2 server instance +/opt/ibm/db2/V10.5/instance/db2icrt -a server -u db2fenc1 db2inst1 +/opt/ibm/db2/V10.5/cfg/db2ln + +# Configure DB2 server instance to communicate via TCP/IP on a particulat port. +echo 'db2c_db2inst1 50000/tcp # DB2 connection service port' >> /etc/services + +# Configure DB2 to use the TCP/IP settings defined above. +su - db2inst1 -c "db2 update database manager configuration using svcename db2c_db2inst1" + +# Start the actual TCP/IP communication. +su - db2inst1 -c "db2set DB2COMM=tcpip" + +# DB2 requires the hostname to be resolved correctly. Delete this entry from the +# /etc/hosts since this is the hostname of the instance where the image is being +# built. The correct hostname will be set in the guest agent. +sed -i "/127.0.0.1[[:space:]]*${host_name}/d" /etc/hosts diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep new file mode 100755 index 00000000..33b42306 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-reddwarf-dep @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep + +source $_LIB/die + +BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default} +ADD_BRANCH=$(basename ${BRANCH_OVERRIDE}) +REQUIREMENTS_FILE=${REDSTACK_SCRIPTS}/files/requirements/ubuntu-requirements-${ADD_BRANCH}.txt + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" +[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found" +[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set" + +sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt + +# Grab the upper constraints file, but don't fail if we can't find it. +# If we are running in the CI environment, $DEST will be set and stackrc +# will use $DEST/requirements as the location for the requirements repo. +# Use that as it will help us chain a job with something that is changing UC. + +UC_FILE=upper-constraints.txt + +if [ -f "${DEST}/requirements/${UC_FILE}" ]; then + echo "Found ${DEST}/requirements/${UC_FILE}, using that" + sudo -Hiu ${HOST_USERNAME} dd if="${DEST}/requirements/${UC_FILE}" \ + of="${TMP_HOOKS_PATH}/${UC_FILE}" +else + UC_DIR=$(pwd) + UC_BRANCH=${BRANCH_OVERRIDE} + if [ "${ADD_BRANCH}" == "default" ]; then + UC_BRANCH=master + fi + + set +e + curl -o "${UC_DIR}/${UC_FILE}" \ + https://git.openstack.org/cgit/openstack/requirements/plain/${UC_FILE}?h=${UC_BRANCH} + set -e + + if [ -f "${UC_DIR}/${UC_FILE}" ]; then + sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE} + rm -f "${UC_DIR}/${UC_FILE}" + fi +fi diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart new file mode 100755 index 00000000..2b221592 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly +# configured + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance" +[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller" +[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host" +[ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir" +[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir" + +sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key new file mode 100755 index 00000000..63453a75 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e +set -o xtrace + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where +# they will be duplicated in the guest VM. +# This process allows the host to log into the guest but more importantly the guest phones home to get the trove +# source + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" + +[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host" + +if [ `whoami` = "root" ]; then + die "This should not be run as root" +fi + +# copy files over the "staging" area for the guest image (they'll later be put in the correct location by the guest user +# not these keys should not be overridden otherwise a) you won't be able to ssh in and b) the guest won't be able to +# rsync the files +if [ -e ${SSH_DIR}/authorized_keys ]; then + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/authorized_keys of=${TMP_HOOKS_PATH}/ssh-authorized-keys + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa + sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub +else + die "SSH Authorized Keys file must exist along with pub and private key" +fi diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps new file mode 100755 index 00000000..5cd392b2 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps @@ -0,0 +1,10 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install basic services and applications + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install ntp apparmor-utils diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep b/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep new file mode 100755 index 00000000..8d989fe0 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/15-reddwarf-dep @@ -0,0 +1,31 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install python-dev libxml2-dev libxslt1-dev python-setuptools \ + python-pip python-sqlalchemy python-lxml \ + python-routes python-eventlet python-webob \ + python-pastedeploy python-paste python-netaddr \ + python-httplib2 python-iso8601 python-pexpect python-mysqldb python-migrate + + +# pick up the requirements file left for us by +# extra-data.d/15-reddwarf-dep + +TMP_HOOKS_DIR="/tmp/in_target.d" + +UPPER_CONSTRAINTS= +if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then + UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt" +fi + +pip install -q --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS} + +echo "diagnostic pip freeze output follows" +pip freeze +echo "diagnostic pip freeze output above" diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc b/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc new file mode 100755 index 00000000..8ac19f7c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/20-etc @@ -0,0 +1,8 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image + +dd if=/tmp/in_target.d/trove-guest.conf of=/etc/init/trove-guest.conf + + diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user new file mode 100755 index 00000000..99f68966 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user @@ -0,0 +1,18 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist + +set -e +set -o xtrace + +if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then + echo "Adding ${GUEST_USERNAME} user" + useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash + chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME} + passwd ${GUEST_USERNAME} <<_EOF_ +${GUEST_USERNAME} +${GUEST_USERNAME} +_EOF_ +fi + diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key new file mode 100755 index 00000000..8085cbaa --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key @@ -0,0 +1,28 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory + +set -e +set -o xtrace + +SSH_DIR="/home/${GUEST_USERNAME}/.ssh" +TMP_HOOKS_DIR="/tmp/in_target.d" + +if [ -e "${TMP_HOOKS_DIR}/ssh-authorized-keys" ]; then + if [ ! -e ${SSH_DIR} ]; then + # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME} + mkdir ${SSH_DIR} + chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR} + fi + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/authorized_keys conv=notrunc if=${TMP_HOOKS_DIR}/ssh-authorized-keys + if [ ! -e "${SSH_DIR}/id_rsa" ]; then + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa + # perms have to be right on this file for ssh to work + sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa + sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub + fi +else + echo "SSH Keys were not staged by host" + exit -1 +fi diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh new file mode 100755 index 00000000..2134c198 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh @@ -0,0 +1,8 @@ +#!/bin/bash +# Regenerate host keys now. XXX: Really should be a cloud-init task, should get +# that working. + +set -e +set -o xtrace + +dpkg-reconfigure openssh-server diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt new file mode 100755 index 00000000..cc348c5c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt @@ -0,0 +1,11 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Delete contents of apt cache on guest (saves image disk space) + +set -e +set -o xtrace + +apt-get clean + + diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding new file mode 100755 index 00000000..48b7ad0d --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +set -o xtrace +sed -i -r 's/^\s*#(net\.ipv4\.ip_forward=1.*)/\1/' /etc/sysctl.conf diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp b/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp new file mode 100755 index 00000000..bc494ae3 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp @@ -0,0 +1,10 @@ +#!/bin/bash +ntpfile=`mktemp` +cat << EOF > $ntpfile +server ntp.ubuntu.com iburst +server 127.127.1.0 +fudge 127.127.1.0 stratum 10 +EOF + +mv /etc/ntp.conf /etc/ntp.conf.orig +mv $ntpfile /etc/ntp.conf diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers new file mode 100755 index 00000000..0581fd2b --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers @@ -0,0 +1,15 @@ +#!/bin/bash +set -e +set -o xtrace + +# CONTEXT: HOST after IMAGE BUILD as SCRIPT USER +# PURPOSE: add the guest user account to the /etc/sudoers files with NOPASSWD + +# Adds user to the sudoers file so they can do everything w/o a pass +# Some binaries might be under /sbin or /usr/sbin, so make sure sudo will +# see them by forcing PATH +TEMPFILE=`mktemp` +echo "${GUEST_USERNAME} ALL=(ALL) NOPASSWD:ALL" > $TEMPFILE +chmod 0440 $TEMPFILE +sudo chown root:root $TEMPFILE +sudo mv $TEMPFILE /etc/sudoers.d/60_trove_guest diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update b/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update new file mode 100755 index 00000000..6cc087a7 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/90-apt-get-update @@ -0,0 +1,9 @@ +#!/bin/bash + +# CONTEXT: GUEST after packages installed +# PURPOSE: do apt-get update to save each instance having to do all the work + +set -e +set -o xtrace + +apt-get update diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs new file mode 100755 index 00000000..8787df7b --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/01-trim-pkgs @@ -0,0 +1,117 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install basic services and applications + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get -y purge acpid\ + apport\ + apport-symptoms\ + apt-transport-https\ + apt-xapian-index\ + aptitude\ + at\ + bash-completion\ + bc\ + bind9-host\ + bsdmainutils\ + busybox-static\ + byobu\ + command-not-found\ + command-not-found-data\ + curl\ + dbus\ + dmidecode\ + dosfstools\ + ed\ + fonts-ubuntu-font-family-console\ + friendly-recovery\ + ftp\ + fuse\ + geoip-database\ + groff-base\ + hdparm\ + info\ + install-info\ + iptables\ + iputils-tracepath\ + irqbalance\ + landscape-client\ + landscape-common\ + language-selector-common\ + laptop-detect\ + libaccountsservice0\ + libbind9-90\ + libclass-accessor-perl\ + libcwidget3\ + libdns100\ + libept1.4.12\ + libevent-2.0-5\ + libgc1c2\ + libgeoip1\ + libio-string-perl\ + libisc95\ + liblwres90\ + libnfnetlink0\ + libparse-debianchangelog-perl\ + libparted0debian1\ + libpcap0.8\ + libpci3\ + libpipeline1\ + libpolkit-gobject-1-0\ + libsasl2-modules\ + libsigc++-2.0-0c2a\ + libsub-name-perl\ + libusb-1.0-0\ + libxapian22\ + lshw\ + lsof\ + ltrace\ + man-db\ + mlocate\ + mtr-tiny\ + nano\ + ntfs-3g\ + parted\ + patch\ + plymouth-theme-ubuntu-text\ + popularity-contest\ + powermgmt-base\ + ppp\ + pppoeconf\ + python-debian\ + python-gdbm\ + python-pam\ + python-twisted-bin\ + python-xapian\ + python-zope.interface\ + screen\ + shared-mime-info\ + strace\ + tasksel\ + tcpdump\ + telnet\ + time\ + tmux\ + ubuntu-standard\ + ufw\ + update-manager-core\ + update-notifier-common\ + usbutils\ + uuid-runtime\ + w3m + +# The following packages cannot be removed as they cause cloud-init to be +# uninstalled in Ubuntu 14.04 +# gir1.2-glib-2.0 +# libdbus-glib-1-2 +# libgirepository-1.0-1 +# python-chardet +# python-serial +# xz-utils + +apt-get -y autoremove + diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools new file mode 100755 index 00000000..1a8647f5 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools @@ -0,0 +1,7 @@ +#!/bin/bash +# Install baseline packages and tools. + +set -e +set -o xtrace + +apt-get install -y language-pack-en python-software-properties
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-mariadb/README.md b/integration/scripts/files/elements/ubuntu-mariadb/README.md new file mode 100644 index 00000000..757f00b8 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mariadb/README.md @@ -0,0 +1,3 @@ +Sets up a MariaDB server install in the image. + +TODO: auto-tune settings based on host resources or metadata service. diff --git a/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb new file mode 100755 index 00000000..065c2f98 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mariadb/install.d/30-mariadb @@ -0,0 +1,34 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +# NOTE(vkmc): Using MariaDB repositories is required +# https://mariadb.com/kb/en/mariadb/installing-mariadb-deb-files/ +apt-get -y install software-properties-common +apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xcbcb082a1bb943db +add-apt-repository 'deb http://ftp.osuosl.org/pub/mariadb/repo/10.1/ubuntu trusty main' + +# Pin MariaDB repository +sudo echo -e "Package: *\nPin: origin ftp.osuosl.org\nPin-Priority: 1000" > /etc/apt/preferences.d/mariadb.pref + +apt-get -y update +# The fix to make versions of percona-xtrabackup > v2.2 work with Trove +# was put into the mysql guestagent code for Mitaka. There are no current +# plans to backport so we need to make sure the guest generated when the +# tests are run for Kilo or Liberty get the 2.2 verson of PXB +if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then + PXB_VERSION_OVERRIDE="-22" +fi +apt-get -y install socat percona-xtrabackup${PXB_VERSION_OVERRIDE} +apt-get -y install libmariadbclient18 mariadb-server + +cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_ +[mysqld] +performance_schema = off +_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key new file mode 100755 index 00000000..ec1d89d5 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/10-percona-apt-key @@ -0,0 +1,40 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo + +set -e +set -o xtrace + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${RELEASE}" ] || die "RELEASE must be set to either Trusty or Precise" + +# Add Percona GPG key +mkdir -p /home/${GUEST_USERNAME}/.gnupg + +# sometimes the primary key server is unavailable and we should try an +# alternate. see +# https://bugs.launchpad.net/percona-server/+bug/907789. Disable +# shell errexit so we can interrogate the exit code and take action +# based on the exit code. We will reenable it later. +set +e +apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + +if [ "$?" -ne "0" ]; +then + echo "Trying alternate keyserver hkp://keyserver.ubuntu.com" + set -e + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A +fi + +set -e + +# Add Percona repo +# Creates the Percona sources list +cat <<EOL > /etc/apt/sources.list.d/percona.list +deb http://repo.percona.com/apt $RELEASE main +deb-src http://repo.percona.com/apt $RELEASE main +EOL + +# Force an update +apt-get -y update diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local new file mode 100755 index 00000000..a3e1dc7c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +#CONTEXT: chroot on host +#PURPOSE: Allows mysqld to create temporary files when restoring backups + +cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld + /tmp/ rw, + /tmp/** rwk, +EOF diff --git a/integration/scripts/files/elements/ubuntu-mongodb/README.md b/integration/scripts/files/elements/ubuntu-mongodb/README.md new file mode 100644 index 00000000..5b9e33bb --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/README.md @@ -0,0 +1 @@ +Sets up a MongoDB install in the image. diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp b/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp new file mode 100755 index 00000000..48d6c8bf --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/10-mongodb-thp @@ -0,0 +1,42 @@ +#!/bin/sh + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +cat > /etc/init.d/disable-transparent-hugepages << '_EOF_' +#!/bin/sh +### BEGIN INIT INFO +# Provides: disable-transparent-hugepages +# Required-Start: $local_fs +# Required-Stop: +# X-Start-Before: mongod mongodb-mms-automation-agent +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: Disable Linux transparent huge pages +# Description: Disable Linux transparent huge pages, to improve +# database performance. +### END INIT INFO + +case $1 in + start) + if [ -d /sys/kernel/mm/transparent_hugepage ]; then + thp_path=/sys/kernel/mm/transparent_hugepage + elif [ -d /sys/kernel/mm/redhat_transparent_hugepage ]; then + thp_path=/sys/kernel/mm/redhat_transparent_hugepage + else + return 0 + fi + + echo 'never' > ${thp_path}/enabled + echo 'never' > ${thp_path}/defrag + + unset thp_path + ;; +esac +_EOF_ + +chmod 755 /etc/init.d/disable-transparent-hugepages + +update-rc.d disable-transparent-hugepages defaults diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb b/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb new file mode 100755 index 00000000..6a95d2f0 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/20-mongodb @@ -0,0 +1,8 @@ +#!/bin/sh + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +apt-get -y install mongodb-org=3.2.6 diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep b/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep new file mode 100755 index 00000000..a3e2633b --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/25-trove-mongo-dep @@ -0,0 +1,9 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install trove guest python dependencies - see redstack functions_qemu + +set -e +set -o xtrace + +pip install pymongo>=3.0.2,!=3.1 diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf b/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf new file mode 100755 index 00000000..afb58284 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/30-mongodb-conf @@ -0,0 +1,26 @@ +#!/bin/sh + +set -e +set -o xtrace + +# Remove the default pid file +rm -f /var/run/mongodb.pid + + +cat > /etc/mongod.conf << '_EOF_' +storage.dbPath: /var/lib/mongodb +security.authorization: enabled +storage.engine: wiredTiger +storage.journal.enabled: true +systemLog.destination: file +systemLog.logAppend: true +systemLog.path: /var/log/mongodb/mongod.log +_EOF_ + + +cat > /etc/mongos.conf << '_EOF_' +security.authorization: enabled +systemLog.destination: file +systemLog.logAppend: true +systemLog.path: /var/log/mongodb/mongos.log +_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init b/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init new file mode 100755 index 00000000..77a12927 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/41-mongod-init @@ -0,0 +1,46 @@ +#!/bin/sh + +set -e +set -o xtrace + +cat > /etc/init/mongod.conf << '_EOF_' +limit fsize unlimited unlimited +limit cpu unlimited unlimited +limit as unlimited unlimited +limit nofile 64000 64000 +limit rss unlimited unlimited +limit nproc 64000 64000 + +kill timeout 300 # wait 300s between SIGTERM and SIGKILL. + +pre-start script + mkdir -p /var/run/mongodb/ + touch /var/run/mongodb/mongod.pid + chown mongodb -R /var/run/mongodb/ +end script + +start on runlevel [2345] +stop on runlevel [06] + +script + CONF=/etc/mongod.conf + + # Handle NUMA access to CPUs (SERVER-3574) + # This verifies the existence of numactl as well as testing that the command works + NUMACTL_ARGS="--interleave=all" + if which numactl >/dev/null 2>/dev/null && numactl $NUMACTL_ARGS ls / >/dev/null 2>/dev/null + then + NUMACTL="$(which numactl) -- $NUMACTL_ARGS" + DAEMON_OPTS=${DAEMON_OPTS:-"--config $CONF"} + else + NUMACTL="" + DAEMON_OPTS="-- "${DAEMON_OPTS:-"--config $CONF"} + fi + + exec start-stop-daemon --start \ + --chuid mongodb \ + --pidfile /var/run/mongod.pid \ + --make-pidfile \ + --exec $NUMACTL /usr/bin/mongod $DAEMON_OPTS +end script +_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init b/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init new file mode 100755 index 00000000..dc91fdba --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/install.d/42-mongos-init @@ -0,0 +1,30 @@ +#!/bin/sh + +set -e +set -o xtrace + +cat > /etc/init/mongos.conf << '_EOF_' +limit fsize unlimited unlimited +limit cpu unlimited unlimited +limit as unlimited unlimited +limit nofile 64000 64000 +limit rss unlimited unlimited +limit nproc 64000 64000 + +pre-start script + mkdir -p /var/run/mongodb/ + touch /var/run/mongodb/mongos.pid + chown mongodb -R /var/run/mongodb/ +end script + +start on runlevel [2345] +stop on runlevel [06] + +script + exec start-stop-daemon --start \ + --chuid mongodb \ + --pidfile /var/run/mongos.pid \ + --make-pidfile \ + --exec /usr/bin/mongos -- --config /etc/mongos.conf +end script +_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key b/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key new file mode 100755 index 00000000..1538d61c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mongodb/pre-install.d/10-mongodb-apt-key @@ -0,0 +1,14 @@ +#!/bin/sh + +set -e +set -o xtrace + +[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal" + +apt-get -y install software-properties-common + +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927 + +echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list + +apt-get -y update diff --git a/integration/scripts/files/elements/ubuntu-mysql/README.md b/integration/scripts/files/elements/ubuntu-mysql/README.md new file mode 100644 index 00000000..39a6ab8c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mysql/README.md @@ -0,0 +1,3 @@ +Sets up a MySQL server install in the image. + +TODO: auto-tune settings based on host resources or metadata service. diff --git a/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql new file mode 100755 index 00000000..d31292ec --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mysql/install.d/30-mysql @@ -0,0 +1,23 @@ +#!/bin/bash + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +# The fix to make versions of percona-xtrabackup > v2.2 work with Trove +# was put into the mysql guestagent code for Mitaka. There are no current +# plans to backport so we need to make sure the guest generated when the +# tests are run for Kilo or Liberty get the 2.2 verson of PXB +if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then + PXB_VERSION_OVERRIDE="-22" +fi +apt-get -y install libmysqlclient18 mysql-server-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE} + +cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_ +[mysqld] +performance_schema = off +_EOF_ diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key new file mode 100755 index 00000000..2a03ad50 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/10-percona-apt-key @@ -0,0 +1,40 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo + +set -e +set -o xtrace + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal" + +# Add Percona GPG key +mkdir -p /home/${GUEST_USERNAME}/.gnupg + +# sometimes the primary key server is unavailable and we should try an +# alternate. see +# https://bugs.launchpad.net/percona-server/+bug/907789. Disable +# shell errexit so we can interrogate the exit code and take action +# based on the exit code. We will reenable it later. +set +e +apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + +if [ "$?" -ne "0" ]; +then + echo "Trying alternate keyserver hkp://keyserver.ubuntu.com" + set -e + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A +fi + +set -e + +# Add Percona repo +# Creates the percona sources list +cat <<EOL > /etc/apt/sources.list.d/percona.list +deb http://repo.percona.com/apt $RELEASE main +deb-src http://repo.percona.com/apt $RELEASE main +EOL + +# Force an update +apt-get update diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local new file mode 100755 index 00000000..a3e1dc7c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +#CONTEXT: chroot on host +#PURPOSE: Allows mysqld to create temporary files when restoring backups + +cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld + /tmp/ rw, + /tmp/** rwk, +EOF diff --git a/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql new file mode 100755 index 00000000..5d5b4265 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-percona/install.d/30-mysql @@ -0,0 +1,17 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +# The fix to make versions of percona-xtrabackup > v2.2 work with Trove +# was put into the mysql guestagent code for Mitaka. There are no current +# plans to backport so we need to make sure the guest generated when the +# tests are run for Kilo or Liberty get the 2.2 verson of PXB +if [[ $BRANCH_OVERRIDE == "stable/kilo" || $BRANCH_OVERRIDE == "stable/liberty" ]]; then + PXB_VERSION_OVERRIDE="-22" +fi +apt-get -y install percona-toolkit percona-server-common-5.6 percona-server-server-5.6 percona-server-test-5.6 percona-server-client-5.6 percona-xtrabackup${PXB_VERSION_OVERRIDE} diff --git a/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key new file mode 100755 index 00000000..c2b686c4 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/10-percona-apt-key @@ -0,0 +1,42 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo + +set -e +set -o xtrace + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal" + +#5 add Percona GPG key +if [ ! -e /home/${GUEST_USERNAME}/.gnupg ]; then + mkdir -p /home/${GUEST_USERNAME}/.gnupg +fi + +# sometimes the primary key server is unavailable and we should try an +# alternate. see +# https://bugs.launchpad.net/percona-server/+bug/907789. Disable +# shell errexit so we can interrogate the exit code and take action +# based on the exit code. We will reenable it later. +set +e +apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + +if [ "$?" -ne "0" ]; +then + echo "Trying alternate keyserver hkp://keyserver.ubuntu.com" + set -e + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A +fi + +set -e + +# add Percona repo +# creates the percona sources list +cat <<EOL > /etc/apt/sources.list.d/percona.list +deb http://repo.percona.com/apt $RELEASE main +deb-src http://repo.percona.com/apt $RELEASE main +EOL + +# force an update +apt-get update diff --git a/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local new file mode 100755 index 00000000..a3e1dc7c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-percona/pre-install.d/20-apparmor-mysql-local @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +#CONTEXT: chroot on host +#PURPOSE: Allows mysqld to create temporary files when restoring backups + +cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld + /tmp/ rw, + /tmp/** rwk, +EOF diff --git a/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql b/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql new file mode 100755 index 00000000..cd853256 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-postgresql/install.d/10-postgresql @@ -0,0 +1,79 @@ +#!/bin/sh + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_ +# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html' +# for best practices. +# It is recommended to disable memory overcommit, +# but the Python interpreter may require it on smaller flavors. +# We therefore stick with the heuristic overcommit setting. +vm.overcommit_memory=0 +vm.nr_hugepages=64 + +_EOF_ + +cat > "/etc/rc.local" << _EOF_ +# See 'http://www.postgresql.org/docs/9.4/static/kernel-resources.html' +# Postgres 9.4 added support for THP. Using huge pages reduces overhead when +# using large contiguous chunks of memory, like PostgreSQL does. +if test -f /sys/kernel/mm/transparent_hugepage/defrag; then + echo never > /sys/kernel/mm/transparent_hugepage/defrag +fi +if test -f /sys/kernel/mm/transparent_hugepage/enabled; then + echo always > /sys/kernel/mm/transparent_hugepage/enabled +fi + +exit \$? + +_EOF_ + +apt-get -y install postgresql-9.4 postgresql-contrib-9.4 postgresql-server-dev-9.4 + +########################################### +# Hack alert: +# For Postgresql 9.4, pg_rewind is not in the main source tree and +# no packages exist in the repos, so it must be compiled manually +# and installed on the image until we can move to 9.5 +# See README at +# https://github.com/vmware/pg_rewind/tree/REL9_4_STABLE + +tmpdir=/tmp/build +mkdir -p $tmpdir +cd $tmpdir +git clone https://github.com/postgres/postgres.git --branch REL9_4_STABLE +cd postgres/contrib +git clone https://github.com/vmware/pg_rewind.git --branch REL9_4_STABLE + +dev_pkgs="libreadline-dev libkrb5-dev libssl-dev libpam-dev libxml2-dev libxslt-dev libedit-dev libselinux1-dev bison flex" + +apt-get install $dev_pkgs -y + +# Unfortunately, on ubuntu, was not able to get pg_rewind to build +# outside of the pgsql source tree. Configure and compile postgres +# but only call make install against the contrib/pg_rewind directory +# so that support library is accessible to the server +cd $tmpdir/postgres +./configure +make +cd contrib/pg_rewind +make install + +# Make the pg_rewind binary and the library used by the +# pg_rewind stored procedures accessible +ln -s /usr/local/pgsql/bin/pg_rewind /usr/bin/pg_rewind +ln -s /usr/local/pgsql/lib/pg_rewind_support.so /usr/lib/postgresql/9.4/lib/pg_rewind_support.so + +cd +rm -rf $tmpdir +apt-get remove -y $dev_pkgs + +# End hack +################################ + +# Install the native Python client. +apt-get -y install libpq-dev +pip install psycopg2 diff --git a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo new file mode 100755 index 00000000..48a25d1d --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo @@ -0,0 +1,12 @@ +#!/bin/sh + +set -e +set -o xtrace + +cat <<EOL > /etc/apt/sources.list.d/postgresql.list +deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main +EOL + +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - + +apt-get update diff --git a/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql new file mode 100755 index 00000000..ae658957 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-pxc/install.d/30-mysql @@ -0,0 +1,14 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive +apt-get -y install percona-xtradb-cluster-server-5.6 percona-xtradb-cluster-client-5.6 percona-xtrabackup + +# Don't auto start mysql (we'll start it up in guest) +update-rc.d mysql defaults +update-rc.d mysql disable diff --git a/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key new file mode 100755 index 00000000..c2b686c4 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/10-percona-apt-key @@ -0,0 +1,42 @@ +#!/bin/sh + +# CONTEXT: GUEST during PRE-CONSTRUCTION as ROOT +# PURPOSE: Setup apt-repo list so that we can connect to Percona's repo + +set -e +set -o xtrace + +[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" +[ -n "${RELEASE}" ] || die "RELEASE must be set to either Precise or Quantal" + +#5 add Percona GPG key +if [ ! -e /home/${GUEST_USERNAME}/.gnupg ]; then + mkdir -p /home/${GUEST_USERNAME}/.gnupg +fi + +# sometimes the primary key server is unavailable and we should try an +# alternate. see +# https://bugs.launchpad.net/percona-server/+bug/907789. Disable +# shell errexit so we can interrogate the exit code and take action +# based on the exit code. We will reenable it later. +set +e +apt-key adv --keyserver hkp://keys.gnupg.net --recv-keys 1C4CBDCDCD2EFD2A + +if [ "$?" -ne "0" ]; +then + echo "Trying alternate keyserver hkp://keyserver.ubuntu.com" + set -e + apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys 1C4CBDCDCD2EFD2A +fi + +set -e + +# add Percona repo +# creates the percona sources list +cat <<EOL > /etc/apt/sources.list.d/percona.list +deb http://repo.percona.com/apt $RELEASE main +deb-src http://repo.percona.com/apt $RELEASE main +EOL + +# force an update +apt-get update diff --git a/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local new file mode 100755 index 00000000..a3e1dc7c --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-pxc/pre-install.d/20-apparmor-mysql-local @@ -0,0 +1,11 @@ +#!/bin/sh + +set -e + +#CONTEXT: chroot on host +#PURPOSE: Allows mysqld to create temporary files when restoring backups + +cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld + /tmp/ rw, + /tmp/** rwk, +EOF diff --git a/integration/scripts/files/elements/ubuntu-redis/README.md b/integration/scripts/files/elements/ubuntu-redis/README.md new file mode 100644 index 00000000..426072cf --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-redis/README.md @@ -0,0 +1 @@ +Sets up a redis server install in the image. diff --git a/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis b/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis new file mode 100755 index 00000000..57fcdf90 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-redis/install.d/10-redis @@ -0,0 +1,53 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT +# PURPOSE: Install controller base required packages + +set -ex + +export DEBIAN_FRONTEND=noninteractive + +cat > "/etc/sysctl.d/10-redis-performance.conf" << _EOF_ +# See 'http://redis.io/topics/admin' for best practices. +# Make sure to set the Linux kernel overcommit memory setting to 1. +vm.overcommit_memory=1 + +# Linux kernel will silently truncate 'tcp-backlog' to the value of +# '/proc/sys/net/core/somaxconn' so make sure to raise both the value of +# 'somaxconn' and 'tcp_max_syn_backlog' in order to get the desired effect. +net.ipv4.tcp_max_syn_backlog=1024 +net.core.somaxconn=1024 + +_EOF_ + +cat > "/etc/rc.local" << _EOF_ +# Make sure to disable Linux kernel feature transparent huge pages, +# it will affect greatly both memory usage and latency in a negative way. +if test -f /sys/kernel/mm/transparent_hugepage/defrag; then + echo never > /sys/kernel/mm/transparent_hugepage/defrag +fi +if test -f /sys/kernel/mm/transparent_hugepage/enabled; then + echo never > /sys/kernel/mm/transparent_hugepage/enabled +fi + +_EOF_ + +add-apt-repository -y ppa:chris-lea/redis-server +apt-get -y update +apt-get install -y redis-server + +cat > "/etc/default/redis-server" << _EOF_ +# Call ulimit -n with this argument prior to invoking Redis itself. +# This may be required for high-concurrency environments. Redis itself cannot +# alter its limits as it is not being run as root. +ULIMIT=65536 + +_EOF_ + +# Install Python driver for Redis ('redis-py'). +pip install redis + +# By default, redis-py will attempt to use the HiredisParser if installed. +# Using Hiredis can provide up to a 10x speed improvement in parsing responses +# from the Redis server. +pip install hiredis diff --git a/integration/scripts/files/elements/ubuntu-vertica/README.md b/integration/scripts/files/elements/ubuntu-vertica/README.md new file mode 100644 index 00000000..86202379 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-vertica/README.md @@ -0,0 +1 @@ +Sets up a Vertica CE 7.1 debian package and other dependencies install in the image. diff --git a/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb b/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb new file mode 100755 index 00000000..56c02cda --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-vertica/extra-data.d/93-copy-vertica-deb @@ -0,0 +1,14 @@ +#!/bin/bash + +# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER +# PURPOSE: Stages the package installer file from DATASTORE_PKG_LOCATION, +# so that guest image has the package file. + +set -e +set -o xtrace + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" +[ -f "$DATASTORE_PKG_LOCATION" ] || die "Datastore package installer file not found at:" $DATASTORE_PKG_LOCATION +dd if=${DATASTORE_PKG_LOCATION} of=${TMP_HOOKS_PATH}/vertica.deb diff --git a/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica b/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica new file mode 100755 index 00000000..2d50bc6a --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-vertica/install.d/97-vertica @@ -0,0 +1,54 @@ +#!/bin/sh + +# CONTEXT: GUEST during CONSTRUCTION as ROOT - install.d +# PURPOSE: Install controller base required packages + +set -e +set -o xtrace +export DEBIAN_FRONTEND=noninteractive + + +# Copy the package file to the image, +# as it needs to be used later during configuration. +dd if=/tmp/in_target.d/vertica.deb of=/vertica.deb + +# Install base packages +apt-get install -qy build-essential bc iptables +apt-get install -qy curl sysstat pstack mcelog +apt-get install -qy python-dev g++ unixODBC unixODBC-dev dialog +apt-get install -qy dialog libbz2-dev libboost-all-dev libcurl4-gnutls-dev +apt-get install -qy openjdk-7-jdk + +# Install Vertica package +dpkg -i /vertica.deb + +# Creating dbadmin user and verticadba group +groupadd verticadba +useradd -g verticadba -d /home/dbadmin -s /bin/bash -m dbadmin +echo "export PATH=/opt/vertica/bin:\$PATH" >> ~dbadmin/.profile +echo "export TZ=`date +%Z`" >> ~dbadmin/.profile + +# Create base directory for to be used for database creation +mkdir /var/lib/vertica +chown dbadmin:verticadba /var/lib/vertica + +# Backup /etc/hosts +cp -p /etc/hosts /etc/hosts.bkp + +# Compile the SDK examples - the supplied UDFs can then be loaded +cd /opt/vertica/sdk/examples +TMPDIR=/tmp JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64 make +cd + +cat > "/etc/rc.local" << _EOF_ +# Vertica requires THP to be turned off +if test -f /sys/kernel/mm/transparent_hugepage/defrag; then + echo never > /sys/kernel/mm/transparent_hugepage/defrag +fi +if test -f /sys/kernel/mm/transparent_hugepage/enabled; then + echo never > /sys/kernel/mm/transparent_hugepage/enabled +fi + +exit \$? + +_EOF_ diff --git a/integration/scripts/files/keys/authorized_keys b/integration/scripts/files/keys/authorized_keys new file mode 100644 index 00000000..4093a714 --- /dev/null +++ b/integration/scripts/files/keys/authorized_keys @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmQeA/uyEyFf9DsmwR+OztWb7Hb/uTC+R3xG1QgBvRwhSbpBnyBESGMZZ07bIw5Ib7BUSDzwoeryUqNAhAhir2KLeIYODS39UmTwOIl+rIvhlTxhsIoQHV90pewD2qw0T8KgVMPUDsQ0Bd98E6e5dbxciZp67ihVD0r7srhdSRo8PIc56hJWrD52j5FeiIGEmLXHXiZLOyma1M7j/EmiV81wHAzgql6sihWSZHm3xPZZ712JtXbmHhe3RLFIK13u9PSb3XbuEIdGwkZdzP+vYNE0CsYqwjXjVRrY/APsiEkbSNVzHI5p2W1L7ZMtSOMUqZ1Ve+sytVb+YcIJ9L8y07 trove@devstack
\ No newline at end of file diff --git a/integration/scripts/files/keys/id_rsa b/integration/scripts/files/keys/id_rsa new file mode 100644 index 00000000..041e61fd --- /dev/null +++ b/integration/scripts/files/keys/id_rsa @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5kHgP7shMhX/Q7JsEfjs7Vm+x2/7kwvkd8RtUIAb0cIUm6QZ +8gREhjGWdO2yMOSG+wVEg88KHq8lKjQIQIYq9ii3iGDg0t/VJk8DiJfqyL4ZU8Yb +CKEB1fdKXsA9qsNE/CoFTD1A7ENAXffBOnuXW8XImaeu4oVQ9K+7K4XUkaPDyHOe +oSVqw+do+RXoiBhJi1x14mSzspmtTO4/xJolfNcBwM4KperIoVkmR5t8T2We9dib +V25h4Xt0SxSCtd7vT0m9127hCHRsJGXcz/r2DRNArGKsI141Ua2PwD7IhJG0jVcx +yOadltS+2TLUjjFKmdVXvrMrVW/mHCCfS/MtOwIDAQABAoIBAQCTAQHbjmwuHu8e +2F24x7Tz//UWPT9fbRtK/6RO3ctlCsS/bXCHHARnrGcDdfHq1yv6PS21/UvXtThE +Dn4qO75X9DzgnAFNgEwELjPyVBM4YG2pF3SQ+MJESaI4hgGY8Rws5eMF/qFhdbo1 +hATggqFqnQZqWy0DP9wkq8ESk1nYNICehj0d6Mo5uW190TDMD6QrfHg4rDYbgITf +SCmsRdybCASlgOHCrYgjweG9czNoFimhaG8WwP59yfCX0A1TrDJ0toryyopupIre +A+5HHBM2Dk3KylrtVBAPRsM9eGUo4bmz8p6hRkuw7mr321d416MHdIXcHK38EWR0 +ZvUWM8QJAoGBAPlPe3ggR9xhEbQJQkOohnZ1hkogE/uxZrcRiUI3pnDNcCXy9Ogt +SYfwxYnHLa5kuynbmCCzSLOtq2DN9QwJ5o+zgRjX9T8DAWub6KFdEnTHhq/ZLF/w +PWPg3Oe8dYDEEcrPLvN25aetY2LrWKsRPFL8//WLJc1+LFRG6Vc1ATftAoGBAOxv +hGxNGrcD5c8g0ZcyeKVbCCRGfp0+mwonnQ25mDyIXNH+PEHa7SPs5dVexA5r0/Ky +lM/jQgs756EvslwA2oukqVz7ehDhJI4RE92OPjpYrAu7HF6eN/fAhUMghs/vAZ6c +YCM6i9emHYHM4mU6H/yLIr+0e2JNf+479bB5hQTHAoGBAOGnFUQXQ7OukE16C+Yd +RQc3PIMfIbcwTJ3qW2f54sY6zAUtMIptYx6NyN35z9kHB3jNb5Y5b9ZhnLqT7/Yj +h/INMQ4BedK8r65sgVR8X1YfukKzuLxlP8uFHa0KIPiZftkoSYDH3vmzsD86cRj1 +ErqykCH4/hBO4WSugkkSirXJAoGARXvQcvOF8lsW4nRGpCSVCCNklSSSeSu47JcP +tMTiVIfOn3gTxVbNck1gjgA3pfVSaHTK/v1On3aPb/NQe3FUyM0vaMAO8372+zlR +mT8AUq1Ugm4OvE/LKuhNQZkBhYI7+50BM9k0179d1JOdxRn75IAPSj+EMzOLcTv1 +zFMqIGkCgYBm5xT3Gu8fJh/8ztelzrDkGga6UpYKKYjHGFHpaqmDn9sjjCu8X2pG +JUGgyUVj1NkJAtHMS45Ud3upQwxpy2aNmMaQbwzHybvX7EYZHHVuCwsSzaXRtwj2 +Q6mG1Ghi0UQ76SPKQr0Vu8Uu+0CAzYAK4IEKeH6BCRjrzHggSpdNzQ== +-----END RSA PRIVATE KEY-----
\ No newline at end of file diff --git a/integration/scripts/files/keys/id_rsa.pub b/integration/scripts/files/keys/id_rsa.pub new file mode 100644 index 00000000..4093a714 --- /dev/null +++ b/integration/scripts/files/keys/id_rsa.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDmQeA/uyEyFf9DsmwR+OztWb7Hb/uTC+R3xG1QgBvRwhSbpBnyBESGMZZ07bIw5Ib7BUSDzwoeryUqNAhAhir2KLeIYODS39UmTwOIl+rIvhlTxhsIoQHV90pewD2qw0T8KgVMPUDsQ0Bd98E6e5dbxciZp67ihVD0r7srhdSRo8PIc56hJWrD52j5FeiIGEmLXHXiZLOyma1M7j/EmiV81wHAzgql6sihWSZHm3xPZZ712JtXbmHhe3RLFIK13u9PSb3XbuEIdGwkZdzP+vYNE0CsYqwjXjVRrY/APsiEkbSNVzHI5p2W1L7ZMtSOMUqZ1Ve+sytVb+YcIJ9L8y07 trove@devstack
\ No newline at end of file diff --git a/integration/scripts/files/requirements/fedora-requirements-default.txt b/integration/scripts/files/requirements/fedora-requirements-default.txt new file mode 100644 index 00000000..c976bfac --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-default.txt @@ -0,0 +1,31 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.22.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0 +kombu>=2.5.0 +babel>=1.3 +python-heatclient>=0.3.0 +passlib +jinja2>=2.6 +PyMySQL>=0.6.2 # MIT License +python-neutronclient>=2.3.11,<3 +netifaces>=0.10.4 +oslo.config>=1.9.3 # Apache-2.0 +oslo.messaging>=1.8.0 # Apache-2.0 +oslo.i18n>=1.5.0 # Apache-2.0 +oslo.serialization>=1.4.0 # Apache-2.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.4.0 # Apache-2.0 +oslo.log>=1.8.0 # Apache-2.0 +osprofiler>=0.3.0 +oslo.concurrency>=1.8.0 # Apache-2.0 +pexpect>=3.1,!=3.3 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain +xmltodict>=0.10.1 # MIT diff --git a/integration/scripts/files/requirements/fedora-requirements-juno.txt b/integration/scripts/files/requirements/fedora-requirements-juno.txt new file mode 100644 index 00000000..a3ddea3a --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-juno.txt @@ -0,0 +1,19 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools>=0.9.34,!=1.4.0,<=1.5.0 +extras +python-novaclient>=2.18.0,<=2.20.0 +python-swiftclient>=2.2.0,<=2.3.1 +python-cinderclient>=1.1.0,<=1.1.1 +kombu>=2.5.0,<=3.0.7 +six>=1.7.0,<=1.9.0 +Babel>=1.3,<=1.3 +python-heatclient>=0.2.9,<0.3.0 +passlib<=1.6.2 +Jinja2<=2.7.2 +python-neutronclient>=2.3.6,<2.4.0 +netifaces>=0.10.4,<=0.10.4 +oslo.config>=1.4.0,<=1.6.0 # Apache-2.0 +oslo.utils>=1.4.0,<1.5.0 # Apache-2.0 diff --git a/integration/scripts/files/requirements/fedora-requirements-kilo.txt b/integration/scripts/files/requirements/fedora-requirements-kilo.txt new file mode 100644 index 00000000..48bda11b --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-kilo.txt @@ -0,0 +1,24 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools>=0.9.36,!=1.2.0 +extras +python-novaclient>=2.22.0,<2.24.0 +python-swiftclient>=2.2.0,<2.5.0 +python-cinderclient>=1.1.0,<1.2.0 +kombu>=2.5.0 +six>=1.9.0 +Babel>=1.3 +python-heatclient>=0.3.0,<0.5.0 +passlib +Jinja2>=2.6 # BSD License (3 clause) +python-neutronclient>=2.3.11,<2.5.0 +netifaces>=0.10.4 +oslo.config>=1.9.3,<1.10.0 # Apache-2.0 +oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0 +oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0 +oslo.utils>=1.4.0,<1.5.0 # Apache-2.0 +oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0 +oslo.messaging>=1.8.0,<1.9.0 # Apache-2.0 +osprofiler>=0.3.0 # Apache-2.0 diff --git a/integration/scripts/files/requirements/fedora-requirements-liberty.txt b/integration/scripts/files/requirements/fedora-requirements-liberty.txt new file mode 100644 index 00000000..f47cbada --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-liberty.txt @@ -0,0 +1,27 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.22.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +kombu>=2.5.0 +babel>=1.3 +python-heatclient>=0.3.0 +passlib +jinja2>=2.6 +python-neutronclient>=2.3.11,<3 +netifaces>=0.10.4 +oslo.context>=0.2.0,<=1.0.0 +oslo.config>=1.9.3,<1.10.0 # Apache-2.0 +oslo.messaging>=1.8.0 # Apache-2.0 +oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0 +oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.4.0,<1.5.0 # Apache-2.0 +oslo.log>=1.8.0 # Apache-2.0 +osprofiler>=0.3.0 +oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0 +pexpect>=3.1,!=3.3 diff --git a/integration/scripts/files/requirements/fedora-requirements-mitaka.txt b/integration/scripts/files/requirements/fedora-requirements-mitaka.txt new file mode 100644 index 00000000..6f8c478c --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-mitaka.txt @@ -0,0 +1,28 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.22.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +kombu>=2.5.0 +babel>=1.3 +python-heatclient>=0.3.0 +passlib +jinja2>=2.6 +python-neutronclient>=2.3.11,<3 +netifaces>=0.10.4 +oslo.config>=1.9.3 # Apache-2.0 +oslo.messaging>=1.8.0 # Apache-2.0 +oslo.i18n>=1.5.0 # Apache-2.0 +oslo.serialization>=1.4.0 # Apache-2.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.4.0 # Apache-2.0 +oslo.log>=1.8.0 # Apache-2.0 +osprofiler>=0.3.0 +oslo.concurrency>=1.8.0 # Apache-2.0 +pexpect>=3.1,!=3.3 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain diff --git a/integration/scripts/files/requirements/fedora-requirements-newton.txt b/integration/scripts/files/requirements/fedora-requirements-newton.txt new file mode 100644 index 00000000..c976bfac --- /dev/null +++ b/integration/scripts/files/requirements/fedora-requirements-newton.txt @@ -0,0 +1,31 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.22.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0 +kombu>=2.5.0 +babel>=1.3 +python-heatclient>=0.3.0 +passlib +jinja2>=2.6 +PyMySQL>=0.6.2 # MIT License +python-neutronclient>=2.3.11,<3 +netifaces>=0.10.4 +oslo.config>=1.9.3 # Apache-2.0 +oslo.messaging>=1.8.0 # Apache-2.0 +oslo.i18n>=1.5.0 # Apache-2.0 +oslo.serialization>=1.4.0 # Apache-2.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.4.0 # Apache-2.0 +oslo.log>=1.8.0 # Apache-2.0 +osprofiler>=0.3.0 +oslo.concurrency>=1.8.0 # Apache-2.0 +pexpect>=3.1,!=3.3 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain +xmltodict>=0.10.1 # MIT diff --git a/integration/scripts/files/requirements/ubuntu-requirements-default.txt b/integration/scripts/files/requirements/ubuntu-requirements-default.txt new file mode 100644 index 00000000..9607060b --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-default.txt @@ -0,0 +1,30 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.18.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0 +kombu>=2.5.0 +six>=1.7.0 +babel +python-heatclient>=0.2.9 +passlib +jinja2 +PyMySQL>=0.6.2 # MIT License +python-neutronclient>=2.3.6,<3 +netifaces>=0.10.4 +oslo.config>=1.4.0 # Apache-2.0 +oslo.messaging>=1.4.0,!=1.5.0 +oslo.i18n>=1.0.0 +oslo.serialization>=1.0.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.1.0 +osprofiler>=0.3.0 +oslo.concurrency>=0.3.0 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain +xmltodict>=0.10.1 # MIT diff --git a/integration/scripts/files/requirements/ubuntu-requirements-juno.txt b/integration/scripts/files/requirements/ubuntu-requirements-juno.txt new file mode 100644 index 00000000..a3ddea3a --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-juno.txt @@ -0,0 +1,19 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools>=0.9.34,!=1.4.0,<=1.5.0 +extras +python-novaclient>=2.18.0,<=2.20.0 +python-swiftclient>=2.2.0,<=2.3.1 +python-cinderclient>=1.1.0,<=1.1.1 +kombu>=2.5.0,<=3.0.7 +six>=1.7.0,<=1.9.0 +Babel>=1.3,<=1.3 +python-heatclient>=0.2.9,<0.3.0 +passlib<=1.6.2 +Jinja2<=2.7.2 +python-neutronclient>=2.3.6,<2.4.0 +netifaces>=0.10.4,<=0.10.4 +oslo.config>=1.4.0,<=1.6.0 # Apache-2.0 +oslo.utils>=1.4.0,<1.5.0 # Apache-2.0 diff --git a/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt b/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt new file mode 100644 index 00000000..48bda11b --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-kilo.txt @@ -0,0 +1,24 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools>=0.9.36,!=1.2.0 +extras +python-novaclient>=2.22.0,<2.24.0 +python-swiftclient>=2.2.0,<2.5.0 +python-cinderclient>=1.1.0,<1.2.0 +kombu>=2.5.0 +six>=1.9.0 +Babel>=1.3 +python-heatclient>=0.3.0,<0.5.0 +passlib +Jinja2>=2.6 # BSD License (3 clause) +python-neutronclient>=2.3.11,<2.5.0 +netifaces>=0.10.4 +oslo.config>=1.9.3,<1.10.0 # Apache-2.0 +oslo.i18n>=1.5.0,<1.6.0 # Apache-2.0 +oslo.serialization>=1.4.0,<1.5.0 # Apache-2.0 +oslo.utils>=1.4.0,<1.5.0 # Apache-2.0 +oslo.concurrency>=1.8.0,<1.9.0 # Apache-2.0 +oslo.messaging>=1.8.0,<1.9.0 # Apache-2.0 +osprofiler>=0.3.0 # Apache-2.0 diff --git a/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt b/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt new file mode 100644 index 00000000..1f3d0d7b --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-liberty.txt @@ -0,0 +1,26 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.18.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +kombu>=2.5.0 +six>=1.7.0 +babel +python-heatclient>=0.2.9 +passlib +jinja2 +python-neutronclient>=2.3.6,<3 +netifaces>=0.10.4 +oslo.context>=0.2.0,<=1.0.0 +oslo.config>=1.4.0 # Apache-2.0 +oslo.messaging>=1.4.0,!=1.5.0 +oslo.i18n>=1.0.0 +oslo.serialization>=1.0.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.1.0 +osprofiler>=0.3.0 +oslo.concurrency>=0.3.0 diff --git a/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt b/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt new file mode 100644 index 00000000..8724d163 --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-mitaka.txt @@ -0,0 +1,27 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.18.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +kombu>=2.5.0 +six>=1.7.0 +babel +python-heatclient>=0.2.9 +passlib +jinja2 +python-neutronclient>=2.3.6,<3 +netifaces>=0.10.4 +oslo.config>=1.4.0 # Apache-2.0 +oslo.messaging>=1.4.0,!=1.5.0 +oslo.i18n>=1.0.0 +oslo.serialization>=1.0.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.1.0 +osprofiler>=0.3.0 +oslo.concurrency>=0.3.0 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain diff --git a/integration/scripts/files/requirements/ubuntu-requirements-newton.txt b/integration/scripts/files/requirements/ubuntu-requirements-newton.txt new file mode 100644 index 00000000..9607060b --- /dev/null +++ b/integration/scripts/files/requirements/ubuntu-requirements-newton.txt @@ -0,0 +1,30 @@ +# The order of packages is significant, because pip processes them in the order +# of appearance. Changing the order has an impact on the overall integration +# process, which may cause wedges in the gate later. +unittest2 +testtools +extras +python-novaclient>=2.18.0 +python-swiftclient>=2.2.0 +python-cinderclient>=1.1.0 +python-keystoneclient>=2.0.0,!=2.1.0 # Apache-2.0 +kombu>=2.5.0 +six>=1.7.0 +babel +python-heatclient>=0.2.9 +passlib +jinja2 +PyMySQL>=0.6.2 # MIT License +python-neutronclient>=2.3.6,<3 +netifaces>=0.10.4 +oslo.config>=1.4.0 # Apache-2.0 +oslo.messaging>=1.4.0,!=1.5.0 +oslo.i18n>=1.0.0 +oslo.serialization>=1.0.0 +oslo.service>=0.1.0 # Apache-2.0 +oslo.utils>=1.1.0 +osprofiler>=0.3.0 +oslo.concurrency>=0.3.0 +enum34;python_version=='2.7' or python_version=='2.6' or python_version=='3.3' # BSD +pycrypto>=2.6 # Public Domain +xmltodict>=0.10.1 # MIT diff --git a/integration/scripts/files/trove-guest.systemd.conf b/integration/scripts/files/trove-guest.systemd.conf new file mode 100644 index 00000000..47bc2cf8 --- /dev/null +++ b/integration/scripts/files/trove-guest.systemd.conf @@ -0,0 +1,32 @@ +[Unit] +Description=Trove Guest +After=syslog.target +After=network.target + +[Service] +Type=simple +User=GUEST_USERNAME +Group=GUEST_USERNAME + +ExecStartPre=/bin/bash -c "sudo mkdir -p GUEST_LOGDIR ; sudo chown GUEST_USERNAME:root GUEST_LOGDIR" + +# If ~/trove-installed does not exist, copy the trove source from +# the user's development environment, then touch the sentinel file +ExecStartPre=/bin/bash -c "test -d /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed" + +# If /etc/trove does not exist, create it and then copy the trove-guestagent.conf +# from /etc/trove on the user's development environment, +ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf" + +ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove" + +ExecStart=/home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d + +# Give a reasonable amount of time for the server to start up/shut down +TimeoutSec=300 + +# PgSql doesn't play nice with PrivateTmp +PrivateTmp=false + +[Install] +WantedBy=multi-user.target diff --git a/integration/scripts/files/trove-guest.upstart.conf b/integration/scripts/files/trove-guest.upstart.conf new file mode 100644 index 00000000..2e20d6cf --- /dev/null +++ b/integration/scripts/files/trove-guest.upstart.conf @@ -0,0 +1,40 @@ +description "Trove Guest" +author "Auto-Gen" + +start on (filesystem and net-device-up IFACE!=lo) +stop on runlevel [016] +chdir /var/run +pre-start script + mkdir -p /var/run/trove + chown GUEST_USERNAME:root /var/run/trove/ + + mkdir -p /var/lock/trove + chown GUEST_USERNAME:root /var/lock/trove/ + + mkdir -p GUEST_LOGDIR + chown GUEST_USERNAME:root GUEST_LOGDIR + + # Copy the trove source from the user's development environment + if [ ! -d /home/GUEST_USERNAME/trove ]; then + sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove + fi + + # Ensure conf dir exists and is readable + mkdir -p /etc/trove/conf.d + chmod -R +r /etc/trove +end script + +script + # For backwards compatibility until https://review.openstack.org/#/c/100381 merges + TROVE_CONFIG="--config-dir=/etc/trove/conf.d" + if [ ! -f /etc/trove/conf.d/guest_info ] && [ ! -f /etc/trove/conf.d/trove-guestagent.conf ]; then + + chmod +r /etc/guest_info + sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ + mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/trove-guestagent.conf + TROVE_CONFIG="--config-file=/etc/guest_info --config-file=/etc/trove/trove-guestagent.conf" + + fi + + exec su -c "/home/GUEST_USERNAME/trove/contrib/trove-guestagent $TROVE_CONFIG" GUEST_USERNAME +end script diff --git a/integration/scripts/functions b/integration/scripts/functions new file mode 100644 index 00000000..1a09f552 --- /dev/null +++ b/integration/scripts/functions @@ -0,0 +1,324 @@ +#!/bin/bash +# This file format was stolen from devstack <3 + +# This method was stolen from devstack +# git clone only if directory doesn't exist already. Since ``DEST`` might not +# be owned by the installation user, we create the directory and change the +# ownership to the proper user. +# Set global RECLONE=yes to simulate a clone when dest-dir exists +# git_clone remote dest-dir branch +function git_clone { + [[ "$OFFLINE" = "True" ]] && return + + GIT_REMOTE=$1 + GIT_DEST=$2 + GIT_BRANCH=$3 + + if echo $GIT_BRANCH | egrep -q "^refs"; then + # If our branch name is a gerrit style refs/changes/... + if [[ ! -d $GIT_DEST ]]; then + git_timed clone $GIT_REMOTE $GIT_DEST + fi + cd $GIT_DEST + git_timed fetch $GIT_REMOTE $GIT_BRANCH && git_timed checkout FETCH_HEAD + else + # do a full clone only if the directory doesn't exist + if [[ ! -d $GIT_DEST ]]; then + git_timed clone $GIT_REMOTE $GIT_DEST + cd $GIT_DEST + # This checkout syntax works for both branches and tags + git_timed checkout $GIT_BRANCH + elif [[ "$RECLONE" == "yes" ]]; then + # if it does exist then simulate what clone does if asked to RECLONE + cd $GIT_DEST + # set the url to pull from and fetch + git_timed remote set-url origin $GIT_REMOTE + git_timed fetch origin + # remove the existing ignored files (like pyc) as they cause breakage + # (due to the py files having older timestamps than our pyc, so python + # thinks the pyc files are correct using them) + find $GIT_DEST -name '*.pyc' -delete + git_timed checkout -f origin/$GIT_BRANCH + # a local branch might not exist + git_timed branch -D $GIT_BRANCH || true + git_timed checkout -b $GIT_BRANCH + fi + fi +} + +# Determinate is the given option present in the INI file +# ini_has_option config-file section option +function ini_has_option() { + local file=$1 + local section=$2 + local option=$3 + local line + line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file") + [ -n "$line" ] +} + +# Get an option from an INI file +# iniget config-file section option +function iniget() { + local file=$1 + local section=$2 + local option=$3 + local line + line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file") + echo ${line#*=} +} + +# Set an option in an INI file +# iniset config-file section option value +function iniset() { + local file=$1 + local section=$2 + local option=$3 + local value=$4 + if ! grep -q "^\[$section\]" "$file"; then + # Add section at the end + echo -e "\n[$section]" >>"$file" + fi + if ! ini_has_option "$file" "$section" "$option"; then + # Add it + sed -i -e "/^\[$section\]/ a\\ +$option = $value +" "$file" + else + # Replace it + sed -i -e "/^\[$section\]/,/^\[.*\]/ s|^\($option[ \t]*=[ \t]*\).*$|\1$value|" "$file" + fi +} + +# Determine OS Vendor, Release and Update +# Tested with OS/X, Ubuntu, RedHat, CentOS, Fedora +# Returns results in global variables: +# os_VENDOR - vendor name +# os_RELEASE - release +# os_UPDATE - update +# os_PACKAGE - package type +# os_CODENAME - vendor's codename for release +# GetOSVersion +GetOSVersion() { + # Figure out which vendor we are + if [[ -x "`which sw_vers 2>/dev/null`" ]]; then + # OS/X + os_VENDOR=`sw_vers -productName` + os_RELEASE=`sw_vers -productVersion` + os_UPDATE=${os_RELEASE##*.} + os_RELEASE=${os_RELEASE%.*} + os_PACKAGE="" + if [[ "$os_RELEASE" =~ "10.7" ]]; then + os_CODENAME="lion" + elif [[ "$os_RELEASE" =~ "10.6" ]]; then + os_CODENAME="snow leopard" + elif [[ "$os_RELEASE" =~ "10.5" ]]; then + os_CODENAME="leopard" + elif [[ "$os_RELEASE" =~ "10.4" ]]; then + os_CODENAME="tiger" + elif [[ "$os_RELEASE" =~ "10.3" ]]; then + os_CODENAME="panther" + else + os_CODENAME="" + fi + elif [[ -x $(which lsb_release 2>/dev/null) ]]; then + os_VENDOR=$(lsb_release -i -s) + os_RELEASE=$(lsb_release -r -s) + os_UPDATE="" + os_PACKAGE="rpm" + if [[ "Debian,Ubuntu,LinuxMint" =~ $os_VENDOR ]]; then + os_PACKAGE="deb" + elif [[ "SUSE LINUX" =~ $os_VENDOR ]]; then + lsb_release -d -s | grep -q openSUSE + if [[ $? -eq 0 ]]; then + os_VENDOR="openSUSE" + fi + elif [[ $os_VENDOR == "openSUSE project" ]]; then + os_VENDOR="openSUSE" + elif [[ $os_VENDOR =~ Red.*Hat ]]; then + os_VENDOR="Red Hat" + fi + os_CODENAME=$(lsb_release -c -s) + elif [[ -r /etc/redhat-release ]]; then + # Red Hat Enterprise Linux Server release 5.5 (Tikanga) + # Red Hat Enterprise Linux Server release 7.0 Beta (Maipo) + # CentOS release 5.5 (Final) + # CentOS Linux release 6.0 (Final) + # Fedora release 16 (Verne) + # XenServer release 6.2.0-70446c (xenenterprise) + os_CODENAME="" + for r in "Red Hat" CentOS Fedora XenServer; do + os_VENDOR=$r + if [[ -n "`grep \"$r\" /etc/redhat-release`" ]]; then + ver=`sed -e 's/^.* \([0-9].*\) (\(.*\)).*$/\1\|\2/' /etc/redhat-release` + os_CODENAME=${ver#*|} + os_RELEASE=${ver%|*} + os_UPDATE=${os_RELEASE##*.} + os_RELEASE=${os_RELEASE%.*} + break + fi + os_VENDOR="" + done + os_PACKAGE="rpm" + elif [[ -r /etc/SuSE-release ]]; then + for r in openSUSE "SUSE Linux"; do + if [[ "$r" = "SUSE Linux" ]]; then + os_VENDOR="SUSE LINUX" + else + os_VENDOR=$r + fi + + if [[ -n "`grep \"$r\" /etc/SuSE-release`" ]]; then + os_CODENAME=`grep "CODENAME = " /etc/SuSE-release | sed 's:.* = ::g'` + os_RELEASE=`grep "VERSION = " /etc/SuSE-release | sed 's:.* = ::g'` + os_UPDATE=`grep "PATCHLEVEL = " /etc/SuSE-release | sed 's:.* = ::g'` + break + fi + os_VENDOR="" + done + os_PACKAGE="rpm" + # If lsb_release is not installed, we should be able to detect Debian OS + elif [[ -f /etc/debian_version ]] && [[ $(cat /proc/version) =~ "Debian" ]]; then + os_VENDOR="Debian" + os_PACKAGE="deb" + os_CODENAME=$(awk '/VERSION=/' /etc/os-release | sed 's/VERSION=//' | sed -r 's/\"|\(|\)//g' | awk '{print $2}') + os_RELEASE=$(awk '/VERSION_ID=/' /etc/os-release | sed 's/VERSION_ID=//' | sed 's/\"//g') + fi + export os_VENDOR os_RELEASE os_UPDATE os_PACKAGE os_CODENAME +} + + +# Translate the OS version values into common nomenclature +# Sets ``DISTRO`` from the ``os_*`` values +function GetDistro() { + GetOSVersion + if [[ "$os_VENDOR" =~ (Ubuntu) || "$os_VENDOR" =~ (Debian) ]]; then + # 'Everyone' refers to Ubuntu / Debian releases by the code name adjective + DISTRO=$os_CODENAME + elif [[ "$os_VENDOR" =~ (Fedora) ]]; then + # For Fedora, just use 'f' and the release + DISTRO="f$os_RELEASE" + elif [[ "$os_VENDOR" =~ (openSUSE) ]]; then + DISTRO="opensuse-$os_RELEASE" + elif [[ "$os_VENDOR" =~ (SUSE LINUX) ]]; then + # For SLE, also use the service pack + if [[ -z "$os_UPDATE" ]]; then + DISTRO="sle${os_RELEASE}" + else + DISTRO="sle${os_RELEASE}sp${os_UPDATE}" + fi + elif [[ "$os_VENDOR" =~ (Red Hat) || "$os_VENDOR" =~ (CentOS) ]]; then + # Drop the . release as we assume it's compatible + DISTRO="rhel${os_RELEASE::1}" + elif [[ "$os_VENDOR" =~ (XenServer) ]]; then + DISTRO="xs$os_RELEASE" + else + # Catch-all for now is Vendor + Release + Update + DISTRO="$os_VENDOR-$os_RELEASE.$os_UPDATE" + fi + export DISTRO +} + + +# Determine if current distribution is a Fedora-based distribution +# (Fedora, RHEL, CentOS, etc). +# is_fedora +function is_fedora { + if [[ -z "$os_VENDOR" ]]; then + GetOSVersion + fi + + [ "$os_VENDOR" = "Fedora" ] || [ "$os_VENDOR" = "Red Hat" ] || [ "$os_VENDOR" = "CentOS" ] +} + + +# Determine if current distribution is a SUSE-based distribution +# (openSUSE, SLE). +# is_suse +function is_suse { + if [[ -z "$os_VENDOR" ]]; then + GetOSVersion + fi + + [ "$os_VENDOR" = "openSUSE" ] || [ "$os_VENDOR" = "SUSE LINUX" ] +} + +# Get the path to the direcotry where python executables are installed. +# get_python_exec_prefix +function get_python_exec_prefix() { + if is_fedora || is_suse; then + echo "/usr/bin" + else + echo "/usr/local/bin" + fi +} + +# Returns 'true' if 'true', 'yes', 'on' or '1' - false, otherwise. +# Converts values to lower case first. +# If no default is provided, assumes false. +function get_bool() { + local VARIABLE="$1" + local DEFAULT=${2:-false} + + VALUE=${!VARIABLE:-$DEFAULT} + + VALUE=$(eval echo "$VALUE" | tr '[:upper:]' '[:lower:]') + if [[ "1 yes true on" =~ "$VALUE" ]]; then + VALUE=true + else + VALUE=false + fi + + echo $VALUE +} + +# Get the project branch to switch to. Uses PROJECT_BRANCH_NAME, +# then BRANCH_DEFAULT, then 'master' +function get_project_branch() { + local PROJECT_BRANCH_NAME=$1 + local BRANCH_DEFAULT=${2:-master} + + PROJECT_BRANCH_OVERRIDE=${!PROJECT_BRANCH_NAME} + + BRANCH=${PROJECT_BRANCH_OVERRIDE:-$BRANCH_DEFAULT} + + echo "$BRANCH" +} + +# Try to switch to a branch or commit in a repo +# Fails if the branch/commit doesn't exist +function git_checkout() { + local PROJECT=$1 + local REPO_DIR=$2 + local REPO_BRANCH=$3 + local REPO_BRANCH_VAR_NAME=$4 + + if [ -n "$REPO_BRANCH" ]; then + pushd "$REPO_DIR" + if [ $PROJECT == "diskimage-builder" ] || [ $PROJECT == "tripleo-image-elements" ]; then + REPO_BRANCH=master + fi + CURRENT_BRANCH=$(git branch | grep "\*" | awk '{print $2}') + GIT_STATUS=$(git checkout "$REPO_BRANCH" &> /dev/null || echo "failed") + if [[ "$GIT_STATUS" = "failed" ]]; then + exclaim "${COLOR_RED}Could not switch to branch/commit '$REPO_BRANCH' in $PROJECT, exiting${COLOR_NONE}" + echo "Please set '$REPO_BRANCH_VAR_NAME' to a valid branch/commit and try again." + if [[ "$CURRENT_BRANCH" != "master" ]]; then + echo "Note: This repo is currently on branch ${CURRENT_BRANCH} - if this is correct," + echo "you should set $REPO_BRANCH_VAR_NAME=${CURRENT_BRANCH} and re-run your command." + else + echo "Note: This error may also mean that there are modified files in $PROJECT." + echo " If that is the case, please stash them and re-run your command." + fi + exit 1 + else + if [[ "$REPO_BRANCH" != "$CURRENT_BRANCH" ]]; then + exclaim "${COLOR_BLUE}Switched to $PROJECT branch '$REPO_BRANCH'${COLOR_NONE}" + else + echo "Using $PROJECT branch '$REPO_BRANCH'" + fi + fi + popd + fi +} + diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu new file mode 100644 index 00000000..62875eec --- /dev/null +++ b/integration/scripts/functions_qemu @@ -0,0 +1,174 @@ +#!/bin/bash +# +# Additional functions that would mostly just pertain to a Ubuntu + Qemu setup +# + +function build_vm() { + exclaim "Actually building the image, this can take up to 15 minutes" + + # set variables here and ensure they are not changed during the duration of this script + readonly HOMEDIR=$1 + readonly HOST_USERNAME=$2 + GUEST_USERNAME=${GUEST_USERNAME:-$2} + HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$2} + VM=$3 + DISTRO=$4 + SERVICE_TYPE=$5 + + readonly SSH_DIR=${KEY_DIR:-${HOMEDIR}/.ssh} + manage_ssh_keys + + if [ $DISTRO == 'ubuntu' ]; then + export RELEASE=trusty + export DIB_RELEASE=$RELEASE + export DIB_CLOUD_IMAGES=cloud-images.ubuntu.com + # Use the apt sources.list on the build host, its almost always preferred + if [ -f /etc/apt/sources.list ]; then + export DIB_APT_SOURCES=/etc/apt/sources.list + EXTRA_ELEMENTS="apt-sources apt-conf-dir" + fi + fi + if [ $DISTRO == 'fedora' ]; then + EXTRA_ELEMENTS=selinux-permissive + fi + + export HOST_USERNAME + export HOST_SCP_USERNAME + export GUEST_USERNAME + export CONTROLLER_IP + export REDSTACK_SCRIPTS + export SERVICE_TYPE + export PATH_TROVE + export ESCAPED_PATH_TROVE + export SSH_DIR + export GUEST_LOGDIR + export ESCAPED_GUEST_LOGDIR + export ELEMENTS_PATH=$REDSTACK_SCRIPTS/files/elements + export ELEMENTS_PATH+=:$PATH_DISKIMAGEBUILDER/elements + export ELEMENTS_PATH+=:$PATH_TRIPLEO_ELEMENTS/elements + export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive" + export DATASTORE_PKG_LOCATION + export BRANCH_OVERRIDE + export DIB_APT_CONF_DIR=/etc/apt/apt.conf.d + export DIB_CLOUD_INIT_ETC_HOSTS=true + local QEMU_IMG_OPTIONS=$(! $(qemu-img | grep -q 'version 1') && echo "--qemu-img-options compat=0.10") + ${PATH_DISKIMAGEBUILDER}/bin/disk-image-create -a amd64 -o "${VM}" \ + -x ${QEMU_IMG_OPTIONS} ${DISTRO} ${EXTRA_ELEMENTS} vm heat-cfntools \ + cloud-init-datasources ${DISTRO}-guest ${DISTRO}-${SERVICE_TYPE} +} + +function build_guest_image() { + exclaim "Building an image for use with development and integration tests." + if [ -z "$1" ] + then + echo "You must pass an image type to build, like mysql" + exit 1 + fi + SERVICE_TYPE=$1 + VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc' + if ! [[ " $VALID_SERVICES " =~ " $SERVICE_TYPE " ]]; then + exclaim "You did not pass in a valid image type. Valid types are:" $VALID_SERVICES + exit 1 + fi + + GUEST_LOGDIR=$(iniget $PATH_TROVE/etc/trove/trove-guestagent.conf.sample DEFAULT log_dir) + GUEST_LOGFILE=$(iniget $PATH_TROVE/etc/trove/trove-guestagent.conf.sample DEFAULT log_file) + + if [ -z $GUEST_LOGDIR ] || [ -z $GUEST_LOGFILE ] + then + exclaim "error: log_dir and log_file are required in: " $PATH_TROVE/etc/trove/trove-guestagent.conf.sample + exit 1 + fi + + ESCAPED_GUEST_LOGDIR=`echo $GUEST_LOGDIR | sed 's/\//\\\\\//g'` + + USERNAME=`whoami` + # To change the distro, edit the redstack.rc file + readonly IMAGENAME=${DISTRO}_${SERVICE_TYPE} + readonly VM_PATH=$USERHOME/images/${IMAGENAME} + readonly VM_PATH_NAME=${VM_PATH}/${IMAGENAME} + mkdir -p $VM_PATH + + # If the path does not exist, build it, otherwise just upload it + # (unless we're explicitly told to rebuild it) + REBUILD_IMAGE=$(echo "${REBUILD_IMAGE}" | tr '[:upper:]' '[:lower:]') + if [ "${REBUILD_IMAGE}" = "true" ] || [ ! -d $VM_PATH ] || [ `ls -1 $VM_PATH | wc -l` -eq '0' ] + then + if [ "${REBUILD_IMAGE}" = "true" ] + then + exclaim "Rebuilding image" + rm -rf "${VM_PATH}" + fi + build_vm $USERHOME $USERNAME $VM_PATH_NAME $DISTRO $SERVICE_TYPE + touch -c "${VM_PATH}" + else + exclaim "Found image in $VM_PATH - using the qcow2 image found here..." + ELEMENTS_DIR="files/elements/${DISTRO}-${SERVICE_TYPE}" + ELEMENTS_DIR_GUEST="files/elements/${DISTRO}-guest" + # Print out a warning on all the elements files that are newer than the image. + # Directories are not excluded as that is the only way to determine if a file + # has been removed. + # The rebuild is not automatically triggered as there are valid reasons for a + # new file to be present (rollback of change, inadvertent .log files present, + # feature half implemented, etc.). + IMAGE_OLD= + while IFS= read -r -d '' ELEMENT_FILE + do + if [ "${ELEMENT_FILE}" -nt "${VM_PATH}" ] + then + IMAGE_OLD=true + exclaim "${COLOR_RED}WARNING: Element file '${ELEMENT_FILE}' is newer than cached image${COLOR_NONE}" + fi + done < <(find "${ELEMENTS_DIR}" "${ELEMENTS_DIR_GUEST}" -depth -print0) + if [ "${IMAGE_OLD}" = "true" ] + then + exclaim "${COLOR_RED}Use ${COLOR_NONE}REBUILD_IMAGE=True${COLOR_RED} to rebuild image${COLOR_NONE}" + fi + + fi +} + +function clean_instances() { + LIST=`virsh -q list|awk '{print $1}'` + for i in $LIST; do sudo virsh destroy $i; done +} + +function manage_ssh_keys() { + if [ -e ${SSH_DIR} ]; then + echo "${SSH_DIR} already exists" + else + echo "Creating ${SSH_DIR} for ${HOST_USERNAME}" + sudo -Hiu ${HOST_USERNAME} mkdir -m go-w -p ${SSH_DIR} + fi + + if [ ! -f ${SSH_DIR}/id_rsa.pub ]; then + sudo ${PKG_MGR} ${PKG_GET_ARGS} install expect + generate_empty_passphrase_ssh_key ${HOST_USERNAME} + fi + + add_host_key_to_authorizedkeys +} + +function generate_empty_passphrase_ssh_key () { + echo "generating a empty passphrase DEV ONLY rsa key" + expect -c " +spawn sudo -Hiu ${HOST_USERNAME} /usr/bin/ssh-keygen -f ${SSH_DIR}/id_rsa -q +expect \"empty for no passphrase\" +send \n +expect assphrase +send \n +expect eof" +} + +function add_host_key_to_authorizedkeys () { + # test to see if the host key is already in its own authorized_keys file - if not then add it. This is then later copied + # to the guest image + is_in_keyfile=`cat ${SSH_DIR}/id_rsa.pub | grep -f - ${SSH_DIR}/authorized_keys | wc -l` + if [ $is_in_keyfile == 0 ]; then + echo "Adding keyfile to authorized_keys, it does not yet exist" + cat ${SSH_DIR}/id_rsa.pub >> ${SSH_DIR}/authorized_keys + chmod 600 ${SSH_DIR}/authorized_keys + else + echo "Keyfile already exists in authorized_keys - skipping" + fi +} diff --git a/integration/scripts/image-projects-list b/integration/scripts/image-projects-list new file mode 100644 index 00000000..76d0da45 --- /dev/null +++ b/integration/scripts/image-projects-list @@ -0,0 +1,3 @@ +diskimage-builder +tripleo-image-elements +trove diff --git a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc new file mode 100644 index 00000000..9b80b6c6 --- /dev/null +++ b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc @@ -0,0 +1,3 @@ +[[post-config|\$CINDER_CONF]] +[DEFAULT] +notification_driver = messagingv2 diff --git a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc new file mode 100644 index 00000000..fe48b022 --- /dev/null +++ b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc @@ -0,0 +1,3 @@ +[[post-config|\$NOVA_CONF]] +[DEFAULT] +instance_usage_audit = True diff --git a/integration/scripts/local.conf.d/ceilometer_services.conf.rc b/integration/scripts/local.conf.d/ceilometer_services.conf.rc new file mode 100644 index 00000000..ce33948e --- /dev/null +++ b/integration/scripts/local.conf.d/ceilometer_services.conf.rc @@ -0,0 +1,3 @@ +[[post-config|\$CEILOMETER_CONF]] +[notification] +store_events = True diff --git a/integration/scripts/local.conf.d/sample.rc b/integration/scripts/local.conf.d/sample.rc new file mode 100644 index 00000000..78753bc7 --- /dev/null +++ b/integration/scripts/local.conf.d/sample.rc @@ -0,0 +1,42 @@ +# +# Files in this directory are automatically added to the devstack +# local.conf file, between a specific set of tags. +# +# Filenames must end with '.rc' to be recognized; sample.rc is +# ignored. +# +# A '\' is required in front of any devstack variables since all +# .rc files are parsed first (using eval). +# +# Meta section headings must be included in each file, such as: +# [[local|localrc]] +# as the order of inserting the files is not guaranteed. +# +# All files are inherently included by default - to exclude a file, +# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in redstack.rc +# For Example: USING_VAGRANT=false (for the using_vagrant.rc file). +# +# Symbolic links are followed, so additional files can be loaded +# by placing them in an external directory and linking it in +# local.conf.d (this should allow complete flexibility in setting +# up testing options). +# For Example: +# cd /path/to/trove-integration/scripts/local.conf.d +# ln -s $HOME/local.conf.d local.conf.d +# cp /path/to/my_conf.rc $HOME/local.conf.d + + +[[local|localrc]] +# Put regular devstack variables under this meta section heading. +# This section is written out to a file and sourced by devstack, +# so it can contain logic as well. + +# The following section types should only contain ini file style +# section headings and name=value pairs +[[post-config|\$TROVE_CONF]] + +[[post-config|\$TROVE_TASKMANAGER_CONF]] + +[[post-config|\$TROVE_CONDUCTOR_CONF]] + +[[post-config|\$TROVE_API_PASTE_INI]] diff --git a/integration/scripts/local.conf.d/trove_services.conf.rc b/integration/scripts/local.conf.d/trove_services.conf.rc new file mode 100644 index 00000000..6eedc1c3 --- /dev/null +++ b/integration/scripts/local.conf.d/trove_services.conf.rc @@ -0,0 +1,24 @@ +[[post-config|\$TROVE_CONF]] +[profiler] +enabled = $ENABLE_PROFILER +trace_sqlalchemy = $PROFILER_TRACE_SQL + +[[post-config|\$TROVE_TASKMANAGER_CONF]] +[profiler] +enabled = $ENABLE_PROFILER +trace_sqlalchemy = $PROFILER_TRACE_SQL + +[[post-config|\$TROVE_CONDUCTOR_CONF]] +[profiler] +enabled = $ENABLE_PROFILER +trace_sqlalchemy = $PROFILER_TRACE_SQL + +[[post-config|\$TROVE_GUESTAGENT_CONF]] +[profiler] +enabled = $ENABLE_PROFILER +trace_sqlalchemy = $PROFILER_TRACE_SQL + +[[post-config|\$TROVE_API_PASTE_INI]] +[filter:osprofiler] +enabled = $ENABLE_PROFILER +hmac_keys = $PROFILER_HMAC_KEYS diff --git a/integration/scripts/local.conf.d/use_kvm.rc b/integration/scripts/local.conf.d/use_kvm.rc new file mode 100644 index 00000000..06bc2ebc --- /dev/null +++ b/integration/scripts/local.conf.d/use_kvm.rc @@ -0,0 +1,4 @@ +[[local|localrc]] + +# force kvm as the libvirt type. +LIBVIRT_TYPE=kvm diff --git a/integration/scripts/local.conf.d/use_uuid_token.rc b/integration/scripts/local.conf.d/use_uuid_token.rc new file mode 100644 index 00000000..587a4064 --- /dev/null +++ b/integration/scripts/local.conf.d/use_uuid_token.rc @@ -0,0 +1,3 @@ +[[local|localrc]] + +KEYSTONE_TOKEN_FORMAT=UUID diff --git a/integration/scripts/local.conf.d/using_vagrant.rc b/integration/scripts/local.conf.d/using_vagrant.rc new file mode 100644 index 00000000..7333cd46 --- /dev/null +++ b/integration/scripts/local.conf.d/using_vagrant.rc @@ -0,0 +1,9 @@ +[[local|localrc]] + +# This is similar to code found at +# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile +# and seems to make instances ping'able in VirtualBox. +FLAT_INTERFACE=eth1 +PUBLIC_INTERFACE=eth1 +FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28 +HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4` diff --git a/integration/scripts/local.conf.rc b/integration/scripts/local.conf.rc new file mode 100644 index 00000000..6b9eb0ad --- /dev/null +++ b/integration/scripts/local.conf.rc @@ -0,0 +1,37 @@ +$TROVE_PRESENT_TAG +# Set some arguments for devstack. +# +# Note: This file contains autogenerated parts. +# All lines are removed from between the tag/end of tag +# markers (lines with '$MARKER_TOKEN' at beginning and end) and +# are replaced by trove-integration. +# Edits to these sections will not persist. +# +# See the '$USER_OPTS_TAG' section +# for ways to insert user args into this file. +# + +# +# This section is for things that belong in localrc +# It comes from $DEFAULT_LOCALRC +# +[[local|localrc]] + +$LOCALRC_OPTS_TAG +$LOCALRC_OPTS_TAG_END + +# +# User options here were inserted from the file USER_LOCAL_CONF +# (defaults to $USERHOME/.$LOCAL_CONF) +# + +$USER_OPTS_TAG +$USER_OPTS_TAG_END + +# +# Additional options here were inserted by trove-integration +# automatically from files in $LOCAL_CONF_D +# + +$ADD_OPTS_TAG +$ADD_OPTS_TAG_END diff --git a/integration/scripts/localrc.rc b/integration/scripts/localrc.rc new file mode 100644 index 00000000..94c598ee --- /dev/null +++ b/integration/scripts/localrc.rc @@ -0,0 +1,100 @@ +# These passwords originally come from redstack.rc. +MYSQL_PASSWORD=$MYSQL_PASSWORD +RABBIT_PASSWORD=$RABBIT_PASSWORD +SERVICE_TOKEN=$SERVICE_TOKEN +ADMIN_PASSWORD=$ADMIN_PASSWORD +SERVICE_PASSWORD=$SERVICE_PASSWORD + +PUBLIC_INTERFACE=eth0 +TROVE_LOGDIR=$TROVE_LOGDIR +TROVE_AUTH_CACHE_DIR=$TROVE_AUTH_CACHE_DIR + +# Enable the Trove plugin for devstack +if [[ $USE_DEVSTACK_TROVE_PLUGIN = true ]]; then + enable_plugin trove $TROVE_REPO $TROVE_BRANCH +fi + +# Enable Trove, Swift, and Heat +ENABLED_SERVICES+=,trove,tr-api,tr-tmgr,tr-cond +ENABLED_SERVICES+=,s-proxy,s-object,s-container,s-account +ENABLED_SERVICES+=,heat,h-api,h-api-cfn,h-api-cw,h-eng +ENABLED_SERVICES+=,-n-novnc,-n-xvnc +if [[ $ENABLE_NEUTRON = true ]]; then + ENABLED_SERVICES+=,neutron,q-svc,q-agt,q-dhcp,q-l3,q-meta + disable_service n-net +else + enable_service n-net + disable_service neutron q-svc q-agt q-dhcp q-l3 q-meta +fi +# OSprofiler depends on Ceilometer +if [[ $ENABLE_CEILOMETER = true ]] || [[ $ENABLE_PROFILER = true ]]; then + CEILOMETER_BACKEND=mysql + CEILOMETER_NOTIFICATION_TOPICS=notifications,profiler + PROJ_BRANCH=$(get_project_branch CEILOMETER_BRANCH $PROJECT_BRANCH) + enable_plugin ceilometer ${GIT_OPENSTACK}/ceilometer.git $PROJ_BRANCH +fi + +# Enable Mistral, if configured +if [[ $ENABLE_MISTRAL = true ]]; then + PROJ_BRANCH=$(get_project_branch MISTRAL_BRANCH $PROJECT_BRANCH) + enable_plugin mistral ${GIT_OPENSTACK}/mistral.git $PROJ_BRANCH +fi + +# Use Git repositories for client components +LIBS_FROM_GIT=python-troveclient +if [[ $LIBS_FROM_GIT_ALL_CLIENTS = true ]]; then + LIBS_FROM_GIT+=,python-ceilometerclient + LIBS_FROM_GIT+=,python-cinderclient + LIBS_FROM_GIT+=,python-glanceclient + LIBS_FROM_GIT+=,python-heatclient + LIBS_FROM_GIT+=,python-keystoneclient + LIBS_FROM_GIT+=,python-mistralclient + LIBS_FROM_GIT+=,python-neutronclient + LIBS_FROM_GIT+=,python-novaclient + LIBS_FROM_GIT+=,python-openstackclient + LIBS_FROM_GIT+=,python-swiftclient +else + if [[ $ENABLE_NEUTRON = true ]]; then + LIBS_FROM_GIT+=,python-neutronclient + fi + if [[ $ENABLE_MISTRAL = true ]]; then + LIBS_FROM_GIT+=,python-mistralclient + fi + if [[ $ENABLE_CEILOMETER = true ]]; then + LIBS_FROM_GIT+=,python-ceilometerclient + fi +fi +# Add Git repositories for libraries +if [[ $LIBS_FROM_GIT_ALL_OSLO = true ]]; then + LIBS_FROM_GIT+=,cliff,futurist + LIBS_FROM_GIT+=,debtcollector,automaton + LIBS_FROM_GIT+=,oslo.cache,oslo.concurrency + LIBS_FROM_GIT+=,oslo.config,oslo.context + LIBS_FROM_GIT+=,oslo.db,oslo.i18n + LIBS_FROM_GIT+=,oslo.log,oslo.messaging + LIBS_FROM_GIT+=,oslo.middleware,oslo.policy + LIBS_FROM_GIT+=,oslo.reports,oslo.rootwrap + LIBS_FROM_GIT+=,oslo.serialization,oslo.service + LIBS_FROM_GIT+=,oslo.utils,oslo.versionedobjects + LIBS_FROM_GIT+=,oslo.vmware + LIBS_FROM_GIT+=,pycadf,stevedore + LIBS_FROM_GIT+=,taskflow,tooz + LIBS_FROM_GIT+=,pbr +fi + +NOVNC_FROM_PACKAGE=false +SWIFT_HASH=$SWIFT_HASH +# Set Cinder Volume from Redstack so that later Redstack can help manage +# reconnecting Volume Group to Backing File +DEST=$DEST +DATA_DIR=$DATA_DIR +SERVICE_DIR=$SERVICE_DIR +VOLUME_GROUP=${VOLUME_GROUP} +VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE} +VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE} +# The lock_path is by default /opt/stack/nova; if this path is a shared +# folder in VirtualBox things seem to break. We fix it by setting EXTRA_OPS +# to force lock_path to /tmp. +EXTRA_OPTS=(lock_path=$USERHOME/nova_locks rescan_timeout=180 resizefs_timeout=240 force_dhcp_release=False) +UNDO_REQUIREMENTS=False + diff --git a/integration/scripts/projects-list b/integration/scripts/projects-list new file mode 100644 index 00000000..eacd4d8e --- /dev/null +++ b/integration/scripts/projects-list @@ -0,0 +1,12 @@ +keystone +nova +glance +horizon +swift +neutron +heat +python-openstackclient +python-novaclient +python-troveclient +python-neutronclient +python-heatclient diff --git a/integration/scripts/redstack b/integration/scripts/redstack new file mode 100755 index 00000000..105aceb9 --- /dev/null +++ b/integration/scripts/redstack @@ -0,0 +1,1433 @@ +#!/usr/bin/env bash +############################################################################### +# RedStack, the Trove Dev Machine Controller # +############################################################################### +# # +# This script provides all the functionality to run all the steps from # +# setting up the environment, resetting the nova database to running the # +# test. # +# # +############################################################################### + +REDSTACK_SCRIPTS=${REDSTACK_SCRIPTS:-`pwd`} +REDSTACK_TESTS=$REDSTACK_SCRIPTS/../tests/ + +DEFAULT_LOCAL_CONF=local.conf.rc +DEFAULT_LOCALRC=localrc.rc +LOCAL_CONF=local.conf +LOCALRC=localrc +LOCALRC_AUTO=.localrc.auto +USER_LOCAL_CONF_NAME=.devstack.$LOCAL_CONF + +USERHOME=$HOME +# Load options not checked into VCS. +if [ -f $USERHOME/.redstack.options.rc ]; then + . $USERHOME/.redstack.options.rc +fi +if [ -f $REDSTACK_SCRIPTS/options.rc ]; then + . $REDSTACK_SCRIPTS/options.rc +fi + +# NOTE(mriedem): The gate-trove-functional-dsvm-* job config in project-config +# sets this value for Jenkins runs. +BRANCH_OVERRIDE=${BRANCH_OVERRIDE:-default} +if [[ $BRANCH_OVERRIDE == "default" && $OVERRIDE_ZUUL_BRANCH != "master" ]]; then + BRANCH_OVERRIDE=$OVERRIDE_ZUUL_BRANCH +fi + +# Bail on errors. +set -e + +# Get default host ip from interface +function get_default_host_ip() { + host_iface=$(ip route | sed -n '/^default/{ s/.*dev \(\w\+\)\s\+.*/\1/; p; }' | head -1) + echo `LC_ALL=C ip -f inet addr show ${host_iface} | awk '/inet/ {split($2,parts,"/"); print parts[1]}' | head -1` +} + +# Load functions devstack style +. $REDSTACK_SCRIPTS/functions +. $REDSTACK_SCRIPTS/functions_qemu + +# Load global configuration variables. +. $REDSTACK_SCRIPTS/redstack.rc +. $REDSTACK_SCRIPTS/reviews.rc + +# allow overrides from devstack if already set +[[ -f $PATH_DEVSTACK_SRC/functions-common ]] && source $PATH_DEVSTACK_SRC/functions-common +[[ -f $PATH_DEVSTACK_SRC/functions ]] && source $PATH_DEVSTACK_SRC/functions + +# Source the old-style localrc, or new-style .local.auto - only one should exist. +# Note: The devstack localrc's have references to 'enable_plugin' which causes +# errors when sourcing them in the stable/juno and stable/kilo branches. +# These errors are safe to ignore when sourcing these files. +set +e +[[ -f $PATH_DEVSTACK_SRC/$LOCALRC ]] && source $PATH_DEVSTACK_SRC/$LOCALRC +[[ -f $PATH_DEVSTACK_SRC/$LOCALRC_AUTO ]] && source $PATH_DEVSTACK_SRC/$LOCALRC_AUTO +set -e + +# Set up variables for the CONF files - this has to happen after loading redstack.rc, since +# TROVE_CONF_DIR is defined there - these will be used by devstack too +export TROVE_CONF=$TROVE_CONF_DIR/trove.conf +export TROVE_TASKMANAGER_CONF=$TROVE_CONF_DIR/trove-taskmanager.conf +export TROVE_CONDUCTOR_CONF=$TROVE_CONF_DIR/trove-conductor.conf +export TROVE_GUESTAGENT_CONF=$TROVE_CONF_DIR/trove-guestagent.conf +export TROVE_API_PASTE_INI=$TROVE_CONF_DIR/api-paste.ini +export TEST_CONF=$TROVE_CONF_DIR/test.conf + +# Public facing bits +SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http} +NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0} +NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24} +NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1} +BRIDGE_IP=${BRIDGE_IP:-172.24.4.1} +KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST} +KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL} +KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357} +GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292} +GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-http} +TROVE_AUTH_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0 + +# The following depends on whether neutron is used or nova-network +# neutron uses a bridge, nova-network does not +[[ $ENABLE_NEUTRON = true ]] && CONTROLLER_IP=$BRIDGE_IP || CONTROLLER_IP=$NETWORK_GATEWAY + +# PATH_TROVE more than likely has file separators, which sed does not like +# This will escape them +ESCAPED_PATH_TROVE=$(echo $PATH_TROVE | sed 's/\//\\\//g') +ESCAPED_REDSTACK_SCRIPTS=$(echo $REDSTACK_SCRIPTS | sed 's/\//\\\//g') +TROVE_AUTH_CACHE_DIR=${TROVE_AUTH_CACHE_DIR:-/var/cache/trove} +TROVE_LOGDIR=${TROVE_LOGDIR:-$DEST/logs} +# DATASTORE_PKG_LOCATION defines the location from where the datastore packages +# can be accessed by the DIB elements. This is applicable only for datastores +# that do not have a public repository from where their packages can be accessed. +# This can either be a url to a private repository or a location on the local +# filesystem that contains the datastore packages. +DATASTORE_PKG_LOCATION=${DATASTORE_PKG_LOCATION:-} + +# Support entry points installation of console scripts +if [[ -d $PATH_TROVE/bin ]]; then + TROVE_BIN_DIR=$PATH_TROVE/bin +else + TROVE_BIN_DIR=$(get_python_exec_prefix) +fi + +# Allow sourcing config values from env.rc for overrides +if [ -f /tmp/integration/env.rc ]; then + source /tmp/integration/env.rc +fi + +# set up respective package managers +if is_fedora; then + PKG_INSTALL_OPTS="" + PKG_MGR=dnf + PKG_GET_ARGS="-y" +else + PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive" + PKG_MGR=apt-get + PKG_GET_ARGS="-y --allow-unauthenticated --force-yes" +fi +PKG_INSTALL_ARG="install" +PKG_UPDATE_ARG="update" + +############################################################################### +# Utility functions +############################################################################### + +# Colors that can be used in 'exclaim' +COLOR_RED='\033[0;31m' +COLOR_GREEN='\033[0;32m' +COLOR_BLUE='\033[0;34m' +COLOR_NONE='\033[0m' + +function exclaim () { + echo "*******************************************************************************" + echo -e "$@" + echo "*******************************************************************************" +} + +# Set the location of the Trove setup commands file for devstack - either in +# the devstack repo, or the Trove one. Also sets the flag which activates the +# devstack trove plugin, if required. We'll use the devstack version if it +# exists, otherwise we assume the plugin method. +function set_trove_plugin_vars () { + FAIL_IF_MISSING=${1:-true} + + TROVE_SETUP_CMD_FILE="$PATH_DEVSTACK_SRC/lib/trove" + if [ -f "$TROVE_SETUP_CMD_FILE" ]; then + USE_DEVSTACK_TROVE_PLUGIN=false + else + TROVE_SETUP_CMD_FILE="$DEST/trove/devstack/plugin.sh" + USE_DEVSTACK_TROVE_PLUGIN=true + fi + # Only complain if we have a devstack directory and are told to + if [ "$FAIL_IF_MISSING" = "true" ] && [ -d "$PATH_DEVSTACK_SRC" ] && [ ! -f "$TROVE_SETUP_CMD_FILE" ]; then + exclaim "${COLOR_RED}Trove setup file '${TROVE_SETUP_CMD_FILE}' not found!${COLOR_NONE}" + exit 1 + fi +} + +function pkg_install () { + echo Installing $@... + sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_INSTALL_ARG $@ +} + +function pkg_update () { + echo Updating $@... + sudo -E $PKG_INSTALL_OPTS $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS $PKG_UPDATE_ARG $@ +} + +function set_home_dir() { + exclaim "set_home_dir has been nooped." + exit 1 +} + +function set_http_proxy() { + if [ ! "${http_proxy}" = '' ]; then + HTTP_PROXY="http_proxy=$http_proxy https_proxy=$https_proxy" + fi +} + +function get_ip_for_device() { + /sbin/ifconfig $1 | awk '/inet addr/{gsub(/addr:/,"");print $2}' +} + +function ip_chunk() { + # Given 1-4 returns a bit of where the ip range starts. + # Full IP= `ip_chunk 1`.`ip_chunk 2`.`ip_chunk 3`.`ip_chunk 4` + get_ip_for_device $1 | cut -d. -f$2 +} + +# Add a flavor and a corresponding flavor.resize +# (flavor.resize adds 16 to the memory and one more vcpu) +function add_flavor() { + local FLAVOR_NAME=$1 + local FLAVOR_ID=$2 + local FLAVOR_MEMORY_MB=$3 + local FLAVOR_ROOT_GB=$4 + local FLAVOR_VCPUS=$5 + + credentials="--os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT" + if [[ -z "$FLAVOR_LIST_FOR_ADD" ]]; then + FLAVOR_LIST_FOR_ADD=$(nova $credentials flavor-list | cut -d'|' -f3 | sed -e's/ /,/g') + fi + + base_id=${FLAVOR_ID} + base_name_prefix=test + ephemeral_name_prefix=${base_name_prefix}.eph + for name_prefix in $base_name_prefix $ephemeral_name_prefix; do + reg_name=${name_prefix}.${FLAVOR_NAME}-${FLAVOR_ROOT_GB} + resize_name=${reg_name}.resize + ephemeral=0 + if [[ $name_prefix == $ephemeral_name_prefix ]]; then + ephemeral=1 + fi + for name in ${reg_name} ${resize_name}; do + id=$base_id + memory=${FLAVOR_MEMORY_MB} + vcpus=${FLAVOR_VCPUS} + if [[ $ephemeral != 0 ]]; then + if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then + id=1${id} + else + id=${id}e + fi + fi + if [[ $name == ${resize_name} ]]; then + if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then + id=2${id} + else + id=${id}r + fi + memory=$((${FLAVOR_MEMORY_MB} + 16)) + vcpus=$((${FLAVOR_VCPUS} + 1)) + fi + if [[ $FLAVOR_LIST_FOR_ADD != *",$name,"* ]]; then + nova $credentials flavor-create $name $id $memory $FLAVOR_ROOT_GB $vcpus --ephemeral $ephemeral + fi + done + done +} + +function get_attribute_id() { + openstack --os-auth-url=$TROVE_AUTH_ENDPOINT --os-username=admin --os-password=$ADMIN_PASSWORD --os-project-name=admin $1 list | grep " $2" | get_field $3 +} + + +############################################################################### +# Install all the required dependencies +############################################################################### + +function install_prep_packages() { + # Called before devstack + exclaim 'Updating dependencies (part 1a)...' + pkg_update + exclaim 'Installing dependencies (part 1b)...' + pkg_install python-pip + if is_fedora; then + pkg_install git gettext + else + pkg_install git-core kvm-ipxe gettext + fi + sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils +} + +function install_devstack_code() { + exclaim "Installing devstack..." + # Installs devstack (if needed). + if [ ! -d $PATH_DEVSTACK_SRC ]; then + echo "DevStack not in a shared folder, cloning from git." + mkdir -p $PATH_DEVSTACK_SRC + git clone $DEVSTACK_REPO $PATH_DEVSTACK_SRC + fi + + source $PATH_DEVSTACK_SRC/functions-common + source $PATH_DEVSTACK_SRC/functions + + # Switch to a branch if specified. The order the variables are checked is: + # DEVSTACK_BRANCH then PROJECT_BRANCH + BRANCH_SPECIFIED=$(test -z "${DEVSTACK_BRANCH}${PROJECT_BRANCH}" || echo 'True') + if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then + PROJ_BRANCH=$(get_project_branch DEVSTACK_BRANCH $PROJECT_BRANCH) + ENV_VARS="DEVSTACK_BRANCH' or 'PROJECT_BRANCH" + git_checkout "devstack" "$PATH_DEVSTACK_SRC" "$PROJ_BRANCH" "$ENV_VARS" + fi + + exclaim "Installing devstack projects..." + # Ensures present user can get to the devstack dirs + sudo mkdir -p $PATH_DEVSTACK_OUTPUT + if [ ! -w $PATH_DEVSTACK_OUTPUT ]; then + sudo chown `whoami` $PATH_DEVSTACK_OUTPUT + fi + # Clones all of the code to where devstack expects it to be + pushd $PATH_DEVSTACK_OUTPUT + cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/projects-list \ + $REDSTACK_SCRIPTS/image-projects-list + popd +} + +function install_reviews_on_top_of_devstack() { + exclaim "Putting gerrit review code on top of the existing devstack code" + run_review_for nova $PATH_NOVA $REVIEW_NOVA + run_review_for python-novaclient $PATH_PYTHON_NOVACLIENT $REVIEW_PYTHON_NOVACLIENT + run_review_for keystone $PATH_KEYSTONE $REVIEW_KEYSTONE + run_review_for python-keystoneclient $PATH_KEYSTONECLIENT $REVIEW_PYTHON_KEYSTONECLIENT + run_review_for python-openstackclient $PATH_OPENSTACKCLIENT $REVIEW_PYTHON_OPENSTACKCLIENT + run_review_for glance $PATH_GLANCE $REVIEW_GLANCE + run_review_for swift $PATH_SWIFT $REVIEW_SWIFT + run_review_for python-swiftclient $PATH_PYTHON_SWIFTCLIENT $REVIEW_PYTHON_SWIFTCLIENT + run_review_for trove $PATH_TROVE $REVIEW_TROVE + run_review_for python-troveclient $PATH_PYTHON_TROVECLIENT $REVIEW_PYTHON_TROVECLIENT +} + +function run_review_for() { + # Splits based on colon in the REVIEW_ARG and pulls from + GIT_NAME=$1 + PATH_ARG=$2 + REVIEW_ARG=$3 + for review in `echo $REVIEW_ARG| tr ":" "\n"` + do + # This should be the ref spec for what we pull + pushd $PATH_ARG + git_timed pull https://review.openstack.org/p/openstack/$GIT_NAME refs/changes/$review + popd + done +} + +function fixup_broken_devstack() { + # Nothing to do here, devstack is working + : +} + +# Delete all the lines from FILE_NAME between START_TAG and END_TAG +# Tags must appear at the beginning of a line +function clear_file_lines() { + local FILE_NAME=$1 + local START_TAG=$2 + local END_TAG=$3 + + sed -i "/^$START_TAG$/,/^$END_TAG$/{/^$START_TAG/!{/^$END_TAG/!d;}}" "$FILE_NAME" +} + +# Checks to see if a variable with the same name as FILE_NAME exists. +# Returns 'true' if no varable exists or if the value of the variable +# is set to 'true' - returns the VAR_NAME to set otherwise. +# FILE_NAME is first converted to uppercase, the extension is removed +# and all remaining '.' and spaces are replaced with '_' +function check_filename_var() { + local FILE_NAME=$1 + + DEREF_VALUE=false + if [ -f "$FILE_NAME" ]; then + VAR_NAME=$(basename "$FILE_NAME" ".rc" | tr '[:lower:][:blank:][:punct:]' '[:upper:]__') + DEREF_VALUE=$(get_bool "$VAR_NAME" "true") + if [ "$DEREF_VALUE" != "true" ]; then + DEREF_VALUE=$VAR_NAME + fi + fi + echo "$DEREF_VALUE" +} + +# Add the contents of one file to another, after the given tag +# Run through 'eval' if PARSE_FILE is true (defaults to true) +# Start with a blank line if BLANK_LINE_TO_START is true (defaults to false) +function add_file_contents() { + local FILE_NAME=$1 + local FILE_TO_ADD=$2 + local TAG=$3 + local PARSE_FILE=${4:-true} + local BLANK_LINE_TO_START=${5:-false} + + TEMP_FILE=".redstack.$$" + rm -f "$TEMP_FILE" + if [ "$BLANK_LINE_TO_START" = "true" ]; then + echo "" > "$TEMP_FILE" + fi + if [ -f "$FILE_TO_ADD" ]; then + echo "Adding $FILE_TO_ADD to $FILE_NAME" + echo "# Contents from $FILE_TO_ADD" >> "$TEMP_FILE" + if [ "$PARSE_FILE" = "true" ]; then + eval echo "\"$(cat "$FILE_TO_ADD")\"" >> "$TEMP_FILE" + else + cat "$FILE_TO_ADD" >> "$TEMP_FILE" + fi + echo "# End Of Contents from $FILE_TO_ADD" >> "$TEMP_FILE" + fi + echo "" >> "$TEMP_FILE" + sed -i "/^$TAG/r $TEMP_FILE" "$FILE_NAME" + rm -f "$TEMP_FILE" +} + +function run_devstack() { + exclaim "Running devstack..." + + # (Re)Creating this lock directory seems sure-fire. + rm -rf "$USERHOME/nova_locks" + mkdir -p "$USERHOME/nova_locks" + + TROVE_PRESENT_TAG="# Trove-integration" + LOCAL_CONF_D=local.conf.d + CONF_MATCH="*.rc" + MARKER_TOKEN="#####" + USER_LOCAL_CONF=$(readlink -f "${USER_LOCAL_CONF:-$USERHOME/$USER_LOCAL_CONF_NAME}") + LOCALRC_OPTS_TAG="$MARKER_TOKEN Redstack Localrc Options $MARKER_TOKEN" + LOCALRC_OPTS_TAG_END="$MARKER_TOKEN End Of Redstack Localrc Options $MARKER_TOKEN" + USER_OPTS_TAG="$MARKER_TOKEN User Specified Options $MARKER_TOKEN" + USER_OPTS_TAG_END="$MARKER_TOKEN End Of User Specified Options $MARKER_TOKEN" + ADD_OPTS_TAG="$MARKER_TOKEN Additional Options $MARKER_TOKEN" + ADD_OPTS_TAG_END="$MARKER_TOKEN End Of Additional Options $MARKER_TOKEN" + set_trove_plugin_vars false + + pushd "$PATH_DEVSTACK_SRC" + DEVSTACK_LOCAL_CONF=$LOCAL_CONF + # remain backwards compatible with existing localrc files + if [ -f "$LOCALRC" ]; then + DEVSTACK_LOCAL_CONF=$LOCALRC + echo "Old-style devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF found." + echo "Consider removing to generate the preferred-sytle config file $LOCAL_CONF." + fi + if [ -f "$DEVSTACK_LOCAL_CONF" ]; then + # Check if we have already configured the devstack config file + already_in_conf=$(grep "$TROVE_PRESENT_TAG" "$DEVSTACK_LOCAL_CONF" | wc -l) + if [ "$already_in_conf" == 0 ]; then + # We can no longer append to an existing old-style localrc file + if [ "$DEVSTACK_LOCAL_CONF" == "$LOCALRC" ]; then + echo "The devstack config file $PATH_DEVSTACK_SRC/$DEVSTACK_LOCAL_CONF is too old to append to." + echo "Please remove and try again." + exit 1 + fi + # Otherwise append the redstack version to the existing file + eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC")\"" >> "$DEVSTACK_LOCAL_CONF" + fi + else + # If a devstack config file doesn't exist, create it + eval echo "\"$(cat "$REDSTACK_SCRIPTS/$DEFAULT_LOCAL_CONF")\"" > "$DEVSTACK_LOCAL_CONF" + fi + + # We can only replace sections from the LOCAL_CONF style files + if [ "$DEVSTACK_LOCAL_CONF" == "$LOCAL_CONF" ]; then + # Clear out all the options + clear_file_lines "$DEVSTACK_LOCAL_CONF" "$LOCALRC_OPTS_TAG" "$LOCALRC_OPTS_TAG_END" + clear_file_lines "$DEVSTACK_LOCAL_CONF" "$USER_OPTS_TAG" "$USER_OPTS_TAG_END" + clear_file_lines "$DEVSTACK_LOCAL_CONF" "$ADD_OPTS_TAG" "$ADD_OPTS_TAG_END" + + # Add the main localrc file + PARSE_FILE="true" + BLANK_LINE_TO_START="true" + if [ -f "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" ]; then + add_file_contents "$DEVSTACK_LOCAL_CONF" "$REDSTACK_SCRIPTS/$DEFAULT_LOCALRC" "$LOCALRC_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" + fi + + # Add any user options + PARSE_FILE="false" + BLANK_LINE_TO_START="true" + if [ -f "$USER_LOCAL_CONF" ]; then + add_file_contents "$DEVSTACK_LOCAL_CONF" "$USER_LOCAL_CONF" "$USER_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" + fi + + # Add all the files in the LOCAL_CONF_D directory that match CONF_MATCH (except for sample files) + # and that aren't excluded. Files are excluded by having a variable + # 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in redstack.rc + # For Example: USING_VAGRANT=false (for the using_vagrant.rc file). + PARSE_FILE="true" + BLANK_LINE_TO_START="false" + while IFS= read -r -d '' CONF_FILE + do + FILE_NAME_VAR=$(check_filename_var "$CONF_FILE") + if [ "$FILE_NAME_VAR" = "true" ]; then + add_file_contents "$DEVSTACK_LOCAL_CONF" "$CONF_FILE" "$ADD_OPTS_TAG" "$PARSE_FILE" "$BLANK_LINE_TO_START" + else + echo "Skipping $CONF_FILE" + echo "Use $FILE_NAME_VAR=true to include" + fi + done < <(find "$REDSTACK_SCRIPTS/${LOCAL_CONF_D}" -name "${CONF_MATCH}" -follow -not -name "sample*.rc" -type f -print0) + # this is to add a blank line for readability + add_file_contents "$DEVSTACK_LOCAL_CONF" "" "$ADD_OPTS_TAG" + fi + ./stack.sh + popd +} + + +function cmd_install() { + sudo mkdir -p $TROVE_LOGDIR # Creates TROVE_LOGDIR if it does not exist + if [ ! -w $TROVE_LOGDIR ]; then + sudo chown `whoami` $TROVE_LOGDIR + fi + install_prep_packages + install_devstack_code + install_reviews_on_top_of_devstack + fixup_broken_devstack + run_devstack + exclaim "${COLOR_GREEN}FINISHED INSTALL${COLOR_NONE}" +} + + +############################################################################### +# Build the image +# see functions_qemu +############################################################################### + +# Grab a numbered field from python prettytable output +# Fields are numbered starting with 1 +# Reverse syntax is supported: -1 is the last field, -2 is second to last, etc. +# get_field field-number +function get_field() { + while read data; do + if [ "$1" -lt 0 ]; then + field="(\$(NF$1))" + else + field="\$$(($1 + 1))" + fi + echo "$data" | awk -F'[ \t]*\\|[ \t]*' "{print $field}" + done +} + +function get_glance_id () { + echo `$@ | grep ' id ' | get_field 2` +} + +function set_bin_path() { + if is_fedora; then + sed -i "s|%bin_path%|/usr/bin|g" $TEST_CONF + else + sed -i "s|%bin_path%|/usr/local/bin|g" $TEST_CONF + fi +} + +function set_mysql_pkg() { + if is_fedora; then + MYSQL_PKG="mysql-community-server" + else + MYSQL_PKG="mysql-server-5.6" + fi +} + + +function cmd_set_datastore() { + IMAGEID=$1 + DATASTORE_TYPE=$2 + # rd_manage datastore_update <datastore_name> <default_version> + rd_manage datastore_update "$DATASTORE_TYPE" "" + PACKAGES=${PACKAGES:-""} + if [ "$DATASTORE_TYPE" == "mysql" ]; then + set_mysql_pkg + PACKAGES=${PACKAGES:-$MYSQL_PKG} + VERSION="5.6" + elif [ "$DATASTORE_TYPE" == "percona" ]; then + PACKAGES=${PACKAGES:-"percona-server-server-5.6"} + VERSION="5.6" + elif [ "$DATASTORE_TYPE" == "pxc" ]; then + PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"} + VERSION="5.6" + elif [ "$DATASTORE_TYPE" == "mariadb" ]; then + PACKAGES=${PACKAGES:-"mariadb-server"} + VERSION="10.1" + elif [ "$DATASTORE_TYPE" == "mongodb" ]; then + PACKAGES=${PACKAGES:-"mongodb-org"} + VERSION="3.2" + elif [ "$DATASTORE_TYPE" == "redis" ]; then + PACKAGES=${PACKAGES:-"redis-server"} + VERSION="3.0" + elif [ "$DATASTORE_TYPE" == "cassandra" ]; then + PACKAGES=${PACKAGES:-"cassandra"} + VERSION="2.1.0" + elif [ "$DATASTORE_TYPE" == "couchbase" ]; then + PACKAGES=${PACKAGES:-"couchbase-server"} + VERSION="2.2.0" + elif [ "$DATASTORE_TYPE" == "postgresql" ]; then + PACKAGES=${PACKAGES:-"postgresql-9.4"} + VERSION="9.4" + elif [ "$DATASTORE_TYPE" == "couchdb" ]; then + PACKAGES=${PACKAGES:-"couchdb"} + VERSION="1.6.1" + elif [ "$DATASTORE_TYPE" == "vertica" ]; then + PACKAGES=${PACKAGES:-"vertica"} + VERSION="7.1" + elif [ "$DATASTORE_TYPE" == "db2" ]; then + PACKAGES=${PACKAGES:-""} + VERSION="10.5" + else + echo "Unrecognized datastore type. ($DATASTORE_TYPE)" + exit 1 + fi + + sed -i "s/%datastore_type%/$DATASTORE_TYPE/g" $TEST_CONF + sed -i "s/%datastore_version%/$VERSION/g" $TEST_CONF + + #rd_manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active> + rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1 + rd_manage datastore_version_update "$DATASTORE_TYPE" "inactive_version" "manager1" $IMAGEID "" 0 + rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION" + rd_manage datastore_update Test_Datastore_1 "" + + if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then + # add the configuration parameters to the database for the kick-start datastore + rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json + fi + + cmd_stop + iniset $TROVE_CONF DEFAULT default_datastore "$DATASTORE_TYPE" + sleep 1.5 + cmd_start +} + +############################################################################### +# Run Unit Tests +############################################################################### + +function cmd_unit_tests() { + exclaim "Running Trove Unit Tests..." + $PATH_TROVE/run_tests.sh -N +} + +############################################################################### +# Start various OpenStack daemons interactively in a screen session +############################################################################### + +function cmd_start_deps() { + if ! sudo vgs $VOLUME_GROUP; then + exclaim "Reconnecting Volume Group to Backing File" + sudo losetup -f --show ${VOLUME_BACKING_FILE} + fi + + if ! egrep -q ${SWIFT_DATA_DIR}/drives/sdb1 /proc/mounts; then + exclaim "Re-mounting Swift Disk Image" + sudo mount -t xfs -o loop,noatime,nodiratime,nobarrier,logbufs=8 ${SWIFT_DISK_IMAGE} ${SWIFT_DATA_DIR}/drives/sdb1 || true + fi + + if [[ -e $PATH_DEVSTACK_SRC/stack-screenrc ]]; then + screen -dmS stack -c $PATH_DEVSTACK_SRC/stack-screenrc + fi +} + +function cmd_stop_deps() { + if screen -ls | grep -q stack; then + screen -S stack -X quit + rm -f $DEST/status/stack/* + fi +} + + +############################################################################### +# Initialize Trove +############################################################################### + +function rd_manage() { + pushd $PATH_TROVE + $TROVE_BIN_DIR/trove-manage --config-file=$TROVE_CONF "$@" + popd +} + +function install_test_packages() { + sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect +} + +function mod_confs() { + DATASTORE_TYPE=$1 + + sudo install -b --mode 0664 $REDSTACK_SCRIPTS/conf/test_begin.conf $TEST_CONF + TROVE_REPORT_DIR=$REDSTACK_SCRIPTS/../report/ + EXTRA_CONF=$REDSTACK_SCRIPTS/conf/test.extra.conf + if [ -e $EXTRA_CONF ]; then + cat $EXTRA_CONF >> $TEST_CONF + fi + # Append datastore specific configuration file + DATASTORE_CONF=$REDSTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf + if [ ! -f $DATASTORE_CONF ]; then + exclaim "Datastore configuration file ${DATASTORE_CONF} not found" + exit 1 + fi + cat $DATASTORE_CONF | sudo tee -a $TEST_CONF > /dev/null + cat $REDSTACK_SCRIPTS/conf/test_end.conf | sudo tee -a $TEST_CONF > /dev/null + + #Add the paths to the test conf + sed -i "s,%report_directory%,$TROVE_REPORT_DIR,g" $TEST_CONF + sed -i "s,%keystone_path%,$PATH_KEYSTONE,g" $TEST_CONF + sed -i "s,%nova_path%,$PATH_NOVA,g" $TEST_CONF + sed -i "s,%glance_path%,$PATH_GLANCE,g" $TEST_CONF + sed -i "s,%trove_path%,$PATH_TROVE,g" $TEST_CONF + sed -i "s,%service_host%,$SERVICE_HOST,g" $TEST_CONF + sed -i "s,%swifth_path%,$PATH_SWIFT,g" $TEST_CONF + + # Add the region name into test.conf + sed -i "s/%region_name%/${REGION_NAME}/g" $TEST_CONF + + # Add the tenant id's into test.conf + sed -i "s/%admin_tenant_id%/$(get_attribute_id project admin 1)/g" $TEST_CONF + sed -i "s/%alt_demo_tenant_id%/$(get_attribute_id project alt_demo 1)/g" $TEST_CONF + sed -i "s/%demo_tenant_id%/$(get_attribute_id project demo 1)/g" $TEST_CONF + sed -i "s/%admin_password%/$ADMIN_PASSWORD/g" $TEST_CONF + + # Enable neutron tests if needed + sed -i "s/%neutron_enabled%/$ENABLE_NEUTRON/g" $TEST_CONF + + # If neutron is enabled, we create a shared network and write this info to the + # confs so that the integration tests can use it. + if [[ $ENABLE_NEUTRON = true ]]; then + management_network_id=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/ alt-private / {print $2}') + management_subnet=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT subnet-list | awk '/ alt-private-subnet / {print $2}') + echo "Using neutron network $management_network_id and subnet $management_subnet" + sed -i "s,%shared_network%,$management_network_id,g" $TEST_CONF + sed -i "s,%shared_network_subnet%,$management_subnet,g" $TEST_CONF + else + # do not leave invalid keys in the configuration when using Nova for networking + sed -i "/%shared_network%/d" $TEST_CONF + sed -i "/%shared_network_subnet%/d" $TEST_CONF + fi + + if [ "$DATASTORE_TYPE" = "vertica" ]; then + # Vertica needs more time than mysql for its boot/start/stop operations. + setup_cluster_configs cluster_member_count 3 + elif [ "$DATASTORE_TYPE" = "pxc" ]; then + setup_cluster_configs min_cluster_member_count 2 + elif [ "$DATASTORE_TYPE" = "cassandra" ]; then + setup_cluster_configs cluster_member_count 2 + elif [ "$DATASTORE_TYPE" = "mongodb" ]; then + setup_cluster_configs cluster_member_count 2 + # Decrease the number of required config servers per cluster to save resources. + iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1 + fi + + set_bin_path + +} + +function setup_cluster_configs() { + # Setting cluster_member_count to 2 to decrease cluster spawn time. + iniset $TROVE_CONF $DATASTORE_TYPE $1 $2 +} + +# Add useful flavors for testing (with corresponding *.resize flavors) +function add_test_flavors() { + # name id ram root_vol vcpu + # the ram and vcpu for name.resize are automatically calculated + # eph and non-eph flavors are created for each entry + add_flavor 'tiny' 10 512 3 1 + + add_flavor 'small' 15 768 3 1 + add_flavor 'small' 16 768 4 1 + add_flavor 'small' 17 768 5 1 + + add_flavor 'medium' 20 1024 4 1 + add_flavor 'medium' 21 1024 5 1 + + add_flavor 'large' 25 2048 5 1 + add_flavor 'large' 26 2048 10 1 + add_flavor 'large' 27 2048 15 1 + + # This will allow Nova to create an instance, but not enough disk to boot the image + add_flavor 'fault_1' 30 512 1 1 + # This should be enough memory to cause Nova to fail entirely due to too much allocation + add_flavor 'fault_2' 31 131072 5 1 +} + +function cmd_test_init() { + exclaim 'Initializing Configuration for Running Tests...' + + exclaim "Installing python test packages." + install_test_packages + + exclaim "Modifying test.conf and guest.conf with appropriate values." + mod_confs $1 + + exclaim "Creating Test Flavors." + add_test_flavors + + if [[ -n $KEY_DIR ]]; then + exclaim "Installing the SSH key from $KEY_DIR to the test environment." + mkdir -m 700 -p $USERHOME/.ssh + install -b --mode 0400 $KEY_DIR/id_rsa $USERHOME/.ssh + cat $KEY_DIR/authorized_keys >> $USERHOME/.ssh/authorized_keys + chmod 600 $USERHOME/.ssh/authorized_keys + fi +} + +function cmd_build_image() { + IMAGE_DATASTORE_TYPE=${1:-'mysql'} + ESCAPED_PATH_TROVE=${2:-'\/opt\/stack\/trove'} + HOST_SCP_USERNAME=${3:-'ubuntu'} + GUEST_USERNAME=${4:-'ubuntu'} + + exclaim "Ensuring we have all packages needed to build image." + sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update + sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu + sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils + + install_devstack_code + + cmd_clone_projects do_not_force_update $REDSTACK_SCRIPTS/image-projects-list + + exclaim "Use tripleo-diskimagebuilder to actually build the Trove Guest Agent Image." + build_guest_image $IMAGE_DATASTORE_TYPE +} + +function cmd_build_and_upload_image() { + local IMAGE_URL="" + # Use /tmp as file_cache + FILES=/tmp + if [[ -n $IMAGE_DOWNLOAD_URL ]]; then + exclaim "Downloading and using cached image" + IMAGE_URL=$IMAGE_DOWNLOAD_URL + else + exclaim "Trying to build image" + build_guest_image $1 + QCOW_IMAGE=`find $VM_PATH -name '*.qcow2'` + IMAGE_URL="file://$QCOW_IMAGE" + fi + + # The devstack openrc has references to 'enable_plugin' which causes errors + # in the stable/juno and stable/kilo branches. These are safe to ignore. + set +e; source $PATH_DEVSTACK_SRC/openrc admin admin; set -e + TOKEN=$(openstack token issue | grep ' id ' | get_field 2) + GLANCE_IMAGEIDS=$(glance image-list | grep $(basename $IMAGE_URL .qcow2) | get_field 1) + if [[ -n $GLANCE_IMAGEIDS ]]; then + glance image-delete $GLANCE_IMAGEIDS + fi + GLANCE_IMAGEID=`get_glance_id upload_image $IMAGE_URL $TOKEN` + set +e; source $PATH_DEVSTACK_SRC/openrc demo demo; set -e + [[ -z "$GLANCE_IMAGEID" ]] && echo "Glance upload failed!" && exit 1 + echo "IMAGE ID: $GLANCE_IMAGEID" + + exclaim "Updating Datastores" + cmd_set_datastore $GLANCE_IMAGEID $1 +} + + +function cmd_initialize() { + exclaim '(Re)Initializing Trove...' + pushd $PATH_DEVSTACK_SRC + ./unstack.sh + ./stack.sh + popd +} + + +############################################################################### +# Start Trove specific daemons interactively in a screen session +############################################################################### + +function tr_screen_it { + if screen -ls | grep -q stack; then + echo "Starting $@..." + screen -S stack -p $1 -X stuff "$2"$'\015' + fi +} + +function init_fake_mode() { + # Create a test conf which, unlike the conf which runs on a user's machine, + # takes advantage of the running keystone service we have in our VM. + # You could think of this fake mode, which runs in the VM as being + # slightly less fake than the default one which runs outside of it. + CONF_FILE=/tmp/trove.conf.test + cp $PATH_TROVE/etc/trove/trove.conf.test $CONF_FILE + # Switch keystone from the fake class to the real one. + sed -i \ + "s/trove.tests.fakes.keystone/keystone.middleware.auth_token/g" \ + $CONF_FILE + sed -i "s/log_file = rdtest.log//g" $CONF_FILE + sed -i "s/use_stderr = False/use_stderr = True/g" $CONF_FILE + cd $PATH_TROVE + set -e + rm -f trove_test.sqlite + set +e + $TROVE_BIN_DIR/trove-manage --config-file=$CONF_FILE db_sync + sqlite3 trove_test.sqlite \ + "INSERT INTO datastores VALUES ('a00000a0-00a0-0a00-00a0-000a000000aa', \ + 'mysql', 'b00000b0-00b0-0b00-00b0-000b000000bb'); \ + INSERT INTO datastores values ('e00000e0-00e0-0e00-00e0-000e000000ee', \ + 'Test_Datastore_1', ''); \ + INSERT INTO datastore_versions VALUES ('b00000b0-00b0-0b00-00b0-000b000000bb', \ + 'a00000a0-00a0-0a00-00a0-000a000000aa', '5.6', \ + 'c00000c0-00c0-0c00-00c0-000c000000cc', $MYSQL_PKG, 1, 'mysql'); \ + INSERT INTO datastore_versions VALUES ('d00000d0-00d0-0d00-00d0-000d000000dd', \ + 'a00000a0-00a0-0a00-00a0-000a000000aa', 'inactive_version', \ + '', '', 0, 'manager1'); \ + INSERT INTO datastore_configuration_parameters VALUES \ + ('00000000-0000-0000-0000-000000000001', \ + 'key_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \ + 0, 4294967296, 0, 'integer', 0, NULL); \ + INSERT INTO datastore_configuration_parameters VALUES \ + ('00000000-0000-0000-0000-000000000002', \ + 'connect_timeout', 'b00000b0-00b0-0b00-00b0-000b000000bb', \ + 0, 65535, 1, 'integer', 0, NULL); \ + INSERT INTO datastore_configuration_parameters VALUES \ + ('00000000-0000-0000-0000-000000000003', \ + 'join_buffer_size', 'b00000b0-00b0-0b00-00b0-000b000000bb', \ + 0, 4294967296, 0, 'integer', 0, NULL); \ + INSERT INTO datastore_configuration_parameters VALUES \ + ('00000000-0000-0000-0000-000000000004', \ + 'local_infile', 'b00000b0-00b0-0b00-00b0-000b000000bb', \ + 0, 1, 0, 'integer', 0, NULL); \ + INSERT INTO datastore_configuration_parameters VALUES \ + ('00000000-0000-0000-0000-000000000005', \ + 'collation_server', 'b00000b0-00b0-0b00-00b0-000b000000bb', \ + 0, NULL, NULL, 'string', 0, NULL); \ + " +} + +function cmd_start() { + if screen -ls | grep -q stack; then + set_trove_plugin_vars + USE_SCREEN=True + TOP_DIR=$PATH_DEVSTACK_SRC + LOGDIR=$TROVE_LOGDIR + RUNNING=$(screen -S stack -Q windows) + if [[ "$RUNNING" =~ " tr-" ]]; then + exclaim "${COLOR_RED}WARNING: Trove services appear to be running. Please run 'stop' or 'restart'${COLOR_NONE}" + else + source /dev/stdin < <(sed -n '/^function start_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE") + start_trove + fi + else + echo "WARNING: Could not start Trove services because there is no stack screen session running" + fi +} + +function cmd_start_fake() { + init_fake_mode + CONF_FILE=/tmp/trove.conf.test + tr_screen_it tr-fake "cd $PATH_TROVE; $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@" +} + +function cmd_run() { + cd $PATH_TROVE; $TROVE_BIN_DIR/trove-api \ + --config-file=$TROVE_CONF $@ +} + +function cmd_run_fake() { + init_fake_mode + CONF_FILE=/tmp/trove.conf.test + $TROVE_BIN_DIR/trove-fake-mode --config-file=$CONF_FILE $@ +} + +############################################################################### +# Stop any active Trove screen session +############################################################################### + +function cmd_stop() { + if screen -ls | grep -q stack; then + set_trove_plugin_vars + rm -f $DEST/status/stack/tr-* + USE_SCREEN=True + source /dev/stdin < <(sed -n '/^function stop_trove\(\)/,/^}/p' "$TROVE_SETUP_CMD_FILE") + MAX_RETRY=5 + COUNT=1 + while true; do + RUNNING=$(screen -S stack -Q windows) + if [[ "$RUNNING" =~ " tr-" ]]; then + stop_trove + else + break + fi + ((COUNT++)) + if [ "$COUNT" -gt "$MAX_RETRY" ]; then + exclaim "${COLOR_RED}WARNING: Could not stop Trove services after ${MAX_RETRY} attempts${COLOR_NONE}" + break + fi + done + else + echo "WARNING: Could not stop Trove services because there is no stack screen session running" + fi +} + + +############################################################################### +# Run Integration Tests +############################################################################### + +function cmd_int_tests() { + exclaim "Running Trove Integration Tests..." + if [ ! $USAGE_ENDPOINT ]; then + export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier + fi + cd $REDSTACK_SCRIPTS + if [ $# -lt 1 ]; then + args="--group=blackbox" + else + args="$@" + fi + + # -- verbose makes it prettier. + # -- logging-clear-handlers keeps the novaclient and other things from + # spewing logs to stdout. + args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args" + echo "python $args" + python $args +} + +function cmd_int_tests_simple() { + exclaim "Running Trove Simple Integration Tests..." + cd $REDSTACK_SCRIPTS + if [ $# -lt 1 ]; then + args="--group=simple_blackbox" + else + args="$@" + fi + + # -- verbose makes it prettier. + # -- logging-clear-handlers keeps the novaclient and other things from + # spewing logs to stdout. + args="$INT_TEST_OPTIONS -B $REDSTACK_TESTS/integration/int_tests.py --verbose --logging-clear-handlers $args" + echo "python $args" + python $args +} + +function cmd_int_tests_white_box() { + export PYTHONPATH=$PYTHONPATH:$PATH_TROVE + export PYTHONPATH=$PYTHONPATH:$PATH_NOVA + cmd_int_tests --test-config white_box=True \ + --config-file=$TROVE_CONF \ + --nova-flags=/etc/nova/nova.conf $@ +} + +function cmd_example_tests() { + set +e + cmd_stop + set -e + cmd_start_fake + sleep 3 + echo " +{ + \"directory\": \"$REDSTACK_TESTS/../apidocs/src/resources/samples/\", + \"auth_url\":\"http://$KEYSTONE_AUTH_HOST:35357/v2.0/tokens\", + \"api_url\":\"http://$SERVICE_HOST:8779\", + \"replace_host\":\"https://ord.databases.api.rackspacecloud.com\", + \"replace_dns_hostname\": \"e09ad9a3f73309469cf1f43d11e79549caf9acf2.rackspaceclouddb.com\", + \"username\":\"examples\", + \"password\":\"examples\", + \"tenant\":\"trove\" +}" > /tmp/example-tests.conf + python $REDSTACK_TESTS/examples/examples/example_generation.py /tmp/example-tests.conf + pushd $REDSTACK_TESTS/../apidocs + mvn clean + mvn generate-sources + popd + cmd_stop +} + + +############################################################################### +# Misc. tools +############################################################################### + +function mysql_nova() { + echo mysql nova --execute "$@" + mysql -u root -p$MYSQL_PASSWORD nova --execute "$@" +} + +function mysql_trove() { + echo mysql trove --execute "$@" + mysql -u root -p$MYSQL_PASSWORD trove --execute "$@" +} + +function cmd_wipe_logs() { + for file in `ls $TROVE_LOGDIR/*.log` + do + echo "Reseting log file $file..." + echo "Reset at `date`" > $file + done +} + +function cmd_rd_sql() { + mysql -u root -p$MYSQL_PASSWORD trove +} + +function cmd_fake_sql() { + pushd $PATH_TROVE + sqlite3 trove_test.sqlite $@ + popd +} + +function cmd_vagrant_ssh() { + # Runs a command on a vagrant VM from the host machine. + VHOST=`vagrant ssh_config host | awk '/HostName/{print $2}'` + VUSER=`vagrant ssh_config host | awk '/User /{print $2}'` + VPORT=`vagrant ssh_config host | awk '/Port/{print $2}'` + VIDFILE=`vagrant ssh_config host | awk '/IdentityFile/{print $2}'` + echo ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@" + ssh ${VUSER}@${VHOST} -p ${VPORT} -i ${VIDFILE} -o NoHostAuthenticationForLocalhost=yes "$@" +} + + +function cmd_run_ci() { + exclaim "Running CI suite..." + set +e + cmd_stop_deps + cmd_stop + set -e + cmd_install + cmd_test_init $1 + # The arg will be the image type + cmd_build_and_upload_image $1 + + # Test in fake mode. + exclaim "Testing in fake mode." + cmd_start_fake + FAKE_MODE=True cmd_int_tests + cmd_stop + + # Test in real mode. + exclaim "Testing in real mode." + cmd_start + FAKE_MODE=False cmd_int_tests +} + +function cmd_wipe_queues() { + # Obliterate rabbit. + for i in stop_app reset start_app "change_password guest $RABBIT_PASSWORD"; \ + do sudo rabbitmqctl $i; done +} + +function cmd_clear() { + cmd_int_tests --group=dbaas.api.instances.delete + clean_instances + mysql_nova "DELETE FROM instance_info_caches;" + mysql_nova "DELETE FROM instances;" + mysql_trove "DELETE FROM instances;" + mysql_trove "DELETE FROM service_statuses;" + cmd_wipe_queues +} + +function exec_cmd_on_output() { + local output_cmd=$1 + local exec_cmd=$2 + local delete_sleep_time=${3:-0} + local skip_pattern=${4:-""} + + echo "Cleaning up objects from '${output_cmd}'" + local skip_cmd="cat" + if [[ -n "${skip_pattern}" ]]; then + local temp_skip_cmd=(grep -v "${skip_pattern}") + skip_cmd=${temp_skip_cmd[*]} + fi + local max_retry=10 + local count=1 + local again= + while true; do + ids=$($output_cmd | ${skip_cmd} | grep -v -e'---' | grep -iv ' id ' | cut -d'|' -f2) + if [[ -n $ids ]]; then + for id in $ids; do + echo -e "Executing: ${exec_cmd} ${id} ${again}" + # don't stop if we get an error executing the delete, and don't print + # out anything from stderr + set +e + ${exec_cmd} "${id}" &> /dev/null + set -e + done + sleep "${delete_sleep_time}" + else + break + fi + ((count++)) + if [[ "$count" -gt "$max_retry" ]]; then + exclaim "${COLOR_RED}WARNING: '$output_cmd' still returning output after ${max_retry} delete attempts${COLOR_NONE}" + break + fi + again="${COLOR_BLUE}(again)${COLOR_NONE}" + done +} + +function cmd_clean() { + echo "Cleaning up project '${OS_PROJECT_NAME}'" + # reset any stuck backups + mysql_trove "update backups set state='COMPLETED'" + # clear out any DS version metadata + mysql_trove "delete from datastore_version_metadata" + # reset any stuck instances, and clear all replicas + mysql_trove "update instances set task_id=2, slave_of_id=null" + # reset any stuck clusters + mysql_trove "update clusters set task_id=1" + # get rid of any extraneous quota usage + mysql_trove "delete from quota_usages" + # mark all instance modules as deleted + mysql_trove "update instance_modules set deleted=1" + + source "${PATH_DEVSTACK_SRC}"/openrc admin "${OS_PROJECT_NAME}" + # delete any trove clusters + exec_cmd_on_output "trove cluster-list" "trove cluster-delete" 20 + # delete any trove instances + exec_cmd_on_output "trove list" "trove delete" 10 + # delete any backups + exec_cmd_on_output "trove backup-list" "trove backup-delete" + # clean up any remaining nova instances or cinder volumes + exec_cmd_on_output "nova list" "nova delete" 5 + exec_cmd_on_output "cinder list" "cinder delete" 1 + # delete any config groups since all instances should be gone now + exec_cmd_on_output "trove configuration-list" "trove configuration-delete" + # delete any modules too + exec_cmd_on_output "trove module-list" "trove module-delete" + # make sure that security groups are also gone, except the default + exec_cmd_on_output "openstack security group list" "nova security group delete" 0 "default" + # delete server groups + exec_cmd_on_output "nova server-group-list" "nova server-group-delete" +} + +function cmd_kick_start() { + cmd_test_init $1 + cmd_build_and_upload_image $1 +} + +function cmd_dsvm_gate_tests() { + DATASTORE_TYPE=${1:-'mysql'} + TEST_GROUP=${2:-${DATASTORE_TYPE}} + HOST_SCP_USERNAME=${3:-'jenkins'} + GUEST_USERNAME=${4:-'ubuntu'} + CONTROLLER_IP=${5:-'10.1.0.1'} + ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'} + + if [[ $BRANCH_OVERRIDE == "stable/liberty" ]]; then + # Devstack in liberty doesn't copy the clouds.yaml file to /etc so we need to + # ensure we have access to the clouds.yaml file set up by devstack-gate + sudo mkdir -p ~/.config/openstack + sudo ln -s $DEST/.config/openstack/clouds.yaml ~/.config/openstack/clouds.yaml + sudo chown -R $(whoami) ~/.config + fi + + # Devstack vm-gate runs as the jenkins user, but needs to connect to the guest image as ubuntu + echo "User=ubuntu" >> /home/jenkins/.ssh/config + + # Fix iptables rules that prevent amqp connections from the devstack box to the guests + sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited || true + + sudo chown -R $(whoami) /etc/trove + sudo chown -R $(whoami) $DEST/trove-integration + iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP + iniset $TROVE_GUESTAGENT_CONF oslo_messaging_rabbit rabbit_hosts $CONTROLLER_IP + cd $DEST/trove-integration/scripts + sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils + + cmd_kick_start $DATASTORE_TYPE + + # Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1 + source $DEST/devstack/accrc/admin/admin + # NOTE(mriedem): We have to treat stable branches before liberty special + # due to constraints with older versions of python-openstackclient. + if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then + SWIFT_ENDPOINT=$(openstack endpoint list | grep 'swift' | get_field 1) + openstack endpoint create swift --region RegionOne --publicurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' \ + --internalurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --adminurl 'http://'$CONTROLLER_IP':8080' + openstack endpoint delete $SWIFT_ENDPOINT + else + OS_CLIENT_ARGS="--os-auth-type v3password --os-auth-url $KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:5000/v3 --os-identity-api-version=3" + SWIFT_ENDPOINTS=$(openstack endpoint list $OS_CLIENT_ARGS --service swift -c ID -f value) + openstack endpoint create $OS_CLIENT_ARGS swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne + openstack endpoint create $OS_CLIENT_ARGS swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne + openstack endpoint create $OS_CLIENT_ARGS swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne + echo $SWIFT_ENDPOINTS | xargs -n 1 openstack endpoint delete $OS_CLIENT_ARGS + fi + + cmd_int_tests --group=$TEST_GROUP +} + +function cmd_reset_task() { + mysql_trove "UPDATE instances SET task_id=1 WHERE id='$1'" +} + +function cmd_clone_projects() { + UPDATE_PROJECTS=$1 + PROJECT_LIST_FILES=${@:2} + for project in $(cat $PROJECT_LIST_FILES); do + if [ ! -d $PATH_DEVSTACK_OUTPUT/$project ]; then + echo "Creating a new clone of $project..." + git_clone $GIT_OPENSTACK/"$project".git ${PATH_DEVSTACK_OUTPUT}/$project master + else + if [ $UPDATE_PROJECTS != "force_update" ]; then + echo "$project was already cloned or exists in a shared folder. Ignoring..." + else + echo "$project was already cloned. Pulling changes to update." + cd $PATH_DEVSTACK_OUTPUT/$project + git pull + fi + fi + # Switch to a branch if specified. The order the variables are checked is: + # <PROJECT>_BRANCH then PROJECT_CLIENT_BRANCH (if a client) then PROJECT_BRANCH + # Note: For the Trove project, only TROVE_BRANCH and PYTHON_TROVECLIENT_BRANCH are used + PROJECT_BRANCH_NAME=$(eval echo "${project}_BRANCH" | tr '[:lower:]-' '[:upper:]_') + PROJECT_BRANCH_VALUE=${!PROJECT_BRANCH_NAME} + # TROVE_BRANCH is defaulted to master if not set, so use the original value here + if [[ "$project" = "trove" ]]; then + PROJECT_BRANCH_VALUE=${TROVE_BRANCH_ORIG} + fi + BRANCH_SPECIFIED=$(test -z "${PROJECT_BRANCH_VALUE}${PROJECT_CLIENT_BRANCH}${PROJECT_BRANCH}" || echo 'True') + if [[ "${BRANCH_SPECIFIED}" = "True" ]]; then + # Set up the default branch and env var names for the project + DEFAULT_BRANCH="$PROJECT_BRANCH" + ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_BRANCH" + # Don't use 'PROJECT_BRANCH' or 'PROJECT_CLIENT_BRANCH' for the Trove project + if [[ "$project" =~ "trove" ]]; then + DEFAULT_BRANCH=master + ENV_VARS="$PROJECT_BRANCH_NAME" + # Use 'PROJECT_CLIENT_BRANCH' first for clients + elif [[ "$project" =~ "client" ]]; then + DEFAULT_BRANCH="${PROJECT_CLIENT_BRANCH:-$PROJECT_BRANCH}" + ENV_VARS="$PROJECT_BRANCH_NAME' or 'PROJECT_CLIENT_BRANCH' or 'PROJECT_BRANCH" + fi + PROJ_BRANCH=$(get_project_branch $PROJECT_BRANCH_NAME $DEFAULT_BRANCH) + git_checkout "$project" "$PATH_DEVSTACK_OUTPUT/$project" "$PROJ_BRANCH" "$ENV_VARS" + fi + done +} + +function cmd_repl() { + INT_TEST_OPTIONS=-i cmd_int_tests_white_box --repl --group=_does_not_exist_ $@ +} + + +############################################################################### +# Process the user provided command and run the appropriate command +############################################################################### + +# Let's not run this as the root user +if [ $EUID -eq 0 ]; then + echo "You are running this script as root. You need to run as a regular user" + exit 1 +fi + +# Set this to exit immediately on error +set -o errexit + +# set_home_dir +set_http_proxy +set_trove_plugin_vars false + +function print_usage() { + echo "Usage: $0 [command]" + echo " + Commands : + --setup environment-- + install - Install all the required dependencies and bring up tr-api and tr-tmgr + - devstack config can be altered by using a USER_LOCAL_CONF file + which will be copied into devstack/local.conf on each 'install' run + (defaults to \$HOME/$USER_LOCAL_CONF_NAME) + - Set DEVSTACK_BRANCH to switch the branch/commit of devstack + (i.e. 'stable/kilo' or '7ef2462') + test-init - Configure the test configuration files and add keystone test users + build-image - Builds the vm image for the trove guest + initialize - Reinitialize the trove database, users, services, and test config + + --helper for environment-- + kick-start - kick start the setup of trove. + (redstack test-init/build-image in one step) + [mysql no-clean] no clean avoids rebuilding packages from scratch + - Set REBUILD_IMAGE=True to force rebuild (won't use cached image) + + --trove dependency services-- + start-deps - Start or resume daemons Trove depends on. + stop-deps - Kill daemons Trove depends on. + + --trove services-- + start - Start or resume daemons Trove depends on. + stop - Kill daemons Trove depends on. + restart - Runs stop then start for Trove services. + + --tests-- + unit-tests - Run the unit tests.dependencies + int-tests - Runs the integration tests (requires all daemons). + simple-tests - Runs the simple integration tests (requires all daemons). + dsvm-gate-tests - Configures and runs the int-tests in a devstack vm-gate environment. + + --tools-- + debug - Debug this script (shows all commands). + wipe-logs - Resets all log files. + rd-sql - Opens the Trove MySQL database. + vagrant-ssh - Runs a command from the host on the server. + clear - Destroy instances and rabbit queues. + clean - Clean up resources created by a failed test run. + run - Starts RD but not in a screen. + run-fake - Runs the server in fake mode. + update-projects - Git pull on all the daemons trove dependencies. + reset-task - Sets an instance task to NONE. + wipe-queues - Resets RabbitMQ queues. + " + exit 1 +} + +function run_command() { + # Print the available commands + if [ $# -lt 1 ]; then + print_usage + fi + + case "$1" in + "install" ) cmd_install;; + "test-init" ) cmd_test_init $@;; + "build-image" ) shift; cmd_build_image $@;; + "initialize" ) cmd_initialize;; + "unit-tests" ) cmd_unit_tests;; + "start-deps" ) cmd_start_deps;; + "stop-deps" ) cmd_stop_deps;; + "start" ) cmd_start;; + "int-tests" ) shift; cmd_int_tests $@;; + "int-tests-wb" ) shift; cmd_int_tests_white_box $@;; + "simple-tests") shift; cmd_int_tests_simple $@;; + "stop" ) cmd_stop;; + "restart" ) cmd_stop; cmd_start;; + "wipe-logs" ) cmd_wipe_logs;; + "rd-sql" ) shift; cmd_rd_sql $@;; + "fake-sql" ) shift; cmd_fake_sql $@;; + "run-ci" ) shift; cmd_run_ci $@;; + "vagrant-ssh" ) shift; cmd_vagrant_ssh $@;; + "debug" ) shift; echo "Enabling debugging."; \ + set -o xtrace; run_command $@;; + "clear" ) shift; cmd_clear $@;; + "clean" ) shift; cmd_clean $@;; + "run" ) shift; cmd_run $@;; + "kick-start" ) shift; cmd_kick_start $@;; + "dsvm-gate-tests" ) shift; cmd_dsvm_gate_tests $@;; + "run-fake" ) shift; cmd_run_fake $@;; + "start-fake" ) shift; cmd_start_fake $@;; + "update-projects" ) cmd_clone_projects force_update \ + $REDSTACK_SCRIPTS/projects-list \ + $REDSTACK_SCRIPTS/image-projects-list;; + "reset-task" ) shift; cmd_reset_task $@;; + "wipe-queues" ) shift; cmd_wipe_queues $@;; + "example-tests" ) shift; cmd_example_tests $@;; + "repl" ) shift; cmd_repl $@;; + "help" ) print_usage;; + * ) + echo "'$1' not a valid command" + exit 1 + esac +} + +run_command $@ diff --git a/integration/scripts/redstack.rc b/integration/scripts/redstack.rc new file mode 100644 index 00000000..b4036ba1 --- /dev/null +++ b/integration/scripts/redstack.rc @@ -0,0 +1,120 @@ +# Paths inside the VM. +[ -z $SERVICE_HOST ] && SERVICE_HOST=`get_default_host_ip` +[ -z $DEST ] && DEST=/opt/stack +[ -z $BRIDGE_IP ] && BRIDGE_IP=172.24.4.1 +[ -z $PATH_DEVSTACK_SRC ] && PATH_DEVSTACK_SRC=~/devstack +[ -z $TROVE_CONF_DIR ] && TROVE_CONF_DIR=/etc/trove +[ -z $MYSQL_HOST ] && MYSQL_HOST=$SERVICE_HOST + +# Set up the region name +# Try REGION_NAME then OS_REGION_NAME then RegionOne (the devstack default) +REGION_NAME=${REGION_NAME:-${OS_REGION_NAME:-RegionOne}} + +# Enable neutron instead of nova-network +# Note: Until a few key changesets land, we can't enable Neutron properly. +# See: https://review.openstack.org/#/c/356026 +# https://review.openstack.org/#/c/356763 +# https://review.openstack.org/#/c/356701 +# NEUTRON_DEFAULT=true +NEUTRON_DEFAULT=false +if [[ $BRANCH_OVERRIDE == "stable/liberty" || $BRANCH_OVERRIDE == "stable/mitaka" ]]; then + NEUTRON_DEFAULT=false +fi +ENABLE_NEUTRON=$(get_bool ENABLE_NEUTRON $NEUTRON_DEFAULT) + +# Enable osprofiler - note: Enables Ceilometer as well +ENABLE_PROFILER=$(get_bool ENABLE_PROFILER false) +PROFILER_TRACE_SQL=$(get_bool PROFILER_TRACE_SQL false) +[ -z $PROFILER_HMAC_KEYS ] && PROFILER_HMAC_KEYS=SECRET_KEY + +# Enable ceilometer +ENABLE_CEILOMETER=$(get_bool ENABLE_CEILOMETER $ENABLE_PROFILER) + +# Enable Mistral +ENABLE_MISTRAL=$(get_bool ENABLE_MISTRAL false) + +# Enable LIBS_FROM_GIT +LIBS_FROM_GIT_ALL_CLIENTS=$(get_bool LIBS_FROM_GIT_ALL_CLIENTS false) +LIBS_FROM_GIT_ALL_OSLO=$(get_bool LIBS_FROM_GIT_ALL_OSLO false) + +# Don't include certain .rc files in local.conf.d by default +USING_VAGRANT=$(get_bool USING_VAGRANT false) +USE_KVM=$(get_bool USE_KVM false) +USE_UUID_TOKEN=$(get_bool USE_UUID_TOKEN false) + +# Specify configuration for Ceilometer +CEILOMETER_SERVICES_CONF=$(get_bool CEILOMETER_SERVICES_CONF $ENABLE_CEILOMETER) +CEILOMETER_CINDER_CONF=$(get_bool CEILOMETER_CINDER_CONF false) +CEILOMETER_NOVA_CONF=$(get_bool CEILOMETER_NOVA_CONF false) + +# Paths for various OpenStack components +PATH_DEVSTACK_OUTPUT=$DEST +PATH_NOVA=$DEST/nova +PATH_KEYSTONE=$DEST/keystone +PATH_GLANCE=$DEST/glance +PATH_SWIFT=$DEST/swift +PATH_TROVE=$DEST/trove +PATH_PYTHON_NOVACLIENT=$DEST/python-novaclient +PATH_KEYSTONECLIENT=$DEST/python-keystoneclient +PATH_OPENSTACKCLIENT=$DEST/python-openstackclient +PATH_PYTHON_SWIFTCLIENT=$DEST/python-swiftclient +PATH_PYTHON_TROVECLIENT=$DEST/python-troveclient +PATH_TROVE_DASHBOARD=$DEST/trove-dashboard +PATH_DISKIMAGEBUILDER=$DEST/diskimage-builder +PATH_TRIPLEO_ELEMENTS=$DEST/tripleo-image-elements + +# Save the state of TROVE_BRANCH first, since it's used in redstack +TROVE_BRANCH_ORIG=${TROVE_BRANCH} +# Devstack and OpenStack git repo source paths, etc. +GIT_BASE=${GIT_BASE:-git://git.openstack.org} +GIT_OPENSTACK=${GIT_OPENSTACK:-${GIT_BASE}/openstack} +DEVSTACK_REPO=${DEVSTACK_REPO:-${GIT_BASE}/openstack-dev/devstack.git} +TROVE_REPO=${TROVE_REPO:-${GIT_OPENSTACK}/trove.git} +TROVE_DIR=${TROVE_DIR:-${PATH_TROVE}} +TROVE_BRANCH=${TROVE_BRANCH:-master} +TROVE_CLIENT_REPO=${TROVE_CLIENT_REPO:-${TROVECLIENT_REPO:-${GIT_OPENSTACK}/python-troveclient.git}} +TROVE_CLIENT_DIR=${TROVE_CLIENT_DIR:-${TROVECLIENT_DIR:-${PATH_PYTHON_TROVECLIENT}}} +TROVE_CLIENT_BRANCH=${TROVE_CLIENT_BRANCH:-${TROVECLIENT_BRANCH:-master}} +TROVE_DASHBOARD_REPO=${TROVE_DASHBOARD_REPO:-${TROVEDASHBOARD_REPO:-${GIT_OPENSTACK}/trove-dashboard.git}} +TROVE_DASHBOARD_DIR=${TROVE_DASHBOARD_DIR:-${TROVEDASHBOARD_DIR:-${PATH_TROVE_DASHBOARD}}} +TROVE_DASHBOARD_BRANCH=${TROVE_DASHBOARD_BRANCH:-${TROVEDASHBOARD_BRANCH:-master}} + +# Destination for working data +DATA_DIR=${DEST}/data +# Destination for status files +SERVICE_DIR=${DEST}/status + +# Cinder Volume Group Name +VOLUME_GROUP=${VOLUME_GROUP:-stack-volumes} +VOLUME_BACKING_FILE=${VOLUME_BACKING_FILE:-${DATA_DIR}/${VOLUME_GROUP}-backing-file} +VOLUME_BACKING_FILE_SIZE=${VOLUME_BACKING_FILE_SIZE:-51200M} + +# Passwords used by devstack. +MYSQL_PASSWORD=e1a2c042c828d3566d0a +RABBIT_PASSWORD=f7999d1955c5014aa32c +SERVICE_TOKEN=be19c524ddc92109a224 +ADMIN_PASSWORD=3de4922d8b6ac5a1aad9 +SERVICE_PASSWORD=7de4162d826bc5a11ad9 + +# Swift hash used by devstack. +SWIFT_HASH=12go358snjw24501 + +# Swift Disk Image +SWIFT_DATA_DIR=${DATA_DIR}/swift +SWIFT_DISK_IMAGE=${SWIFT_DATA_DIR}/drives/images/swift.img + +DISTRO=${DISTRO:-ubuntu} +#DISTRO=fedora + +# The following values can be used to tweak how devstack sets +# up Trove. If not explicitly set, the defaults in the code are used. +# To make changes without modifying the repo, add these variables +# to options.rc or ~/redstack.options.rc +#export TROVE_MAX_ACCEPTED_VOLUME_SIZE=10 +#export TROVE_MAX_INSTANCES_PER_TENANT=10 +#export TROVE_MAX_VOLUMES_PER_TENANT=40 +#export TROVE_AGENT_CALL_LOW_TIMEOUT=15 +#export TROVE_AGENT_CALL_HIGH_TIMEOUT=300 +#export TROVE_RESIZE_TIME_OUT=900 +#export TROVE_USAGE_TIMEOUT=1500 +#export TROVE_STATE_CHANGE_WAIT_TIME=180 diff --git a/integration/scripts/reviews.rc b/integration/scripts/reviews.rc new file mode 100644 index 00000000..accccdcd --- /dev/null +++ b/integration/scripts/reviews.rc @@ -0,0 +1,5 @@ +# This file will contain variables such as below, uncommented. +# There will be a : separator between multiple inflight reviews +# The path comes from the gerrit review system, and is the only +# unique portion of each gerrit review. +#REVIEW_PYTHON_NOVACLIENT=18/5018/2 |