summaryrefslogtreecommitdiff
path: root/integration/scripts
diff options
context:
space:
mode:
authorLingxian Kong <anlin.kong@gmail.com>2020-04-07 10:52:16 +1200
committerLingxian Kong <anlin.kong@gmail.com>2020-05-27 10:31:50 +1200
commitaa1d4d224674f44d9cd882eddb2537907adf5382 (patch)
treeb9baf26bfb4b15497696d020fea41364bd0576c8 /integration/scripts
parent523d66e8fd5d29be8dbae9aa79c5348d3dce8c64 (diff)
downloadtrove-aa1d4d224674f44d9cd882eddb2537907adf5382.tar.gz
Datastore containerization
Significant changes: * Using docker image to install datastore. * Datastore image is common to different datastores. * Using backup docker image to do backup and restore. * Support MariaDB replication * Set most of the functional jobs as non-voting as nested virtualization is not supported in CI. Change-Id: Ia9c97a63a961eebc336b70d28dc77638144c1834
Diffstat (limited to 'integration/scripts')
-rw-r--r--integration/scripts/conf.json.example12
-rw-r--r--integration/scripts/files/elements/apt-conf-dir/README.rst16
-rwxr-xr-xintegration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd21
-rw-r--r--integration/scripts/files/elements/guest-agent/element-deps1
-rw-r--r--integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash34
-rwxr-xr-xintegration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install51
-rwxr-xr-xintegration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install45
-rw-r--r--integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service31
-rw-r--r--integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service7
-rw-r--r--integration/scripts/files/elements/guest-agent/package-installs.yaml54
-rwxr-xr-xintegration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd (renamed from integration/scripts/files/elements/guest-agent/post-install.d/11-enable-guest-agent-systemd)0
-rw-r--r--integration/scripts/files/elements/no-resolvconf/README.rst8
-rwxr-xr-xintegration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf19
-rw-r--r--integration/scripts/files/elements/ubuntu-docker/element-deps1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-docker/install.d/21-docker19
-rw-r--r--integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash34
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev17
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep28
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key27
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps10
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/11-user (renamed from integration/scripts/files/elements/guest-agent/install.d/50-user)2
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev22
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep37
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/50-user18
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key25
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/98-ssh8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt11
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/11-ipforwarding (renamed from integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding)0
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/12-ntp (renamed from integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp)0
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/post-install.d/13-trove-guest-sudoers (renamed from integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers)0
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools7
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools7
-rw-r--r--integration/scripts/files/elements/ubuntu-mariadb/README.md3
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local11
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local12
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql25
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo14
-rw-r--r--integration/scripts/files/elements/ubuntu-xenial-guest/element-deps1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd21
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc8
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates12
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel29
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs90
-rw-r--r--integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb39
-rw-r--r--integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps1
-rwxr-xr-xintegration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql39
-rw-r--r--integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps1
-rw-r--r--integration/scripts/functions_qemu85
-rw-r--r--integration/scripts/local.conf.d/ceilometer_cinder.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/ceilometer_nova.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/ceilometer_services.conf.rc3
-rw-r--r--integration/scripts/local.conf.d/sample.rc42
-rw-r--r--integration/scripts/local.conf.d/trove_services.conf.rc24
-rw-r--r--integration/scripts/local.conf.d/use_kvm.rc4
-rw-r--r--integration/scripts/local.conf.d/use_uuid_token.rc3
-rw-r--r--integration/scripts/local.conf.d/using_vagrant.rc9
-rw-r--r--integration/scripts/local.conf.rc37
-rwxr-xr-xintegration/scripts/trovestack162
59 files changed, 242 insertions, 1012 deletions
diff --git a/integration/scripts/conf.json.example b/integration/scripts/conf.json.example
deleted file mode 100644
index ee6bc7bf..00000000
--- a/integration/scripts/conf.json.example
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "devstack":null,
- "glance":null,
- "horizon":null,
- "keystone":null,
- "nova":null,
- "python_openstackclient":null,
- "python_novaclient":null,
- "trove":null,
- "python_troveclient":null,
- "tempest":null
-}
diff --git a/integration/scripts/files/elements/apt-conf-dir/README.rst b/integration/scripts/files/elements/apt-conf-dir/README.rst
deleted file mode 100644
index c94e00ea..00000000
--- a/integration/scripts/files/elements/apt-conf-dir/README.rst
+++ /dev/null
@@ -1,16 +0,0 @@
-============
-apt-conf-dir
-============
-
-This element overrides the default apt.conf.d directory for APT based systems.
-
-Environment Variables
----------------------
-
-DIB_APT_CONF_DIR:
- :Required: No
- :Default: None
- :Description: To override `DIB_APT_CONF_DIR`, set it to the path to your
- apt.conf.d. The new apt.conf.d will take effect at build time
- and run time.
- :Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf``
diff --git a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd
deleted file mode 100755
index e286d684..00000000
--- a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
- set -x
-fi
-set -eu
-set -o pipefail
-
-# exit directly if DIB_APT_CONF_DIR is not defined properly
-if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then
- echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in"
- exit 0
-elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then
- echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory."
- echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR"
- exit 1
-fi
-
-# copy the apt.conf to cloudimg
-sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt
diff --git a/integration/scripts/files/elements/guest-agent/element-deps b/integration/scripts/files/elements/guest-agent/element-deps
index 6dcd66dc..ef309837 100644
--- a/integration/scripts/files/elements/guest-agent/element-deps
+++ b/integration/scripts/files/elements/guest-agent/element-deps
@@ -4,3 +4,4 @@ pkg-map
source-repositories
svc-map
pip-and-virtualenv
+ubuntu-docker
diff --git a/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash b/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash
deleted file mode 100644
index 9622b292..00000000
--- a/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash
+++ /dev/null
@@ -1,34 +0,0 @@
-# sometimes the primary key server is unavailable and we should try an
-# alternate. see
-# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
-# shell errexit so we can interrogate the exit code and take action
-# based on the exit code. We will reenable it later.
-#
-# NOTE(zhaochao): we still have this problem from time to time, so it's
-# better use more reliable keyservers and just retry on that(for now, 3
-# tries should be fine).
-# According to:
-# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
-# [2] https://sks-keyservers.net/overview-of-pools.php
-# we'll just the primary suggested pool: pool.sks-keyservers.net.
-function get_key_robust() {
- KEY=$1
- set +e
-
- tries=1
- while [ $tries -le 3 ]; do
- if [ $tries -eq 3 ]; then
- set -e
- fi
-
- echo "Importing the key, try: $tries"
- # Behind a firewall should use the port 80 instead of the default port 11371
- apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
-
- tries=$((tries+1))
- done
-
- set -e
-}
-
-export -f get_key_robust
diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install
new file mode 100755
index 00000000..1eb66ffd
--- /dev/null
+++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install
@@ -0,0 +1,51 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+DEV_MODE=${DEV_MODE:-"true"}
+SCRIPTDIR=$(dirname $0)
+GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
+GUEST_VENV=/opt/guest-agent-venv
+
+for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove" "/opt/trove-guestagent"; do
+ mkdir -p ${folder}
+ chown -R ${GUEST_USERNAME}:root ${folder}
+done
+
+install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
+
+# Create a virtual environment (with dependencies installed) for guest agent service
+${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
+${GUEST_VENV}/bin/pip install pip --upgrade
+${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
+chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
+
+if [[ ${DEV_MODE} == "true" ]]; then
+ [[ -n "${HOST_SCP_USERNAME}" ]] || die "HOST_SCP_USERNAME needs to be set to the trovestack host user"
+ [[ -n "${ESCAPED_PATH_TROVE}" ]] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
+
+ sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${SCRIPTDIR}/guest-agent-dev.service > /etc/systemd/system/guest-agent.service
+else
+ # Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
+ ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
+
+ case "$DIB_INIT_SYSTEM" in
+ systemd)
+ sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /etc/systemd/system/guest-agent.service
+ ;;
+ upstart)
+ install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
+ ;;
+ sysv)
+ install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
+ ;;
+ *)
+ echo "Unsupported init system"
+ exit 1
+ ;;
+ esac
+fi
diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install
deleted file mode 100755
index 87a11958..00000000
--- a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/bin/bash
-
-if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
- set -x
-fi
-set -eu
-set -o pipefail
-
-SCRIPTDIR=$(dirname $0)
-GUEST_VENV=/opt/guest-agent-venv
-GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
-
-# Create a virtual environment for guest agent
-${DIB_PYTHON} -m virtualenv ${GUEST_VENV}
-${GUEST_VENV}/bin/pip install pip --upgrade
-${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent
-chown -R ${GUEST_USERNAME}:root ${GUEST_VENV}
-
-# Link the trove-guestagent out to /usr/local/bin where the startup scripts look for
-ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true
-
-for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove"; do
- mkdir -p ${folder}
- chown -R ${GUEST_USERNAME}:root ${folder}
-done
-
-install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent
-
-case "$DIB_INIT_SYSTEM" in
- systemd)
- mkdir -p /usr/lib/systemd/system
- touch /usr/lib/systemd/system/guest-agent.service
- sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /usr/lib/systemd/system/guest-agent.service
- ;;
- upstart)
- install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf
- ;;
- sysv)
- install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init
- ;;
- *)
- echo "Unsupported init system"
- exit 1
- ;;
-esac
diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service
new file mode 100644
index 00000000..261ac811
--- /dev/null
+++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service
@@ -0,0 +1,31 @@
+[Unit]
+Description=OpenStack Trove Guest Agent Service for Development
+After=syslog.target network.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=simple
+User=GUEST_USERNAME
+Group=GUEST_USERNAME
+
+# This script is only for testing purpose for dev_mode=true, the controller
+# IP address should be defined in /etc/trove/controller.conf, e.g.
+# CONTROLLER=192.168.32.151
+EnvironmentFile=/etc/trove/controller.conf
+
+# If ~/trove-installed does not exist, copy the trove source from
+# the user's development environment, then touch the sentinel file
+ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/GUEST_USERNAME/.ssh/id_rsa' -az --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
+
+ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/ /home/GUEST_USERNAME/trove"
+
+# Take care of the changes in requirements.txt
+ExecStartPre=/bin/bash -c "sudo /opt/guest-agent-venv/bin/pip install -r /home/GUEST_USERNAME/trove/requirements.txt -c /opt/upper-constraints.txt"
+
+# Start guest-agent.service in virtual environment
+ExecStart=/bin/bash -c "/opt/guest-agent-venv/bin/python /home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d"
+
+TimeoutSec=300
+Restart=on-failure \ No newline at end of file
diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service
index 788bebc9..a468de54 100644
--- a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service
+++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service
@@ -1,15 +1,16 @@
[Unit]
-Description=OpenStack Trove Guest Agent
+Description=OpenStack Trove Guest Agent Service
After=network.target syslog.service
Wants=syslog.service
[Service]
User=GUEST_USERNAME
Group=GUEST_USERNAME
-ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove/conf.d"
-ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
KillMode=mixed
Restart=always
+ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/"
+ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d
+
[Install]
WantedBy=multi-user.target
diff --git a/integration/scripts/files/elements/guest-agent/package-installs.yaml b/integration/scripts/files/elements/guest-agent/package-installs.yaml
index 582d48bc..37e7daa4 100644
--- a/integration/scripts/files/elements/guest-agent/package-installs.yaml
+++ b/integration/scripts/files/elements/guest-agent/package-installs.yaml
@@ -1,53 +1,15 @@
guest-agent:
installtype: package
build-essential:
- installtype: source
+python3-all:
+python3-all-dev:
+python3-pip:
+python3-sqlalchemy:
+libxml2-dev:
+libxslt1-dev:
libffi-dev:
- installtype: source
libssl-dev:
- installtype: source
-python-dev:
- installtype: source
-
-acl:
-acpid:
-apparmor:
-apparmor-utils:
-apt-transport-https:
-at:
-bash-completion:
-cloud-guest-utils:
-cloud-init:
-cron:
-curl:
-dbus:
-dkms:
-dmeventd:
-ethtool:
-gawk:
-ifenslave:
-ifupdown:
-iptables:
-iputils-tracepath:
-irqbalance:
-isc-dhcp-client:
-less:
-logrotate:
-lsof:
-net-tools:
-netbase:
-netcat-openbsd:
-open-vm-tools:
- arch: i386, amd64
+libyaml-dev:
openssh-client:
openssh-server:
-pollinate:
-psmisc:
-rsyslog:
-socat:
-tcpdump:
-ubuntu-cloudimage-keyring:
-ureadahead:
-uuid-runtime:
-vim-tiny:
-vlan:
+rsync:
diff --git a/integration/scripts/files/elements/guest-agent/post-install.d/11-enable-guest-agent-systemd b/integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd
index 1f1eb7a4..1f1eb7a4 100755
--- a/integration/scripts/files/elements/guest-agent/post-install.d/11-enable-guest-agent-systemd
+++ b/integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd
diff --git a/integration/scripts/files/elements/no-resolvconf/README.rst b/integration/scripts/files/elements/no-resolvconf/README.rst
deleted file mode 100644
index 8a3dfc7d..00000000
--- a/integration/scripts/files/elements/no-resolvconf/README.rst
+++ /dev/null
@@ -1,8 +0,0 @@
-This element clears out /etc/resolv.conf and prevents dhclient from populating
-it with data from DHCP. This means that DNS resolution will not work from the
-guest. This is OK because all outbound connections from the guest will
-be based using raw IP addresses.
-
-In addition we remove dns from the nsswitch.conf hosts setting.
-
-This means that the guest never waits for DNS timeouts to occur.
diff --git a/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf b/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf
deleted file mode 100755
index 5bf5fded..00000000
--- a/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-echo "" > /etc/resolv.conf
-echo "" > /etc/resolv.conf.ORIG
-if [ -d /etc/dhcp/dhclient-enter-hooks.d ]; then
- # Debian/Ubuntu
- echo "#!/bin/sh
-make_resolv_conf() { : ; }" > /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
- chmod +x /etc/dhcp/dhclient-enter-hooks.d/noresolvconf
- rm -f /etc/dhcp/dhclient-enter-hooks.d/resolvconf
-else
- # RHEL/CentOS/Fedora
- echo "#!/bin/sh
-make_resolv_conf() { : ; }" > /etc/dhclient-enter-hooks
- chmod +x /etc/dhclient-enter-hooks
-fi
-
-if [ -e /etc/nsswitch.conf ]; then
- sed -i -e "/hosts:/ s/dns//g" /etc/nsswitch.conf
-fi
diff --git a/integration/scripts/files/elements/ubuntu-docker/element-deps b/integration/scripts/files/elements/ubuntu-docker/element-deps
new file mode 100644
index 00000000..10e54583
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-docker/element-deps
@@ -0,0 +1 @@
+ubuntu-guest \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker
new file mode 100755
index 00000000..44041384
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker
@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"}
+
+echo "Installing docker"
+export DEBIAN_FRONTEND=noninteractive
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
+add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${DIB_RELEASE} stable"
+apt-get update
+apt-get install -y -qq docker-ce >/dev/null
+
+echo "Adding ${GUEST_USERNAME} user to docker group"
+usermod -aG docker ${GUEST_USERNAME}
diff --git a/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash b/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash
deleted file mode 100644
index 9622b292..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash
+++ /dev/null
@@ -1,34 +0,0 @@
-# sometimes the primary key server is unavailable and we should try an
-# alternate. see
-# https://bugs.launchpad.net/percona-server/+bug/907789. Disable
-# shell errexit so we can interrogate the exit code and take action
-# based on the exit code. We will reenable it later.
-#
-# NOTE(zhaochao): we still have this problem from time to time, so it's
-# better use more reliable keyservers and just retry on that(for now, 3
-# tries should be fine).
-# According to:
-# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver
-# [2] https://sks-keyservers.net/overview-of-pools.php
-# we'll just the primary suggested pool: pool.sks-keyservers.net.
-function get_key_robust() {
- KEY=$1
- set +e
-
- tries=1
- while [ $tries -le 3 ]; do
- if [ $tries -eq 3 ]; then
- set -e
- fi
-
- echo "Importing the key, try: $tries"
- # Behind a firewall should use the port 80 instead of the default port 11371
- apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break
-
- tries=$((tries+1))
- done
-
- set -e
-}
-
-export -f get_key_robust
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev
new file mode 100755
index 00000000..498169c3
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+source $_LIB/die
+
+[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
+
+# Guest agent needs to ssh into the controller to download code in dev mode.
+if [[ ${DEV_MODE} == "true" && -e ${SSH_DIR}/id_rsa ]]; then
+ sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
+ sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
+fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep
deleted file mode 100755
index 5a0c1ecd..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep
-
-source $_LIB/die
-
-TROVE_BRANCH=${TROVE_BRANCH:-'master'}
-REQUIREMENTS_FILE=${TROVESTACK_SCRIPTS}/../../requirements.txt
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found"
-[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set"
-
-sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt
-
-UC_FILE=upper-constraints.txt
-UC_DIR=$(pwd)
-UC_BRANCH=${TROVE_BRANCH##stable/}
-
-curl -L -o "${UC_DIR}/${UC_FILE}" "https://releases.openstack.org/constraints/upper/${UC_BRANCH}"
-if [ -f "${UC_DIR}/${UC_FILE}" ]; then
- sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE}
- rm -f "${UC_DIR}/${UC_FILE}"
-fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key
deleted file mode 100755
index 894950db..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where
-# they will be duplicated in the guest VM.
-# This process allows the host to log into the guest but more importantly the guest phones home to get the trove
-# source
-
-source $_LIB/die
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host"
-
-if [ `whoami` = "root" ]; then
- die "This should not be run as root"
-fi
-
-# Guest agent needs to ssh into the controller to download code in dev mode.
-if [ -e ${SSH_DIR}/id_rsa ]; then
- sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa
- sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub
-else
- die "SSH keys must exist"
-fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps
deleted file mode 100755
index 3a8cacfb..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install basic services and applications
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get --allow-unauthenticated -y install ntp apparmor-utils
diff --git a/integration/scripts/files/elements/guest-agent/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/11-user
index 8a2b145f..456073f7 100755
--- a/integration/scripts/files/elements/guest-agent/install.d/50-user
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/11-user
@@ -19,4 +19,4 @@ if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
${GUEST_USERNAME}
${GUEST_USERNAME}
_EOF_
-fi
+fi \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev b/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev
new file mode 100755
index 00000000..8be7882a
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev
@@ -0,0 +1,22 @@
+#!/bin/bash
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+ set -x
+fi
+set -eu
+set -o pipefail
+
+GUEST_SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
+TMP_HOOKS_DIR="/tmp/in_target.d"
+
+if [ ! -e ${GUEST_SSH_DIR} ]; then
+ # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
+ mkdir ${GUEST_SSH_DIR}
+ chown -R ${GUEST_USERNAME}:${GUEST_USERNAME} ${GUEST_SSH_DIR}
+fi
+
+if [[ ${DEV_MODE} == "true" && -e "${TMP_HOOKS_DIR}/id_rsa" ]]; then
+ sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
+ sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
+ sudo -Hiu ${GUEST_USERNAME} chmod 600 ${GUEST_SSH_DIR}/id_rsa
+fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep b/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep
deleted file mode 100755
index 25b48459..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get --allow-unauthenticated -y install \
- libxml2-dev libxslt1-dev libffi-dev libssl-dev libyaml-dev \
- python3-pip python3-sqlalchemy python3-setuptools
-
-# Install python 3.7, some python lib (e.g. oslo.concurrency>4.0.0) requries
-# Python 3.7
-add-apt-repository --yes ppa:deadsnakes/ppa
-apt update
-apt install -y python3.7 python3.7-dev
-
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10
-python3.5 -m pip install pip==9.0.3
-python3.5 -m pip install -U wheel setuptools
-
-TMP_HOOKS_DIR="/tmp/in_target.d"
-
-UPPER_CONSTRAINTS=
-if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then
- UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt"
-fi
-
-python3.7 -m pip install pip==9.0.3
-python3.7 -m pip install -U wheel setuptools
-python3.7 -m pip install --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS}
-
-echo "diagnostic pip freeze output follows"
-python3.7 -m pip freeze
-echo "diagnostic pip freeze output above"
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user
deleted file mode 100755
index 99f68966..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist
-
-set -e
-set -o xtrace
-
-if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then
- echo "Adding ${GUEST_USERNAME} user"
- useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash
- chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME}
- passwd ${GUEST_USERNAME} <<_EOF_
-${GUEST_USERNAME}
-${GUEST_USERNAME}
-_EOF_
-fi
-
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key
deleted file mode 100755
index 21a54d28..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory
-# In future, this should be removed and use Nova keypair to inject ssh keys.
-
-set -e
-set -o xtrace
-
-SSH_DIR="/home/${GUEST_USERNAME}/.ssh"
-TMP_HOOKS_DIR="/tmp/in_target.d"
-
-if [ ! -e ${SSH_DIR} ]; then
- # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME}
- mkdir ${SSH_DIR}
- chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR}
-fi
-
-if [ -e "${TMP_HOOKS_DIR}/id_rsa" ]; then
- sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub
- sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa
- sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa
-else
- echo "SSH Keys were not staged by host"
- exit -1
-fi
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh
deleted file mode 100755
index 2134c198..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/bash
-# Regenerate host keys now. XXX: Really should be a cloud-init task, should get
-# that working.
-
-set -e
-set -o xtrace
-
-dpkg-reconfigure openssh-server
diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt
deleted file mode 100755
index cc348c5c..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Delete contents of apt cache on guest (saves image disk space)
-
-set -e
-set -o xtrace
-
-apt-get clean
-
-
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/ubuntu-guest/post-install.d/11-ipforwarding
index 48b7ad0d..48b7ad0d 100755
--- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/11-ipforwarding
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp b/integration/scripts/files/elements/ubuntu-guest/post-install.d/12-ntp
index bc494ae3..bc494ae3 100755
--- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/12-ntp
diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/ubuntu-guest/post-install.d/13-trove-guest-sudoers
index 0581fd2b..0581fd2b 100755
--- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers
+++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/13-trove-guest-sudoers
diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools
deleted file mode 100755
index 7b9221cf..00000000
--- a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/bash
-# Install baseline packages and tools.
-
-set -e
-set -o xtrace
-
-apt-get --allow-unauthenticated install -y language-pack-en python-software-properties software-properties-common \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools
new file mode 100755
index 00000000..12c864d6
--- /dev/null
+++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools
@@ -0,0 +1,7 @@
+#!/bin/bash
+# Install baseline packages and tools.
+
+set -e
+set -o xtrace
+
+DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated install -y -qq software-properties-common apt-transport-https ca-certificates ntp >/dev/null \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/README.md b/integration/scripts/files/elements/ubuntu-mariadb/README.md
deleted file mode 100644
index 757f00b8..00000000
--- a/integration/scripts/files/elements/ubuntu-mariadb/README.md
+++ /dev/null
@@ -1,3 +0,0 @@
-Sets up a MariaDB server install in the image.
-
-TODO: auto-tune settings based on host resources or metadata service.
diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local
deleted file mode 100755
index a3e1dc7c..00000000
--- a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/sh
-
-set -e
-
-#CONTEXT: chroot on host
-#PURPOSE: Allows mysqld to create temporary files when restoring backups
-
-cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
- /tmp/ rw,
- /tmp/** rwk,
-EOF
diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local
deleted file mode 100755
index 90bd85b1..00000000
--- a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-set -e
-
-#CONTEXT: chroot on host
-#PURPOSE: Allows mysqld to create temporary files when restoring backups
-
-mkdir -p /etc/apparmor.d/local/
-cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld
- /tmp/ rw,
- /tmp/** rwk,
-EOF
diff --git a/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql b/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql
deleted file mode 100755
index cb33bb6c..00000000
--- a/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/sh
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_
-# See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html'
-# for best practices.
-# It is recommended to disable memory overcommit,
-# but the Python interpreter may require it on smaller flavors.
-# We therefore stick with the heuristic overcommit setting.
-vm.overcommit_memory=0
-_EOF_
-
-apt-get --allow-unauthenticated -y install libpq-dev postgresql-12 postgresql-server-dev-12 postgresql-client-12
-
-pgsql_conf=/etc/postgresql/12/main/postgresql.conf
-sed -i "/listen_addresses/c listen_addresses = '*'" ${pgsql_conf}
-
-systemctl restart postgresql
-
-# Install the native Python client.
-pip3 install psycopg2
diff --git a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo
deleted file mode 100755
index 7f387b23..00000000
--- a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/bin/sh
-
-set -e
-set -o xtrace
-
-[ -n "${DIB_RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)"
-
-cat <<EOL > /etc/apt/sources.list.d/postgresql.list
-deb http://apt.postgresql.org/pub/repos/apt/ ${DIB_RELEASE}-pgdg main
-EOL
-
-wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
-
-apt-get update
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps b/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps
deleted file mode 100644
index eaa808e1..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-guest
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd b/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd
deleted file mode 100755
index 030efc89..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER
-# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly
-# configured
-
-source $_LIB/die
-
-[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set"
-
-[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
-[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
-[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host"
-[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir"
-[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
-
-sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
-
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc b/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc
deleted file mode 100755
index c36eb31b..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image
-
-dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service
-
-systemctl enable trove-guest.service
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates b/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates
deleted file mode 100755
index b55a0ea2..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/sh
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: configure trove-guest service to use system store of trusted certificates
-
-GUEST_UNIT_DROPINS="/etc/systemd/system/trove-guest.service.d"
-
-mkdir -v -p ${GUEST_UNIT_DROPINS}
-cat <<EOF > ${GUEST_UNIT_DROPINS}/30-use-system-certificates.conf
-[Service]
-Environment=REQUESTS_CA_BUNDLE=/etc/ssl/certs
-EOF
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel b/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel
deleted file mode 100755
index b60a0a95..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-set -e
-set -o xtrace
-
-# The HWE stack must be installed for nested virtualization on ppc64el. This
-# environment variable is set automatically by trovestack, but it may also be
-# set by the user when manually invoking disk-image-create.
-
-case "$DIB_USE_HWE_KERNEL" in
- true|True|TRUE|yes|Yes|YES)
- DIB_USE_HWE_KERNEL=true
- ;;
- *)
- DIB_USE_HWE_KERNEL=false
- ;;
-esac
-
-if [ "$DIB_USE_HWE_KERNEL" == "true" ]; then
- export DEBIAN_FRONTEND=noninteractive
-
- PKG_ARCH=$(dpkg --print-architecture)
-
- case "$PKG_ARCH" in
- amd64|arm64|ppc64el|s390x)
- apt-get --allow-unauthenticated install -y linux-generic-hwe-16.04
- ;;
- esac
-fi
diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs b/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs
deleted file mode 100755
index 125f6c78..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs
+++ /dev/null
@@ -1,90 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install basic services and applications
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-apt-get -y purge acpid\
- apport\
- apport-symptoms\
- apt-transport-https\
- aptitude\
- at\
- bash-completion\
- bc\
- bind9-host\
- bsdmainutils\
- busybox-static\
- byobu\
- command-not-found\
- command-not-found-data\
- curl\
- dbus\
- dmidecode\
- dosfstools\
- ed\
- fonts-ubuntu-font-family-console\
- friendly-recovery\
- ftp\
- fuse\
- geoip-database\
- groff-base\
- hdparm\
- info\
- install-info\
- iptables\
- iputils-tracepath\
- irqbalance\
- language-selector-common\
- libaccountsservice0\
- libevent-2.0-5\
- libgeoip1\
- libnfnetlink0\
- libpcap0.8\
- libpci3\
- libpipeline1\
- libpolkit-gobject-1-0\
- libsasl2-modules\
- libusb-1.0-0\
- lshw\
- lsof\
- ltrace\
- man-db\
- mlocate\
- mtr-tiny\
- nano\
- ntfs-3g\
- parted\
- patch\
- plymouth-theme-ubuntu-text\
- popularity-contest\
- powermgmt-base\
- ppp\
- screen\
- shared-mime-info\
- strace\
- tcpdump\
- telnet\
- time\
- tmux\
- ubuntu-standard\
- ufw\
- update-manager-core\
- update-notifier-common\
- usbutils\
- uuid-runtime\
-
-# The following packages cannot be removed as they cause cloud-init to be
-# uninstalled in Ubuntu 14.04
-# gir1.2-glib-2.0
-# libdbus-glib-1-2
-# libgirepository-1.0-1
-# python-chardet
-# python-serial
-# xz-utils
-
-apt-get -y autoremove
-
diff --git a/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps b/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps
deleted file mode 100644
index b215f584..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-mariadb \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb
deleted file mode 100755
index 6d12202c..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install controller base required packages
-# Refer to https://mariadb.com/kb/en/library/installing-mariadb-deb-files
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-# These GPG key IDs are used to fetch keys from a keyserver on Ubuntu & Debian
-apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8
-curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup |
- bash -s -- --mariadb-server-version="mariadb-10.4" --skip-key-import --skip-maxscale
-
-apt-get install -y -qq apt-transport-https ca-certificates gnupg2
-
-# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
-wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-
-# Disable password prompt
-debconf-set-selections <<< "mariadb-server mysql-server/root_password password ''"
-debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password ''"
-
-apt-get update -qq
-apt-get install -y -qq --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common
-
-cat <<EOF >/etc/mysql/conf.d/no_perf_schema.cnf
-[mysqld]
-performance_schema = off
-EOF
-
-chown mysql:mysql /etc/mysql/my.cnf
-rm -f /etc/init.d/mysql
-
-systemctl daemon-reload
-systemctl enable mariadb \ No newline at end of file
diff --git a/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps b/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps
deleted file mode 100644
index bd3447a6..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-mysql
diff --git a/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql
deleted file mode 100755
index e9f2d8c6..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/bin/bash
-
-# CONTEXT: GUEST during CONSTRUCTION as ROOT
-# PURPOSE: Install controller base required packages
-
-set -e
-set -o xtrace
-
-export DEBIAN_FRONTEND=noninteractive
-
-apt-get --allow-unauthenticated -y install mysql-client mysql-server gnupg2
-
-# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html
-wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-apt-get update
-
-# Xenial provides mysql 5.7 which requires percona-xtrabackup-24
-PXB_VERSION_OVERRIDE=24
-apt-get --allow-unauthenticated -y install percona-xtrabackup-${PXB_VERSION_OVERRIDE}
-
-cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_
-[mysqld]
-performance_schema = off
-show_compatibility_56 = on
-_EOF_
-
-mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf
-chown mysql:mysql /etc/mysql/my.cnf
-cat >/etc/mysql/my.cnf <<_EOF_
-[mysql]
-!includedir /etc/mysql/conf.d/
-_EOF_
-
-if [ -e /etc/init/mysql.conf ]; then
- rm -f /etc/init/mysql.conf
-fi
-
-systemctl enable mysql
diff --git a/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps b/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps
deleted file mode 100644
index 6a0e1b09..00000000
--- a/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps
+++ /dev/null
@@ -1 +0,0 @@
-ubuntu-postgresql \ No newline at end of file
diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu
index f43078d9..783ae9c6 100644
--- a/integration/scripts/functions_qemu
+++ b/integration/scripts/functions_qemu
@@ -3,44 +3,36 @@
# Additional functions that would mostly just pertain to a Ubuntu + Qemu setup
#
-function build_vm() {
- exclaim "Actually building the image, this can take up to 15 minutes"
- rm -rf ~/.cache/image-create
+function build_guest_image() {
+ exclaim "Actually building the image, params: $@"
- local datastore_type=$1
- local guest_os=$2
- local guest_release=$3
- local dev_mode=$4
- local guest_username=$5
- local image_output=$6
+ local guest_os=$1
+ local guest_release=$2
+ local dev_mode=$3
+ local guest_username=$4
+ local image_output=$5
local elementes="base vm"
local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements
- local GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"}
- local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4}
+ local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-3}
local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"}
+ rm -rf ${GUEST_CACHEDIR}
local working_dir=$(dirname ${image_output})
export GUEST_USERNAME=${guest_username}
+ export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
+ export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
+ export DEV_MODE=${dev_mode,,}
# In dev mode, the trove guest agent needs to download trove code from
# trove-taskmanager host during service initialization.
- if [[ "${dev_mode,,}" == "true" ]]; then
- export PATH_TROVE=${PATH_TROVE}
- export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g')
- export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"}
- export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g')
- export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS}
- export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)}
- export HOST_USERNAME=${HOST_SCP_USERNAME}
+ if [[ "${DEV_MODE}" == "true" ]]; then
export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
- export DEST=${DEST:-'/opt/stack'}
- export TROVE_BRANCH=${TROVE_BRANCH:-'master'}
manage_ssh_keys
fi
# For system-wide installs, DIB will automatically find the elements, so we only check local path
- if [ "${DIB_LOCAL_ELEMENTS_PATH}" ]; then
+ if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then
export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH}
else
export ELEMENTS_PATH=${trove_elements_path}
@@ -50,36 +42,26 @@ function build_vm() {
export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive"
# https://cloud-images.ubuntu.com/releases is more stable than the daily
- # builds(https://cloud-images.ubuntu.com/xenial/current/),
- # e.g. sometimes SHA256SUMS file is missing in the daily builds
- declare -A releasemapping=( ["xenial"]="16.04" ["bionic"]="18.04")
+ # builds (https://cloud-images.ubuntu.com/xenial/current/),
+ # e.g. sometimes SHA256SUMS file is missing in the daily builds website.
+ # Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball
+ declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" )
export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/"
- export BASE_IMAGE_FILE="ubuntu-${releasemapping[${DIB_RELEASE}]}-server-cloudimg-amd64-root.tar.gz"
+ export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]}
TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX)
pushd $TEMP > /dev/null
elementes="$elementes ${guest_os}"
-
- if [[ "${dev_mode,,}" == "false" ]]; then
- elementes="$elementes pip-and-virtualenv"
- elementes="$elementes pip-cache"
- elementes="$elementes guest-agent"
- else
- # Install guest agent dependencies, user, etc.
- elementes="$elementes ${guest_os}-guest"
- # Install guest agent service
- elementes="$elementes ${guest_os}-${guest_release}-guest"
- fi
-
- elementes="$elementes ${guest_os}-${datastore_type}"
- elementes="$elementes ${guest_os}-${guest_release}-${datastore_type}"
+ elementes="$elementes pip-and-virtualenv"
+ elementes="$elementes pip-cache"
+ elementes="$elementes guest-agent"
+ elementes="$elementes ${guest_os}-docker"
# Build the image
disk-image-create -x \
-a amd64 \
-o ${image_output} \
- -t ${GUEST_IMAGETYPE} \
--image-size ${GUEST_IMAGESIZE} \
--image-cache ${GUEST_CACHEDIR} \
$elementes
@@ -91,25 +73,6 @@ function build_vm() {
exclaim "Image ${image_output} was built successfully."
}
-function build_guest_image() {
- exclaim "Params for build_guest_image function: $@"
-
- local datastore_type=${1:-"mysql"}
- local guest_os=${2:-"ubuntu"}
- local guest_release=${3:-"xenial"}
- local dev_mode=${4:-"true"}
- local guest_username=${5:-"ubuntu"}
- local output=$6
-
- VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc'
- if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then
- exclaim "You did not pass in a valid datastore type. Valid types are:" $VALID_SERVICES
- exit 1
- fi
-
- build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
-}
-
function clean_instances() {
LIST=`virsh -q list|awk '{print $1}'`
for i in $LIST; do sudo virsh destroy $i; done
@@ -117,6 +80,8 @@ function clean_instances() {
# In dev mode, guest agent needs to ssh into the controller to download code.
function manage_ssh_keys() {
+ SSH_DIR=${SSH_DIR:-"$HOME/.ssh"}
+
if [ -d ${SSH_DIR} ]; then
echo "${SSH_DIR} already exists"
else
diff --git a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc
deleted file mode 100644
index 9b80b6c6..00000000
--- a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc
+++ /dev/null
@@ -1,3 +0,0 @@
-[[post-config|\$CINDER_CONF]]
-[DEFAULT]
-notification_driver = messagingv2
diff --git a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc
deleted file mode 100644
index fe48b022..00000000
--- a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc
+++ /dev/null
@@ -1,3 +0,0 @@
-[[post-config|\$NOVA_CONF]]
-[DEFAULT]
-instance_usage_audit = True
diff --git a/integration/scripts/local.conf.d/ceilometer_services.conf.rc b/integration/scripts/local.conf.d/ceilometer_services.conf.rc
deleted file mode 100644
index ce33948e..00000000
--- a/integration/scripts/local.conf.d/ceilometer_services.conf.rc
+++ /dev/null
@@ -1,3 +0,0 @@
-[[post-config|\$CEILOMETER_CONF]]
-[notification]
-store_events = True
diff --git a/integration/scripts/local.conf.d/sample.rc b/integration/scripts/local.conf.d/sample.rc
deleted file mode 100644
index 401418d8..00000000
--- a/integration/scripts/local.conf.d/sample.rc
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Files in this directory are automatically added to the devstack
-# local.conf file, between a specific set of tags.
-#
-# Filenames must end with '.rc' to be recognized; sample.rc is
-# ignored.
-#
-# A '\' is required in front of any devstack variables since all
-# .rc files are parsed first (using eval).
-#
-# Meta section headings must be included in each file, such as:
-# [[local|localrc]]
-# as the order of inserting the files is not guaranteed.
-#
-# All files are inherently included by default - to exclude a file,
-# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in trovestack.rc
-# For Example: USING_VAGRANT=false (for the using_vagrant.rc file).
-#
-# Symbolic links are followed, so additional files can be loaded
-# by placing them in an external directory and linking it in
-# local.conf.d (this should allow complete flexibility in setting
-# up testing options).
-# For Example:
-# cd /path/to/trove/integration/scripts/local.conf.d
-# ln -s $HOME/local.conf.d local.conf.d
-# cp /path/to/my_conf.rc $HOME/local.conf.d
-
-
-[[local|localrc]]
-# Put regular devstack variables under this meta section heading.
-# This section is written out to a file and sourced by devstack,
-# so it can contain logic as well.
-
-# The following section types should only contain ini file style
-# section headings and name=value pairs
-[[post-config|\$TROVE_CONF]]
-
-[[post-config|\$TROVE_TASKMANAGER_CONF]]
-
-[[post-config|\$TROVE_CONDUCTOR_CONF]]
-
-[[post-config|\$TROVE_API_PASTE_INI]]
diff --git a/integration/scripts/local.conf.d/trove_services.conf.rc b/integration/scripts/local.conf.d/trove_services.conf.rc
deleted file mode 100644
index 6eedc1c3..00000000
--- a/integration/scripts/local.conf.d/trove_services.conf.rc
+++ /dev/null
@@ -1,24 +0,0 @@
-[[post-config|\$TROVE_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_TASKMANAGER_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_CONDUCTOR_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_GUESTAGENT_CONF]]
-[profiler]
-enabled = $ENABLE_PROFILER
-trace_sqlalchemy = $PROFILER_TRACE_SQL
-
-[[post-config|\$TROVE_API_PASTE_INI]]
-[filter:osprofiler]
-enabled = $ENABLE_PROFILER
-hmac_keys = $PROFILER_HMAC_KEYS
diff --git a/integration/scripts/local.conf.d/use_kvm.rc b/integration/scripts/local.conf.d/use_kvm.rc
deleted file mode 100644
index 06bc2ebc..00000000
--- a/integration/scripts/local.conf.d/use_kvm.rc
+++ /dev/null
@@ -1,4 +0,0 @@
-[[local|localrc]]
-
-# force kvm as the libvirt type.
-LIBVIRT_TYPE=kvm
diff --git a/integration/scripts/local.conf.d/use_uuid_token.rc b/integration/scripts/local.conf.d/use_uuid_token.rc
deleted file mode 100644
index 587a4064..00000000
--- a/integration/scripts/local.conf.d/use_uuid_token.rc
+++ /dev/null
@@ -1,3 +0,0 @@
-[[local|localrc]]
-
-KEYSTONE_TOKEN_FORMAT=UUID
diff --git a/integration/scripts/local.conf.d/using_vagrant.rc b/integration/scripts/local.conf.d/using_vagrant.rc
deleted file mode 100644
index 7333cd46..00000000
--- a/integration/scripts/local.conf.d/using_vagrant.rc
+++ /dev/null
@@ -1,9 +0,0 @@
-[[local|localrc]]
-
-# This is similar to code found at
-# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile
-# and seems to make instances ping'able in VirtualBox.
-FLAT_INTERFACE=eth1
-PUBLIC_INTERFACE=eth1
-FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28
-HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4`
diff --git a/integration/scripts/local.conf.rc b/integration/scripts/local.conf.rc
deleted file mode 100644
index b9a2edab..00000000
--- a/integration/scripts/local.conf.rc
+++ /dev/null
@@ -1,37 +0,0 @@
-$TROVE_PRESENT_TAG
-# Set some arguments for devstack.
-#
-# Note: This file contains autogenerated parts.
-# All lines are removed from between the tag/end of tag
-# markers (lines with '$MARKER_TOKEN' at beginning and end) and
-# are replaced by trovestack.
-# Edits to these sections will not persist.
-#
-# See the '$USER_OPTS_TAG' section
-# for ways to insert user args into this file.
-#
-
-#
-# This section is for things that belong in localrc
-# It comes from $DEFAULT_LOCALRC
-#
-[[local|localrc]]
-
-$LOCALRC_OPTS_TAG
-$LOCALRC_OPTS_TAG_END
-
-#
-# User options here were inserted from the file USER_LOCAL_CONF
-# (defaults to $USERHOME/.$LOCAL_CONF)
-#
-
-$USER_OPTS_TAG
-$USER_OPTS_TAG_END
-
-#
-# Additional options here were inserted by trovestack
-# automatically from files in $LOCAL_CONF_D
-#
-
-$ADD_OPTS_TAG
-$ADD_OPTS_TAG_END
diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack
index 4f6273e6..d41b236f 100755
--- a/integration/scripts/trovestack
+++ b/integration/scripts/trovestack
@@ -124,7 +124,7 @@ if is_fedora; then
else
PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive"
PKG_MGR=apt-get
- PKG_GET_ARGS="-y --allow-unauthenticated --force-yes"
+ PKG_GET_ARGS="-y --allow-unauthenticated --force-yes -qq"
fi
PKG_INSTALL_ARG="install"
PKG_UPDATE_ARG="update"
@@ -522,57 +522,15 @@ function set_bin_path() {
function cmd_set_datastore() {
local IMAGEID=$1
- local DATASTORE_TYPE=$2
-
- # rd_manage datastore_update <datastore_name> <default_version>
- rd_manage datastore_update "$DATASTORE_TYPE" ""
- PACKAGES=${PACKAGES:-""}
-
- if [ "$DATASTORE_TYPE" == "mysql" ]; then
- VERSION="5.7"
- elif [ "$DATASTORE_TYPE" == "percona" ]; then
- PACKAGES=${PACKAGES:-"percona-server-server-5.6"}
- VERSION="5.6"
- elif [ "$DATASTORE_TYPE" == "pxc" ]; then
- PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"}
- VERSION="5.6"
- elif [ "$DATASTORE_TYPE" == "mariadb" ]; then
- VERSION="10.4"
- elif [ "$DATASTORE_TYPE" == "mongodb" ]; then
- PACKAGES=${PACKAGES:-"mongodb-org"}
- VERSION="3.2"
- elif [ "$DATASTORE_TYPE" == "redis" ]; then
- PACKAGES=${PACKAGES:-""}
- VERSION="3.2.6"
- elif [ "$DATASTORE_TYPE" == "cassandra" ]; then
- PACKAGES=${PACKAGES:-"cassandra"}
- VERSION="2.1.0"
- elif [ "$DATASTORE_TYPE" == "couchbase" ]; then
- PACKAGES=${PACKAGES:-"couchbase-server"}
- VERSION="2.2.0"
- elif [ "$DATASTORE_TYPE" == "postgresql" ]; then
- VERSION="9.6"
- elif [ "$DATASTORE_TYPE" == "couchdb" ]; then
- PACKAGES=${PACKAGES:-"couchdb"}
- VERSION="1.6.1"
- elif [ "$DATASTORE_TYPE" == "vertica" ]; then
- PACKAGES=${PACKAGES:-"vertica"}
- VERSION="9.0.1"
- elif [ "$DATASTORE_TYPE" == "db2" ]; then
- PACKAGES=${PACKAGES:-""}
- VERSION="11.1"
- else
- echo "Unrecognized datastore type. ($DATASTORE_TYPE)"
- exit 1
- fi
+ rd_manage datastore_update "$datastore" ""
# trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active>
- rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1
- rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION"
+ rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" $IMAGEID "" 1
+ rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"
- if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then
+ if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then
# add the configuration parameters to the database for the kick-start datastore
- rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json
+ rd_manage db_load_datastore_config_parameters "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json
fi
}
@@ -627,8 +585,8 @@ function install_test_packages() {
DATASTORE_TYPE=$1
sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect
- if [ "$DATASTORE_TYPE" = "couchbase" ]; then
- if [ "$DISTRO" == "ubuntu" ]; then
+ if [[ "$DATASTORE_TYPE" = "couchbase" ]]; then
+ if [[ "$DISTRO" == "ubuntu" ]]; then
# Install Couchbase SDK for scenario tests.
sudo -H $HTTP_PROXY curl http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add -
echo "deb http://packages.couchbase.com/ubuntu trusty trusty/main" | sudo tee /etc/apt/sources.list.d/couchbase-csdk.list
@@ -649,12 +607,12 @@ function mod_confs() {
TROVE_REPORT_DIR=${TROVE_REPORT_DIR:=$TROVESTACK_SCRIPTS/../report/}
EXTRA_CONF=$TROVESTACK_SCRIPTS/conf/test.extra.conf
- if [ -e $EXTRA_CONF ]; then
+ if [[ -e $EXTRA_CONF ]]; then
cat $EXTRA_CONF >> $TEST_CONF
fi
# Append datastore specific configuration file
DATASTORE_CONF=$TROVESTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf
- if [ ! -f $DATASTORE_CONF ]; then
+ if [[ ! -f $DATASTORE_CONF ]]; then
exclaim "Datastore configuration file ${DATASTORE_CONF} not found"
exit 1
fi
@@ -695,14 +653,14 @@ function mod_confs() {
sed -i "/%shared_network_subnet%/d" $TEST_CONF
fi
- if [ "$DATASTORE_TYPE" = "vertica" ]; then
+ if [[ "$DATASTORE_TYPE" = "vertica" ]]; then
# Vertica needs more time than mysql for its boot/start/stop operations.
setup_cluster_configs cluster_member_count 3
- elif [ "$DATASTORE_TYPE" = "pxc" ]; then
+ elif [[ "$DATASTORE_TYPE" = "pxc" ]]; then
setup_cluster_configs min_cluster_member_count 2
- elif [ "$DATASTORE_TYPE" = "cassandra" ]; then
+ elif [[ "$DATASTORE_TYPE" = "cassandra" ]]; then
setup_cluster_configs cluster_member_count 2
- elif [ "$DATASTORE_TYPE" = "mongodb" ]; then
+ elif [[ "$DATASTORE_TYPE" = "mongodb" ]]; then
setup_cluster_configs cluster_member_count 2
# Decrease the number of required config servers per cluster to save resources.
iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1
@@ -747,7 +705,7 @@ function cmd_test_init() {
local DATASTORE_TYPE=$1
local DATASTORE_VERSION=$2
- if [ -z "${DATASTORE_TYPE}" ]; then
+ if [[ -z "${DATASTORE_TYPE}" ]]; then
exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
exit 1
fi
@@ -768,73 +726,60 @@ function cmd_test_init() {
pip3 install -U git+https://opendev.org/openstack/python-troveclient@master#egg=python-troveclient
}
+# Build trove guest image
function cmd_build_image() {
exclaim "Params for cmd_build_image function: $@"
- local IMAGE_DATASTORE_TYPE=${1:-'mysql'}
- local IMAGE_GUEST_OS=${2:-'ubuntu'}
- local IMAGE_GUEST_RELEASE=${3:-'xenial'}
- local DEV_MODE=${4:-'true'}
- local guest_username=${5:-'ubuntu'}
- local output=$6
+ local image_guest_os=${1:-'ubuntu'}
+ local image_guest_release=${2:-'bionic'}
+ local dev_mode=${3:-'true'}
+ local guest_username=${4:-'ubuntu'}
+ local output=$5
if [[ -z "$output" ]]; then
- image_name="trove-datastore-${IMAGE_GUEST_OS}-${IMAGE_GUEST_RELEASE}-${IMAGE_DATASTORE_TYPE}"
+ image_name="trove-guest-${image_guest_os}-${image_guest_release}"
+ if [[ ${dev_mode} == "true" ]]; then
+ image_name="${image_name}-dev"
+ fi
image_folder=$HOME/images
- output="${image_folder}/${image_name}"
+ output="${image_folder}/${image_name}.qcow2"
fi
+
# Always rebuild the image.
- sudo rm -f $output
- sudo mkdir -p $(dirname $output); sudo chmod 777 -R $(dirname $output)
+ sudo rm -rf ${output}
+ sudo mkdir -p $(dirname ${output}); sudo chmod 777 -R $(dirname ${output})
echo "Ensuring we have all packages needed to build image."
sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update
- sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap
+ sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap squashfs-tools
sudo -H $HTTP_PROXY pip install diskimage-builder
- exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image."
- build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE ${guest_username} $output
+ build_guest_image ${image_guest_os} ${image_guest_release} ${dev_mode} ${guest_username} ${output}
}
# Build guest image and upload to Glance, register the datastore and configuration parameters.
-# We could skip the image build and upload by:
-# 1. MYSQL_IMAGE_ID is passed, or
-# 2. There is an image in Glance contains the datastore name
function cmd_build_and_upload_image() {
- local datastore_type=$1
- local guest_os=${2:-"ubuntu"}
- local guest_release=${3:-"xenial"}
- local dev_mode=${4:-"true"}
- local guest_username=${5:-"ubuntu"}
- local output_dir=${6:-"$HOME/images"}
-
- if [ -z "${datastore_type}" ]; then
- exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
- exit 1
- fi
-
- image_var="${datastore_type^^}_IMAGE_ID"
- glance_imageid=`eval echo '$'"$image_var"`
-
- if [[ -z $glance_imageid ]]; then
- # Find the first image id with the name contains datastore_type.
- glance_imageid=$(openstack $CLOUD_ADMIN_ARG image list | grep "$datastore_type" | awk 'NR==1 {print}' | awk '{print $2}')
-
- if [[ -z $glance_imageid ]]; then
- mkdir -p ${output_dir}
- name=trove-datastore-${guest_os}-${guest_release}-${datastore_type}
- output=${output_dir}/$name.qcow2
- cmd_build_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} $output
-
- glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create $name --public --disk-format qcow2 --container-format bare --file $output --property hw_rng_model='virtio' -c id -f value)
- [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1
- fi
+ local guest_os=${1:-"ubuntu"}
+ local guest_release=${2:-"bionic"}
+ local dev_mode=${3:-"true"}
+ local guest_username=${4:-"ubuntu"}
+ local output_dir=${5:-"$HOME/images"}
+
+ name=trove-guest-${guest_os}-${guest_release}
+ glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image list --name $name -f value -c ID)
+ if [[ -z ${glance_imageid} ]]; then
+ mkdir -p ${output_dir}
+ output=${output_dir}/${name}
+ cmd_build_image ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output}
+
+ glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create ${name} --public --disk-format qcow2 --container-format bare --file ${output} --property hw_rng_model='virtio' --tag trove -c id -f value)
+ [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1
fi
exclaim "Using Glance image ID: $glance_imageid"
exclaim "Updating Datastores"
- cmd_set_datastore "${glance_imageid}" "${datastore_type}"
+ cmd_set_datastore "${glance_imageid}"
}
@@ -991,11 +936,11 @@ function cmd_stop() {
function cmd_int_tests() {
exclaim "Running Trove Integration Tests..."
- if [ ! $USAGE_ENDPOINT ]; then
+ if [[ ! $USAGE_ENDPOINT ]]; then
export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier
fi
cd $TROVESTACK_SCRIPTS
- if [ $# -lt 1 ]; then
+ if [[ $# -lt 1 ]]; then
args="--group=mysql"
else
args="$@"
@@ -1203,7 +1148,7 @@ function cmd_kick_start() {
local DATASTORE_TYPE=$1
local DATASTORE_VERSION=$2
- if [ -z "${DATASTORE_TYPE}" ]; then
+ if [[ -z "${DATASTORE_TYPE}" ]]; then
exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}"
exit 1
fi
@@ -1220,10 +1165,13 @@ function cmd_kick_start() {
function cmd_gate_tests() {
local DATASTORE_TYPE=${1:-'mysql'}
local TEST_GROUP=${2:-${DATASTORE_TYPE}}
- local DATASTORE_VERSION=${3:-'5.7'}
+ local DATASTORE_VERSION=${3:-'5.7.29'}
local HOST_SCP_USERNAME=${4:-$(whoami)}
local GUEST_USERNAME=${5:-'ubuntu'}
+ export DATASTORE_TYPE=${DATASTORE_TYPE}
+ export DATASTORE_VERSION=${DATASTORE_VERSION}
+
exclaim "Running cmd_gate_tests ..."
export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/}
@@ -1238,7 +1186,7 @@ function cmd_gate_tests() {
cd $TROVESTACK_SCRIPTS
# Build and upload guest image, register datastore version.
- cmd_build_and_upload_image ${DATASTORE_TYPE}
+ cmd_build_and_upload_image
cmd_kick_start "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"