diff options
author | Lingxian Kong <anlin.kong@gmail.com> | 2020-04-07 10:52:16 +1200 |
---|---|---|
committer | Lingxian Kong <anlin.kong@gmail.com> | 2020-05-27 10:31:50 +1200 |
commit | aa1d4d224674f44d9cd882eddb2537907adf5382 (patch) | |
tree | b9baf26bfb4b15497696d020fea41364bd0576c8 /integration/scripts | |
parent | 523d66e8fd5d29be8dbae9aa79c5348d3dce8c64 (diff) | |
download | trove-aa1d4d224674f44d9cd882eddb2537907adf5382.tar.gz |
Datastore containerization
Significant changes:
* Using docker image to install datastore.
* Datastore image is common to different datastores.
* Using backup docker image to do backup and restore.
* Support MariaDB replication
* Set most of the functional jobs as non-voting as nested
virtualization is not supported in CI.
Change-Id: Ia9c97a63a961eebc336b70d28dc77638144c1834
Diffstat (limited to 'integration/scripts')
59 files changed, 242 insertions, 1012 deletions
diff --git a/integration/scripts/conf.json.example b/integration/scripts/conf.json.example deleted file mode 100644 index ee6bc7bf..00000000 --- a/integration/scripts/conf.json.example +++ /dev/null @@ -1,12 +0,0 @@ -{ - "devstack":null, - "glance":null, - "horizon":null, - "keystone":null, - "nova":null, - "python_openstackclient":null, - "python_novaclient":null, - "trove":null, - "python_troveclient":null, - "tempest":null -} diff --git a/integration/scripts/files/elements/apt-conf-dir/README.rst b/integration/scripts/files/elements/apt-conf-dir/README.rst deleted file mode 100644 index c94e00ea..00000000 --- a/integration/scripts/files/elements/apt-conf-dir/README.rst +++ /dev/null @@ -1,16 +0,0 @@ -============ -apt-conf-dir -============ - -This element overrides the default apt.conf.d directory for APT based systems. - -Environment Variables ---------------------- - -DIB_APT_CONF_DIR: - :Required: No - :Default: None - :Description: To override `DIB_APT_CONF_DIR`, set it to the path to your - apt.conf.d. The new apt.conf.d will take effect at build time - and run time. - :Example: ``DIB_APT_CONF_DIR=/etc/apt/apt.conf`` diff --git a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd b/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd deleted file mode 100755 index e286d684..00000000 --- a/integration/scripts/files/elements/apt-conf-dir/extra-data.d/99-use-host-apt-confd +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Override the default /etc/apt/apt.conf.d directory with $DIB_APT_CONF_DIR - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -# exit directly if DIB_APT_CONF_DIR is not defined properly -if [ -z "${DIB_APT_CONF_DIR:-}" ] ; then - echo "DIB_APT_CONF_DIR is not set - no apt.conf.d will be copied in" - exit 0 -elif [ ! -d "$DIB_APT_CONF_DIR" ] ; then - echo "$DIB_APT_CONF_DIR is not a valid apt.conf.d directory." - echo "You should assign a proper apt.conf.d directory in DIB_APT_CONF_DIR" - exit 1 -fi - -# copy the apt.conf to cloudimg -sudo cp -L -f -R $DIB_APT_CONF_DIR $TMP_MOUNT_PATH/etc/apt diff --git a/integration/scripts/files/elements/guest-agent/element-deps b/integration/scripts/files/elements/guest-agent/element-deps index 6dcd66dc..ef309837 100644 --- a/integration/scripts/files/elements/guest-agent/element-deps +++ b/integration/scripts/files/elements/guest-agent/element-deps @@ -4,3 +4,4 @@ pkg-map source-repositories svc-map pip-and-virtualenv +ubuntu-docker diff --git a/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash b/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash deleted file mode 100644 index 9622b292..00000000 --- a/integration/scripts/files/elements/guest-agent/environment.d/99-reliable-apt-key-importing.bash +++ /dev/null @@ -1,34 +0,0 @@ -# sometimes the primary key server is unavailable and we should try an -# alternate. see -# https://bugs.launchpad.net/percona-server/+bug/907789. Disable -# shell errexit so we can interrogate the exit code and take action -# based on the exit code. We will reenable it later. -# -# NOTE(zhaochao): we still have this problem from time to time, so it's -# better use more reliable keyservers and just retry on that(for now, 3 -# tries should be fine). -# According to: -# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver -# [2] https://sks-keyservers.net/overview-of-pools.php -# we'll just the primary suggested pool: pool.sks-keyservers.net. -function get_key_robust() { - KEY=$1 - set +e - - tries=1 - while [ $tries -le 3 ]; do - if [ $tries -eq 3 ]; then - set -e - fi - - echo "Importing the key, try: $tries" - # Behind a firewall should use the port 80 instead of the default port 11371 - apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break - - tries=$((tries+1)) - done - - set -e -} - -export -f get_key_robust diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install new file mode 100755 index 00000000..1eb66ffd --- /dev/null +++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/31-guest-agent-install @@ -0,0 +1,51 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +DEV_MODE=${DEV_MODE:-"true"} +SCRIPTDIR=$(dirname $0) +GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"} +GUEST_VENV=/opt/guest-agent-venv + +for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove" "/opt/trove-guestagent"; do + mkdir -p ${folder} + chown -R ${GUEST_USERNAME}:root ${folder} +done + +install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent + +# Create a virtual environment (with dependencies installed) for guest agent service +${DIB_PYTHON} -m virtualenv ${GUEST_VENV} +${GUEST_VENV}/bin/pip install pip --upgrade +${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent +chown -R ${GUEST_USERNAME}:root ${GUEST_VENV} + +if [[ ${DEV_MODE} == "true" ]]; then + [[ -n "${HOST_SCP_USERNAME}" ]] || die "HOST_SCP_USERNAME needs to be set to the trovestack host user" + [[ -n "${ESCAPED_PATH_TROVE}" ]] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host" + + sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${SCRIPTDIR}/guest-agent-dev.service > /etc/systemd/system/guest-agent.service +else + # Link the trove-guestagent out to /usr/local/bin where the startup scripts look for + ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true + + case "$DIB_INIT_SYSTEM" in + systemd) + sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /etc/systemd/system/guest-agent.service + ;; + upstart) + install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf + ;; + sysv) + install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init + ;; + *) + echo "Unsupported init system" + exit 1 + ;; + esac +fi diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install deleted file mode 100755 index 87a11958..00000000 --- a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/75-guest-agent-install +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash - -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then - set -x -fi -set -eu -set -o pipefail - -SCRIPTDIR=$(dirname $0) -GUEST_VENV=/opt/guest-agent-venv -GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"} - -# Create a virtual environment for guest agent -${DIB_PYTHON} -m virtualenv ${GUEST_VENV} -${GUEST_VENV}/bin/pip install pip --upgrade -${GUEST_VENV}/bin/pip install -U -c /opt/upper-constraints.txt /opt/guest-agent -chown -R ${GUEST_USERNAME}:root ${GUEST_VENV} - -# Link the trove-guestagent out to /usr/local/bin where the startup scripts look for -ln -s ${GUEST_VENV}/bin/trove-guestagent /usr/local/bin/guest-agent || true - -for folder in "/var/lib/trove" "/etc/trove" "/etc/trove/certs" "/etc/trove/conf.d" "/var/log/trove"; do - mkdir -p ${folder} - chown -R ${GUEST_USERNAME}:root ${folder} -done - -install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.logrotate /etc/logrotate.d/guest-agent - -case "$DIB_INIT_SYSTEM" in - systemd) - mkdir -p /usr/lib/systemd/system - touch /usr/lib/systemd/system/guest-agent.service - sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g" ${SCRIPTDIR}/guest-agent.service > /usr/lib/systemd/system/guest-agent.service - ;; - upstart) - install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.conf /etc/init/guest-agent.conf - ;; - sysv) - install -D -g root -o ${GUEST_USERNAME} -m 0644 ${SCRIPTDIR}/guest-agent.init /etc/init.d/guest-agent.init - ;; - *) - echo "Unsupported init system" - exit 1 - ;; -esac diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service new file mode 100644 index 00000000..261ac811 --- /dev/null +++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent-dev.service @@ -0,0 +1,31 @@ +[Unit] +Description=OpenStack Trove Guest Agent Service for Development +After=syslog.target network.target + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +User=GUEST_USERNAME +Group=GUEST_USERNAME + +# This script is only for testing purpose for dev_mode=true, the controller +# IP address should be defined in /etc/trove/controller.conf, e.g. +# CONTROLLER=192.168.32.151 +EnvironmentFile=/etc/trove/controller.conf + +# If ~/trove-installed does not exist, copy the trove source from +# the user's development environment, then touch the sentinel file +ExecStartPre=/bin/bash -c "test -e /home/GUEST_USERNAME/trove-installed || sudo rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i /home/GUEST_USERNAME/.ssh/id_rsa' -az --exclude='.*' HOST_SCP_USERNAME@$CONTROLLER:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed" + +ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/ /home/GUEST_USERNAME/trove" + +# Take care of the changes in requirements.txt +ExecStartPre=/bin/bash -c "sudo /opt/guest-agent-venv/bin/pip install -r /home/GUEST_USERNAME/trove/requirements.txt -c /opt/upper-constraints.txt" + +# Start guest-agent.service in virtual environment +ExecStart=/bin/bash -c "/opt/guest-agent-venv/bin/python /home/GUEST_USERNAME/trove/contrib/trove-guestagent --config-dir=/etc/trove/conf.d" + +TimeoutSec=300 +Restart=on-failure
\ No newline at end of file diff --git a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service index 788bebc9..a468de54 100644 --- a/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service +++ b/integration/scripts/files/elements/guest-agent/install.d/guest-agent-source-install/guest-agent.service @@ -1,15 +1,16 @@ [Unit] -Description=OpenStack Trove Guest Agent +Description=OpenStack Trove Guest Agent Service After=network.target syslog.service Wants=syslog.service [Service] User=GUEST_USERNAME Group=GUEST_USERNAME -ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove/conf.d" -ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d KillMode=mixed Restart=always +ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove /var/log/trove/" +ExecStart=/usr/local/bin/guest-agent --config-dir=/etc/trove/conf.d + [Install] WantedBy=multi-user.target diff --git a/integration/scripts/files/elements/guest-agent/package-installs.yaml b/integration/scripts/files/elements/guest-agent/package-installs.yaml index 582d48bc..37e7daa4 100644 --- a/integration/scripts/files/elements/guest-agent/package-installs.yaml +++ b/integration/scripts/files/elements/guest-agent/package-installs.yaml @@ -1,53 +1,15 @@ guest-agent: installtype: package build-essential: - installtype: source +python3-all: +python3-all-dev: +python3-pip: +python3-sqlalchemy: +libxml2-dev: +libxslt1-dev: libffi-dev: - installtype: source libssl-dev: - installtype: source -python-dev: - installtype: source - -acl: -acpid: -apparmor: -apparmor-utils: -apt-transport-https: -at: -bash-completion: -cloud-guest-utils: -cloud-init: -cron: -curl: -dbus: -dkms: -dmeventd: -ethtool: -gawk: -ifenslave: -ifupdown: -iptables: -iputils-tracepath: -irqbalance: -isc-dhcp-client: -less: -logrotate: -lsof: -net-tools: -netbase: -netcat-openbsd: -open-vm-tools: - arch: i386, amd64 +libyaml-dev: openssh-client: openssh-server: -pollinate: -psmisc: -rsyslog: -socat: -tcpdump: -ubuntu-cloudimage-keyring: -ureadahead: -uuid-runtime: -vim-tiny: -vlan: +rsync: diff --git a/integration/scripts/files/elements/guest-agent/post-install.d/11-enable-guest-agent-systemd b/integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd index 1f1eb7a4..1f1eb7a4 100755 --- a/integration/scripts/files/elements/guest-agent/post-install.d/11-enable-guest-agent-systemd +++ b/integration/scripts/files/elements/guest-agent/post-install.d/31-enable-guest-agent-systemd diff --git a/integration/scripts/files/elements/no-resolvconf/README.rst b/integration/scripts/files/elements/no-resolvconf/README.rst deleted file mode 100644 index 8a3dfc7d..00000000 --- a/integration/scripts/files/elements/no-resolvconf/README.rst +++ /dev/null @@ -1,8 +0,0 @@ -This element clears out /etc/resolv.conf and prevents dhclient from populating -it with data from DHCP. This means that DNS resolution will not work from the -guest. This is OK because all outbound connections from the guest will -be based using raw IP addresses. - -In addition we remove dns from the nsswitch.conf hosts setting. - -This means that the guest never waits for DNS timeouts to occur. diff --git a/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf b/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf deleted file mode 100755 index 5bf5fded..00000000 --- a/integration/scripts/files/elements/no-resolvconf/finalise.d/99-disable-resolv-conf +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash -echo "" > /etc/resolv.conf -echo "" > /etc/resolv.conf.ORIG -if [ -d /etc/dhcp/dhclient-enter-hooks.d ]; then - # Debian/Ubuntu - echo "#!/bin/sh -make_resolv_conf() { : ; }" > /etc/dhcp/dhclient-enter-hooks.d/noresolvconf - chmod +x /etc/dhcp/dhclient-enter-hooks.d/noresolvconf - rm -f /etc/dhcp/dhclient-enter-hooks.d/resolvconf -else - # RHEL/CentOS/Fedora - echo "#!/bin/sh -make_resolv_conf() { : ; }" > /etc/dhclient-enter-hooks - chmod +x /etc/dhclient-enter-hooks -fi - -if [ -e /etc/nsswitch.conf ]; then - sed -i -e "/hosts:/ s/dns//g" /etc/nsswitch.conf -fi diff --git a/integration/scripts/files/elements/ubuntu-docker/element-deps b/integration/scripts/files/elements/ubuntu-docker/element-deps new file mode 100644 index 00000000..10e54583 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-docker/element-deps @@ -0,0 +1 @@ +ubuntu-guest
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker new file mode 100755 index 00000000..44041384 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-docker/install.d/21-docker @@ -0,0 +1,19 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +GUEST_USERNAME=${GUEST_USERNAME:-"ubuntu"} + +echo "Installing docker" +export DEBIAN_FRONTEND=noninteractive +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu ${DIB_RELEASE} stable" +apt-get update +apt-get install -y -qq docker-ce >/dev/null + +echo "Adding ${GUEST_USERNAME} user to docker group" +usermod -aG docker ${GUEST_USERNAME} diff --git a/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash b/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash deleted file mode 100644 index 9622b292..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/environment.d/99-reliable-apt-key-importing.bash +++ /dev/null @@ -1,34 +0,0 @@ -# sometimes the primary key server is unavailable and we should try an -# alternate. see -# https://bugs.launchpad.net/percona-server/+bug/907789. Disable -# shell errexit so we can interrogate the exit code and take action -# based on the exit code. We will reenable it later. -# -# NOTE(zhaochao): we still have this problem from time to time, so it's -# better use more reliable keyservers and just retry on that(for now, 3 -# tries should be fine). -# According to: -# [1] https://www.gnupg.org/faq/gnupg-faq.html#new_user_default_keyserver -# [2] https://sks-keyservers.net/overview-of-pools.php -# we'll just the primary suggested pool: pool.sks-keyservers.net. -function get_key_robust() { - KEY=$1 - set +e - - tries=1 - while [ $tries -le 3 ]; do - if [ $tries -eq 3 ]; then - set -e - fi - - echo "Importing the key, try: $tries" - # Behind a firewall should use the port 80 instead of the default port 11371 - apt-key adv --keyserver hkp://pool.sks-keyservers.net:80 --recv-keys ${KEY} && break - - tries=$((tries+1)) - done - - set -e -} - -export -f get_key_robust diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev new file mode 100755 index 00000000..498169c3 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/11-ssh-key-dev @@ -0,0 +1,17 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +source $_LIB/die + +[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" + +# Guest agent needs to ssh into the controller to download code in dev mode. +if [[ ${DEV_MODE} == "true" && -e ${SSH_DIR}/id_rsa ]]; then + sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa + sudo -Hiu ${HOST_SCP_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub +fi diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep deleted file mode 100755 index 5a0c1ecd..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/15-trove-dep +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER -# PURPOSE: Setup the requirements file for use by 15-reddwarf-dep - -source $_LIB/die - -TROVE_BRANCH=${TROVE_BRANCH:-'master'} -REQUIREMENTS_FILE=${TROVESTACK_SCRIPTS}/../../requirements.txt - -[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" -[ -e ${REQUIREMENTS_FILE} ] || die "Requirements not found" -[ -n "$HOST_USERNAME" ] || die "HOST_USERNAME not set" - -sudo -Hiu ${HOST_USERNAME} dd if=${REQUIREMENTS_FILE} of=${TMP_HOOKS_PATH}/requirements.txt - -UC_FILE=upper-constraints.txt -UC_DIR=$(pwd) -UC_BRANCH=${TROVE_BRANCH##stable/} - -curl -L -o "${UC_DIR}/${UC_FILE}" "https://releases.openstack.org/constraints/upper/${UC_BRANCH}" -if [ -f "${UC_DIR}/${UC_FILE}" ]; then - sudo -Hiu ${HOST_USERNAME} dd if="${UC_DIR}/${UC_FILE}" of=${TMP_HOOKS_PATH}/${UC_FILE} - rm -f "${UC_DIR}/${UC_FILE}" -fi diff --git a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key deleted file mode 100755 index 894950db..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/extra-data.d/62-ssh-key +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER -# PURPOSE: creates the SSH key on the host if it doesn't exist. Then this copies the keys over to a staging area where -# they will be duplicated in the guest VM. -# This process allows the host to log into the guest but more importantly the guest phones home to get the trove -# source - -source $_LIB/die - -[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" -[ -n "${HOST_USERNAME}" ] || die "HOST_USERNAME needs to be set to the user for the current user on the host" - -if [ `whoami` = "root" ]; then - die "This should not be run as root" -fi - -# Guest agent needs to ssh into the controller to download code in dev mode. -if [ -e ${SSH_DIR}/id_rsa ]; then - sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa of=${TMP_HOOKS_PATH}/id_rsa - sudo -Hiu ${HOST_USERNAME} dd if=${SSH_DIR}/id_rsa.pub of=${TMP_HOOKS_PATH}/id_rsa.pub -else - die "SSH keys must exist" -fi diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps b/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps deleted file mode 100755 index 3a8cacfb..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/05-base-apps +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install basic services and applications - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive -apt-get --allow-unauthenticated -y install ntp apparmor-utils diff --git a/integration/scripts/files/elements/guest-agent/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/11-user index 8a2b145f..456073f7 100755 --- a/integration/scripts/files/elements/guest-agent/install.d/50-user +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/11-user @@ -19,4 +19,4 @@ if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then ${GUEST_USERNAME} ${GUEST_USERNAME} _EOF_ -fi +fi
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev b/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev new file mode 100755 index 00000000..8be7882a --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/install.d/12-ssh-key-dev @@ -0,0 +1,22 @@ +#!/bin/bash + +if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +GUEST_SSH_DIR="/home/${GUEST_USERNAME}/.ssh" +TMP_HOOKS_DIR="/tmp/in_target.d" + +if [ ! -e ${GUEST_SSH_DIR} ]; then + # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME} + mkdir ${GUEST_SSH_DIR} + chown -R ${GUEST_USERNAME}:${GUEST_USERNAME} ${GUEST_SSH_DIR} +fi + +if [[ ${DEV_MODE} == "true" && -e "${TMP_HOOKS_DIR}/id_rsa" ]]; then + sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub + sudo -Hiu ${GUEST_USERNAME} dd of=${GUEST_SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa + sudo -Hiu ${GUEST_USERNAME} chmod 600 ${GUEST_SSH_DIR}/id_rsa +fi diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep b/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep deleted file mode 100755 index 25b48459..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/15-trove-dep +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install trove guest python dependencies - see trovestack functions_qemu - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive -apt-get --allow-unauthenticated -y install \ - libxml2-dev libxslt1-dev libffi-dev libssl-dev libyaml-dev \ - python3-pip python3-sqlalchemy python3-setuptools - -# Install python 3.7, some python lib (e.g. oslo.concurrency>4.0.0) requries -# Python 3.7 -add-apt-repository --yes ppa:deadsnakes/ppa -apt update -apt install -y python3.7 python3.7-dev - -update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.7 10 -python3.5 -m pip install pip==9.0.3 -python3.5 -m pip install -U wheel setuptools - -TMP_HOOKS_DIR="/tmp/in_target.d" - -UPPER_CONSTRAINTS= -if [ -f ${TMP_HOOKS_DIR}/upper-constraints.txt ]; then - UPPER_CONSTRAINTS=" -c ${TMP_HOOKS_DIR}/upper-constraints.txt" -fi - -python3.7 -m pip install pip==9.0.3 -python3.7 -m pip install -U wheel setuptools -python3.7 -m pip install --upgrade -r ${TMP_HOOKS_DIR}/requirements.txt ${UPPER_CONSTRAINTS} - -echo "diagnostic pip freeze output follows" -python3.7 -m pip freeze -echo "diagnostic pip freeze output above" diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user b/integration/scripts/files/elements/ubuntu-guest/install.d/50-user deleted file mode 100755 index 99f68966..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/50-user +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Add the guest image user that will own the trove agent source...if the user does not already exist - -set -e -set -o xtrace - -if ! id -u ${GUEST_USERNAME} >/dev/null 2>&1; then - echo "Adding ${GUEST_USERNAME} user" - useradd -G sudo -m ${GUEST_USERNAME} -s /bin/bash - chown ${GUEST_USERNAME}:${GUEST_USERNAME} /home/${GUEST_USERNAME} - passwd ${GUEST_USERNAME} <<_EOF_ -${GUEST_USERNAME} -${GUEST_USERNAME} -_EOF_ -fi - diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key b/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key deleted file mode 100755 index 21a54d28..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/62-ssh-key +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -# PURPOSE: take "staged" ssh keys (see extra-data.d/62-ssh-key) and put them in the GUEST_USERS home directory -# In future, this should be removed and use Nova keypair to inject ssh keys. - -set -e -set -o xtrace - -SSH_DIR="/home/${GUEST_USERNAME}/.ssh" -TMP_HOOKS_DIR="/tmp/in_target.d" - -if [ ! -e ${SSH_DIR} ]; then - # this method worked more reliable in vmware fusion over doing sudo -Hiu ${GUEST_USERNAME} - mkdir ${SSH_DIR} - chown ${GUEST_USERNAME}:${GUEST_USERNAME} ${SSH_DIR} -fi - -if [ -e "${TMP_HOOKS_DIR}/id_rsa" ]; then - sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa.pub if=${TMP_HOOKS_DIR}/id_rsa.pub - sudo -Hiu ${GUEST_USERNAME} dd of=${SSH_DIR}/id_rsa if=${TMP_HOOKS_DIR}/id_rsa - sudo -Hiu ${GUEST_USERNAME} chmod 600 ${SSH_DIR}/id_rsa -else - echo "SSH Keys were not staged by host" - exit -1 -fi diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh b/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh deleted file mode 100755 index 2134c198..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/98-ssh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Regenerate host keys now. XXX: Really should be a cloud-init task, should get -# that working. - -set -e -set -o xtrace - -dpkg-reconfigure openssh-server diff --git a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt b/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt deleted file mode 100755 index cc348c5c..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/install.d/99-clean-apt +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Delete contents of apt cache on guest (saves image disk space) - -set -e -set -o xtrace - -apt-get clean - - diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding b/integration/scripts/files/elements/ubuntu-guest/post-install.d/11-ipforwarding index 48b7ad0d..48b7ad0d 100755 --- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/05-ipforwarding +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/11-ipforwarding diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp b/integration/scripts/files/elements/ubuntu-guest/post-install.d/12-ntp index bc494ae3..bc494ae3 100755 --- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/10-ntp +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/12-ntp diff --git a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers b/integration/scripts/files/elements/ubuntu-guest/post-install.d/13-trove-guest-sudoers index 0581fd2b..0581fd2b 100755 --- a/integration/scripts/files/elements/ubuntu-guest/post-install.d/62-trove-guest-sudoers +++ b/integration/scripts/files/elements/ubuntu-guest/post-install.d/13-trove-guest-sudoers diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools deleted file mode 100755 index 7b9221cf..00000000 --- a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/04-baseline-tools +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -# Install baseline packages and tools. - -set -e -set -o xtrace - -apt-get --allow-unauthenticated install -y language-pack-en python-software-properties software-properties-common
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools new file mode 100755 index 00000000..12c864d6 --- /dev/null +++ b/integration/scripts/files/elements/ubuntu-guest/pre-install.d/11-baseline-tools @@ -0,0 +1,7 @@ +#!/bin/bash +# Install baseline packages and tools. + +set -e +set -o xtrace + +DEBIAN_FRONTEND=noninteractive apt-get --allow-unauthenticated install -y -qq software-properties-common apt-transport-https ca-certificates ntp >/dev/null
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-mariadb/README.md b/integration/scripts/files/elements/ubuntu-mariadb/README.md deleted file mode 100644 index 757f00b8..00000000 --- a/integration/scripts/files/elements/ubuntu-mariadb/README.md +++ /dev/null @@ -1,3 +0,0 @@ -Sets up a MariaDB server install in the image. - -TODO: auto-tune settings based on host resources or metadata service. diff --git a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local deleted file mode 100755 index a3e1dc7c..00000000 --- a/integration/scripts/files/elements/ubuntu-mariadb/pre-install.d/20-apparmor-mysql-local +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -#CONTEXT: chroot on host -#PURPOSE: Allows mysqld to create temporary files when restoring backups - -cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld - /tmp/ rw, - /tmp/** rwk, -EOF diff --git a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local b/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local deleted file mode 100755 index 90bd85b1..00000000 --- a/integration/scripts/files/elements/ubuntu-mysql/pre-install.d/20-apparmor-mysql-local +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -set -e - -#CONTEXT: chroot on host -#PURPOSE: Allows mysqld to create temporary files when restoring backups - -mkdir -p /etc/apparmor.d/local/ -cat <<EOF >>/etc/apparmor.d/local/usr.sbin.mysqld - /tmp/ rw, - /tmp/** rwk, -EOF diff --git a/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql b/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql deleted file mode 100755 index cb33bb6c..00000000 --- a/integration/scripts/files/elements/ubuntu-postgresql/install.d/30-postgresql +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -cat > "/etc/sysctl.d/10-postgresql-performance.conf" << _EOF_ -# See 'http://www.postgresql.org/docs/9.6/static/kernel-resources.html' -# for best practices. -# It is recommended to disable memory overcommit, -# but the Python interpreter may require it on smaller flavors. -# We therefore stick with the heuristic overcommit setting. -vm.overcommit_memory=0 -_EOF_ - -apt-get --allow-unauthenticated -y install libpq-dev postgresql-12 postgresql-server-dev-12 postgresql-client-12 - -pgsql_conf=/etc/postgresql/12/main/postgresql.conf -sed -i "/listen_addresses/c listen_addresses = '*'" ${pgsql_conf} - -systemctl restart postgresql - -# Install the native Python client. -pip3 install psycopg2 diff --git a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo b/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo deleted file mode 100755 index 7f387b23..00000000 --- a/integration/scripts/files/elements/ubuntu-postgresql/pre-install.d/10-postgresql-repo +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e -set -o xtrace - -[ -n "${DIB_RELEASE}" ] || die "RELEASE must be set to a valid Ubuntu release (e.g. trusty)" - -cat <<EOL > /etc/apt/sources.list.d/postgresql.list -deb http://apt.postgresql.org/pub/repos/apt/ ${DIB_RELEASE}-pgdg main -EOL - -wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - -apt-get update diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps b/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps deleted file mode 100644 index eaa808e1..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-guest diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd b/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd deleted file mode 100755 index 030efc89..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/extra-data.d/20-guest-systemd +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# CONTEXT: HOST prior to IMAGE BUILD as SCRIPT USER -# PURPOSE: stages the bootstrap file and upstart conf file while replacing variables so that guest image is properly -# configured - -source $_LIB/die - -[ -n "$TMP_HOOKS_PATH" ] || die "Temp hook path not set" - -[ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image" -[ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance" -[ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the trovestack host" -[ -n "${TROVESTACK_SCRIPTS}" ] || die "TROVESTACK_SCRIPTS needs to be set to the trove/integration/scripts dir" -[ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir" - -sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${TROVESTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service - diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc b/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc deleted file mode 100755 index c36eb31b..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/20-etc +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: take "staged" trove-guest.conf file and put it in the init directory on guest image - -dd if=/tmp/in_target.d/trove-guest.service of=/etc/systemd/system/trove-guest.service - -systemctl enable trove-guest.service diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates b/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates deleted file mode 100755 index b55a0ea2..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/install.d/21-use-ubuntu-certificates +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: configure trove-guest service to use system store of trusted certificates - -GUEST_UNIT_DROPINS="/etc/systemd/system/trove-guest.service.d" - -mkdir -v -p ${GUEST_UNIT_DROPINS} -cat <<EOF > ${GUEST_UNIT_DROPINS}/30-use-system-certificates.conf -[Service] -Environment=REQUESTS_CA_BUNDLE=/etc/ssl/certs -EOF diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel b/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel deleted file mode 100755 index b60a0a95..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/post-install.d/91-hwe-kernel +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -set -e -set -o xtrace - -# The HWE stack must be installed for nested virtualization on ppc64el. This -# environment variable is set automatically by trovestack, but it may also be -# set by the user when manually invoking disk-image-create. - -case "$DIB_USE_HWE_KERNEL" in - true|True|TRUE|yes|Yes|YES) - DIB_USE_HWE_KERNEL=true - ;; - *) - DIB_USE_HWE_KERNEL=false - ;; -esac - -if [ "$DIB_USE_HWE_KERNEL" == "true" ]; then - export DEBIAN_FRONTEND=noninteractive - - PKG_ARCH=$(dpkg --print-architecture) - - case "$PKG_ARCH" in - amd64|arm64|ppc64el|s390x) - apt-get --allow-unauthenticated install -y linux-generic-hwe-16.04 - ;; - esac -fi diff --git a/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs b/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs deleted file mode 100755 index 125f6c78..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-guest/pre-install.d/01-trim-pkgs +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install basic services and applications - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive -apt-get -y purge acpid\ - apport\ - apport-symptoms\ - apt-transport-https\ - aptitude\ - at\ - bash-completion\ - bc\ - bind9-host\ - bsdmainutils\ - busybox-static\ - byobu\ - command-not-found\ - command-not-found-data\ - curl\ - dbus\ - dmidecode\ - dosfstools\ - ed\ - fonts-ubuntu-font-family-console\ - friendly-recovery\ - ftp\ - fuse\ - geoip-database\ - groff-base\ - hdparm\ - info\ - install-info\ - iptables\ - iputils-tracepath\ - irqbalance\ - language-selector-common\ - libaccountsservice0\ - libevent-2.0-5\ - libgeoip1\ - libnfnetlink0\ - libpcap0.8\ - libpci3\ - libpipeline1\ - libpolkit-gobject-1-0\ - libsasl2-modules\ - libusb-1.0-0\ - lshw\ - lsof\ - ltrace\ - man-db\ - mlocate\ - mtr-tiny\ - nano\ - ntfs-3g\ - parted\ - patch\ - plymouth-theme-ubuntu-text\ - popularity-contest\ - powermgmt-base\ - ppp\ - screen\ - shared-mime-info\ - strace\ - tcpdump\ - telnet\ - time\ - tmux\ - ubuntu-standard\ - ufw\ - update-manager-core\ - update-notifier-common\ - usbutils\ - uuid-runtime\ - -# The following packages cannot be removed as they cause cloud-init to be -# uninstalled in Ubuntu 14.04 -# gir1.2-glib-2.0 -# libdbus-glib-1-2 -# libgirepository-1.0-1 -# python-chardet -# python-serial -# xz-utils - -apt-get -y autoremove - diff --git a/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps b/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps deleted file mode 100644 index b215f584..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-mariadb/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-mariadb
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb deleted file mode 100755 index 6d12202c..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install controller base required packages -# Refer to https://mariadb.com/kb/en/library/installing-mariadb-deb-files - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -# These GPG key IDs are used to fetch keys from a keyserver on Ubuntu & Debian -apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 -curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | - bash -s -- --mariadb-server-version="mariadb-10.4" --skip-key-import --skip-maxscale - -apt-get install -y -qq apt-transport-https ca-certificates gnupg2 - -# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html -wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - -# Disable password prompt -debconf-set-selections <<< "mariadb-server mysql-server/root_password password ''" -debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password ''" - -apt-get update -qq -apt-get install -y -qq --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common - -cat <<EOF >/etc/mysql/conf.d/no_perf_schema.cnf -[mysqld] -performance_schema = off -EOF - -chown mysql:mysql /etc/mysql/my.cnf -rm -f /etc/init.d/mysql - -systemctl daemon-reload -systemctl enable mariadb
\ No newline at end of file diff --git a/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps b/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps deleted file mode 100644 index bd3447a6..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-mysql/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-mysql diff --git a/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql b/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql deleted file mode 100755 index e9f2d8c6..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-mysql/install.d/30-mysql +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -# CONTEXT: GUEST during CONSTRUCTION as ROOT -# PURPOSE: Install controller base required packages - -set -e -set -o xtrace - -export DEBIAN_FRONTEND=noninteractive - -apt-get --allow-unauthenticated -y install mysql-client mysql-server gnupg2 - -# NOTE(lxkong): Refer to https://www.percona.com/doc/percona-xtrabackup/2.4/installation/apt_repo.html -wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb -apt-get update - -# Xenial provides mysql 5.7 which requires percona-xtrabackup-24 -PXB_VERSION_OVERRIDE=24 -apt-get --allow-unauthenticated -y install percona-xtrabackup-${PXB_VERSION_OVERRIDE} - -cat >/etc/mysql/conf.d/no_perf_schema.cnf <<_EOF_ -[mysqld] -performance_schema = off -show_compatibility_56 = on -_EOF_ - -mv /etc/mysql/my.cnf.fallback /etc/mysql/my.cnf -chown mysql:mysql /etc/mysql/my.cnf -cat >/etc/mysql/my.cnf <<_EOF_ -[mysql] -!includedir /etc/mysql/conf.d/ -_EOF_ - -if [ -e /etc/init/mysql.conf ]; then - rm -f /etc/init/mysql.conf -fi - -systemctl enable mysql diff --git a/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps b/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps deleted file mode 100644 index 6a0e1b09..00000000 --- a/integration/scripts/files/elements/ubuntu-xenial-postgresql/element-deps +++ /dev/null @@ -1 +0,0 @@ -ubuntu-postgresql
\ No newline at end of file diff --git a/integration/scripts/functions_qemu b/integration/scripts/functions_qemu index f43078d9..783ae9c6 100644 --- a/integration/scripts/functions_qemu +++ b/integration/scripts/functions_qemu @@ -3,44 +3,36 @@ # Additional functions that would mostly just pertain to a Ubuntu + Qemu setup # -function build_vm() { - exclaim "Actually building the image, this can take up to 15 minutes" - rm -rf ~/.cache/image-create +function build_guest_image() { + exclaim "Actually building the image, params: $@" - local datastore_type=$1 - local guest_os=$2 - local guest_release=$3 - local dev_mode=$4 - local guest_username=$5 - local image_output=$6 + local guest_os=$1 + local guest_release=$2 + local dev_mode=$3 + local guest_username=$4 + local image_output=$5 local elementes="base vm" local trove_elements_path=${PATH_TROVE}/integration/scripts/files/elements - local GUEST_IMAGETYPE=${GUEST_IMAGETYPE:-"qcow2"} - local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-4} + local GUEST_IMAGESIZE=${GUEST_IMAGESIZE:-3} local GUEST_CACHEDIR=${GUEST_CACHEDIR:-"$HOME/.cache/image-create"} + rm -rf ${GUEST_CACHEDIR} local working_dir=$(dirname ${image_output}) export GUEST_USERNAME=${guest_username} + export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)} + export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g') + export DEV_MODE=${dev_mode,,} # In dev mode, the trove guest agent needs to download trove code from # trove-taskmanager host during service initialization. - if [[ "${dev_mode,,}" == "true" ]]; then - export PATH_TROVE=${PATH_TROVE} - export ESCAPED_PATH_TROVE=$(echo ${PATH_TROVE} | sed 's/\//\\\//g') - export GUEST_LOGDIR=${GUEST_LOGDIR:-"/var/log/trove/"} - export ESCAPED_GUEST_LOGDIR=$(echo ${GUEST_LOGDIR} | sed 's/\//\\\//g') - export TROVESTACK_SCRIPTS=${TROVESTACK_SCRIPTS} - export HOST_SCP_USERNAME=${HOST_SCP_USERNAME:-$(whoami)} - export HOST_USERNAME=${HOST_SCP_USERNAME} + if [[ "${DEV_MODE}" == "true" ]]; then export SSH_DIR=${SSH_DIR:-"$HOME/.ssh"} - export DEST=${DEST:-'/opt/stack'} - export TROVE_BRANCH=${TROVE_BRANCH:-'master'} manage_ssh_keys fi # For system-wide installs, DIB will automatically find the elements, so we only check local path - if [ "${DIB_LOCAL_ELEMENTS_PATH}" ]; then + if [[ "${DIB_LOCAL_ELEMENTS_PATH}" ]]; then export ELEMENTS_PATH=${trove_elements_path}:${DIB_LOCAL_ELEMENTS_PATH} else export ELEMENTS_PATH=${trove_elements_path} @@ -50,36 +42,26 @@ function build_vm() { export DIB_CLOUD_INIT_DATASOURCES="ConfigDrive" # https://cloud-images.ubuntu.com/releases is more stable than the daily - # builds(https://cloud-images.ubuntu.com/xenial/current/), - # e.g. sometimes SHA256SUMS file is missing in the daily builds - declare -A releasemapping=( ["xenial"]="16.04" ["bionic"]="18.04") + # builds (https://cloud-images.ubuntu.com/xenial/current/), + # e.g. sometimes SHA256SUMS file is missing in the daily builds website. + # Ref: diskimage_builder/elements/ubuntu/root.d/10-cache-ubuntu-tarball + declare -A image_file_mapping=( ["xenial"]="ubuntu-16.04-server-cloudimg-amd64-root.tar.gz" ["bionic"]="ubuntu-18.04-server-cloudimg-amd64.squashfs" ) export DIB_CLOUD_IMAGES="https://cloud-images.ubuntu.com/releases/${DIB_RELEASE}/release/" - export BASE_IMAGE_FILE="ubuntu-${releasemapping[${DIB_RELEASE}]}-server-cloudimg-amd64-root.tar.gz" + export BASE_IMAGE_FILE=${image_file_mapping[${DIB_RELEASE}]} TEMP=$(mktemp -d ${working_dir}/diskimage-create.XXXXXXX) pushd $TEMP > /dev/null elementes="$elementes ${guest_os}" - - if [[ "${dev_mode,,}" == "false" ]]; then - elementes="$elementes pip-and-virtualenv" - elementes="$elementes pip-cache" - elementes="$elementes guest-agent" - else - # Install guest agent dependencies, user, etc. - elementes="$elementes ${guest_os}-guest" - # Install guest agent service - elementes="$elementes ${guest_os}-${guest_release}-guest" - fi - - elementes="$elementes ${guest_os}-${datastore_type}" - elementes="$elementes ${guest_os}-${guest_release}-${datastore_type}" + elementes="$elementes pip-and-virtualenv" + elementes="$elementes pip-cache" + elementes="$elementes guest-agent" + elementes="$elementes ${guest_os}-docker" # Build the image disk-image-create -x \ -a amd64 \ -o ${image_output} \ - -t ${GUEST_IMAGETYPE} \ --image-size ${GUEST_IMAGESIZE} \ --image-cache ${GUEST_CACHEDIR} \ $elementes @@ -91,25 +73,6 @@ function build_vm() { exclaim "Image ${image_output} was built successfully." } -function build_guest_image() { - exclaim "Params for build_guest_image function: $@" - - local datastore_type=${1:-"mysql"} - local guest_os=${2:-"ubuntu"} - local guest_release=${3:-"xenial"} - local dev_mode=${4:-"true"} - local guest_username=${5:-"ubuntu"} - local output=$6 - - VALID_SERVICES='mysql percona mariadb redis cassandra couchbase mongodb postgresql couchdb vertica db2 pxc' - if ! [[ " $VALID_SERVICES " =~ " $datastore_type " ]]; then - exclaim "You did not pass in a valid datastore type. Valid types are:" $VALID_SERVICES - exit 1 - fi - - build_vm ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output} -} - function clean_instances() { LIST=`virsh -q list|awk '{print $1}'` for i in $LIST; do sudo virsh destroy $i; done @@ -117,6 +80,8 @@ function clean_instances() { # In dev mode, guest agent needs to ssh into the controller to download code. function manage_ssh_keys() { + SSH_DIR=${SSH_DIR:-"$HOME/.ssh"} + if [ -d ${SSH_DIR} ]; then echo "${SSH_DIR} already exists" else diff --git a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc b/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc deleted file mode 100644 index 9b80b6c6..00000000 --- a/integration/scripts/local.conf.d/ceilometer_cinder.conf.rc +++ /dev/null @@ -1,3 +0,0 @@ -[[post-config|\$CINDER_CONF]] -[DEFAULT] -notification_driver = messagingv2 diff --git a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc b/integration/scripts/local.conf.d/ceilometer_nova.conf.rc deleted file mode 100644 index fe48b022..00000000 --- a/integration/scripts/local.conf.d/ceilometer_nova.conf.rc +++ /dev/null @@ -1,3 +0,0 @@ -[[post-config|\$NOVA_CONF]] -[DEFAULT] -instance_usage_audit = True diff --git a/integration/scripts/local.conf.d/ceilometer_services.conf.rc b/integration/scripts/local.conf.d/ceilometer_services.conf.rc deleted file mode 100644 index ce33948e..00000000 --- a/integration/scripts/local.conf.d/ceilometer_services.conf.rc +++ /dev/null @@ -1,3 +0,0 @@ -[[post-config|\$CEILOMETER_CONF]] -[notification] -store_events = True diff --git a/integration/scripts/local.conf.d/sample.rc b/integration/scripts/local.conf.d/sample.rc deleted file mode 100644 index 401418d8..00000000 --- a/integration/scripts/local.conf.d/sample.rc +++ /dev/null @@ -1,42 +0,0 @@ -# -# Files in this directory are automatically added to the devstack -# local.conf file, between a specific set of tags. -# -# Filenames must end with '.rc' to be recognized; sample.rc is -# ignored. -# -# A '\' is required in front of any devstack variables since all -# .rc files are parsed first (using eval). -# -# Meta section headings must be included in each file, such as: -# [[local|localrc]] -# as the order of inserting the files is not guaranteed. -# -# All files are inherently included by default - to exclude a file, -# add a variable 'FILENAME_IN_UPPERCASE_MINUS_RC=false' in trovestack.rc -# For Example: USING_VAGRANT=false (for the using_vagrant.rc file). -# -# Symbolic links are followed, so additional files can be loaded -# by placing them in an external directory and linking it in -# local.conf.d (this should allow complete flexibility in setting -# up testing options). -# For Example: -# cd /path/to/trove/integration/scripts/local.conf.d -# ln -s $HOME/local.conf.d local.conf.d -# cp /path/to/my_conf.rc $HOME/local.conf.d - - -[[local|localrc]] -# Put regular devstack variables under this meta section heading. -# This section is written out to a file and sourced by devstack, -# so it can contain logic as well. - -# The following section types should only contain ini file style -# section headings and name=value pairs -[[post-config|\$TROVE_CONF]] - -[[post-config|\$TROVE_TASKMANAGER_CONF]] - -[[post-config|\$TROVE_CONDUCTOR_CONF]] - -[[post-config|\$TROVE_API_PASTE_INI]] diff --git a/integration/scripts/local.conf.d/trove_services.conf.rc b/integration/scripts/local.conf.d/trove_services.conf.rc deleted file mode 100644 index 6eedc1c3..00000000 --- a/integration/scripts/local.conf.d/trove_services.conf.rc +++ /dev/null @@ -1,24 +0,0 @@ -[[post-config|\$TROVE_CONF]] -[profiler] -enabled = $ENABLE_PROFILER -trace_sqlalchemy = $PROFILER_TRACE_SQL - -[[post-config|\$TROVE_TASKMANAGER_CONF]] -[profiler] -enabled = $ENABLE_PROFILER -trace_sqlalchemy = $PROFILER_TRACE_SQL - -[[post-config|\$TROVE_CONDUCTOR_CONF]] -[profiler] -enabled = $ENABLE_PROFILER -trace_sqlalchemy = $PROFILER_TRACE_SQL - -[[post-config|\$TROVE_GUESTAGENT_CONF]] -[profiler] -enabled = $ENABLE_PROFILER -trace_sqlalchemy = $PROFILER_TRACE_SQL - -[[post-config|\$TROVE_API_PASTE_INI]] -[filter:osprofiler] -enabled = $ENABLE_PROFILER -hmac_keys = $PROFILER_HMAC_KEYS diff --git a/integration/scripts/local.conf.d/use_kvm.rc b/integration/scripts/local.conf.d/use_kvm.rc deleted file mode 100644 index 06bc2ebc..00000000 --- a/integration/scripts/local.conf.d/use_kvm.rc +++ /dev/null @@ -1,4 +0,0 @@ -[[local|localrc]] - -# force kvm as the libvirt type. -LIBVIRT_TYPE=kvm diff --git a/integration/scripts/local.conf.d/use_uuid_token.rc b/integration/scripts/local.conf.d/use_uuid_token.rc deleted file mode 100644 index 587a4064..00000000 --- a/integration/scripts/local.conf.d/use_uuid_token.rc +++ /dev/null @@ -1,3 +0,0 @@ -[[local|localrc]] - -KEYSTONE_TOKEN_FORMAT=UUID diff --git a/integration/scripts/local.conf.d/using_vagrant.rc b/integration/scripts/local.conf.d/using_vagrant.rc deleted file mode 100644 index 7333cd46..00000000 --- a/integration/scripts/local.conf.d/using_vagrant.rc +++ /dev/null @@ -1,9 +0,0 @@ -[[local|localrc]] - -# This is similar to code found at -# https://github.com/bcwaldon/vagrant_devstack/blob/master/Vagrantfile -# and seems to make instances ping'able in VirtualBox. -FLAT_INTERFACE=eth1 -PUBLIC_INTERFACE=eth1 -FLOATING_RANGE=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.128/28 -HOST_IP=`ip_chunk eth0 1`.`ip_chunk eth0 2`.`ip_chunk eth0 3`.`ip_chunk eth0 4` diff --git a/integration/scripts/local.conf.rc b/integration/scripts/local.conf.rc deleted file mode 100644 index b9a2edab..00000000 --- a/integration/scripts/local.conf.rc +++ /dev/null @@ -1,37 +0,0 @@ -$TROVE_PRESENT_TAG -# Set some arguments for devstack. -# -# Note: This file contains autogenerated parts. -# All lines are removed from between the tag/end of tag -# markers (lines with '$MARKER_TOKEN' at beginning and end) and -# are replaced by trovestack. -# Edits to these sections will not persist. -# -# See the '$USER_OPTS_TAG' section -# for ways to insert user args into this file. -# - -# -# This section is for things that belong in localrc -# It comes from $DEFAULT_LOCALRC -# -[[local|localrc]] - -$LOCALRC_OPTS_TAG -$LOCALRC_OPTS_TAG_END - -# -# User options here were inserted from the file USER_LOCAL_CONF -# (defaults to $USERHOME/.$LOCAL_CONF) -# - -$USER_OPTS_TAG -$USER_OPTS_TAG_END - -# -# Additional options here were inserted by trovestack -# automatically from files in $LOCAL_CONF_D -# - -$ADD_OPTS_TAG -$ADD_OPTS_TAG_END diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index 4f6273e6..d41b236f 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -124,7 +124,7 @@ if is_fedora; then else PKG_INSTALL_OPTS="DEBIAN_FRONTEND=noninteractive" PKG_MGR=apt-get - PKG_GET_ARGS="-y --allow-unauthenticated --force-yes" + PKG_GET_ARGS="-y --allow-unauthenticated --force-yes -qq" fi PKG_INSTALL_ARG="install" PKG_UPDATE_ARG="update" @@ -522,57 +522,15 @@ function set_bin_path() { function cmd_set_datastore() { local IMAGEID=$1 - local DATASTORE_TYPE=$2 - - # rd_manage datastore_update <datastore_name> <default_version> - rd_manage datastore_update "$DATASTORE_TYPE" "" - PACKAGES=${PACKAGES:-""} - - if [ "$DATASTORE_TYPE" == "mysql" ]; then - VERSION="5.7" - elif [ "$DATASTORE_TYPE" == "percona" ]; then - PACKAGES=${PACKAGES:-"percona-server-server-5.6"} - VERSION="5.6" - elif [ "$DATASTORE_TYPE" == "pxc" ]; then - PACKAGES=${PACKAGES:-"percona-xtradb-cluster-server-5.6"} - VERSION="5.6" - elif [ "$DATASTORE_TYPE" == "mariadb" ]; then - VERSION="10.4" - elif [ "$DATASTORE_TYPE" == "mongodb" ]; then - PACKAGES=${PACKAGES:-"mongodb-org"} - VERSION="3.2" - elif [ "$DATASTORE_TYPE" == "redis" ]; then - PACKAGES=${PACKAGES:-""} - VERSION="3.2.6" - elif [ "$DATASTORE_TYPE" == "cassandra" ]; then - PACKAGES=${PACKAGES:-"cassandra"} - VERSION="2.1.0" - elif [ "$DATASTORE_TYPE" == "couchbase" ]; then - PACKAGES=${PACKAGES:-"couchbase-server"} - VERSION="2.2.0" - elif [ "$DATASTORE_TYPE" == "postgresql" ]; then - VERSION="9.6" - elif [ "$DATASTORE_TYPE" == "couchdb" ]; then - PACKAGES=${PACKAGES:-"couchdb"} - VERSION="1.6.1" - elif [ "$DATASTORE_TYPE" == "vertica" ]; then - PACKAGES=${PACKAGES:-"vertica"} - VERSION="9.0.1" - elif [ "$DATASTORE_TYPE" == "db2" ]; then - PACKAGES=${PACKAGES:-""} - VERSION="11.1" - else - echo "Unrecognized datastore type. ($DATASTORE_TYPE)" - exit 1 - fi + rd_manage datastore_update "$datastore" "" # trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <packages> <active> - rd_manage datastore_version_update "$DATASTORE_TYPE" "$VERSION" "$DATASTORE_TYPE" $IMAGEID "$PACKAGES" 1 - rd_manage datastore_update "$DATASTORE_TYPE" "$VERSION" + rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" $IMAGEID "" 1 + rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" - if [ -f "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json ]; then + if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then # add the configuration parameters to the database for the kick-start datastore - rd_manage db_load_datastore_config_parameters "$DATASTORE_TYPE" "$VERSION" "$PATH_TROVE"/trove/templates/$DATASTORE_TYPE/validation-rules.json + rd_manage db_load_datastore_config_parameters "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json fi } @@ -627,8 +585,8 @@ function install_test_packages() { DATASTORE_TYPE=$1 sudo -H $HTTP_PROXY pip install openstack.nose_plugin proboscis pexpect - if [ "$DATASTORE_TYPE" = "couchbase" ]; then - if [ "$DISTRO" == "ubuntu" ]; then + if [[ "$DATASTORE_TYPE" = "couchbase" ]]; then + if [[ "$DISTRO" == "ubuntu" ]]; then # Install Couchbase SDK for scenario tests. sudo -H $HTTP_PROXY curl http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add - echo "deb http://packages.couchbase.com/ubuntu trusty trusty/main" | sudo tee /etc/apt/sources.list.d/couchbase-csdk.list @@ -649,12 +607,12 @@ function mod_confs() { TROVE_REPORT_DIR=${TROVE_REPORT_DIR:=$TROVESTACK_SCRIPTS/../report/} EXTRA_CONF=$TROVESTACK_SCRIPTS/conf/test.extra.conf - if [ -e $EXTRA_CONF ]; then + if [[ -e $EXTRA_CONF ]]; then cat $EXTRA_CONF >> $TEST_CONF fi # Append datastore specific configuration file DATASTORE_CONF=$TROVESTACK_SCRIPTS/conf/$DATASTORE_TYPE.conf - if [ ! -f $DATASTORE_CONF ]; then + if [[ ! -f $DATASTORE_CONF ]]; then exclaim "Datastore configuration file ${DATASTORE_CONF} not found" exit 1 fi @@ -695,14 +653,14 @@ function mod_confs() { sed -i "/%shared_network_subnet%/d" $TEST_CONF fi - if [ "$DATASTORE_TYPE" = "vertica" ]; then + if [[ "$DATASTORE_TYPE" = "vertica" ]]; then # Vertica needs more time than mysql for its boot/start/stop operations. setup_cluster_configs cluster_member_count 3 - elif [ "$DATASTORE_TYPE" = "pxc" ]; then + elif [[ "$DATASTORE_TYPE" = "pxc" ]]; then setup_cluster_configs min_cluster_member_count 2 - elif [ "$DATASTORE_TYPE" = "cassandra" ]; then + elif [[ "$DATASTORE_TYPE" = "cassandra" ]]; then setup_cluster_configs cluster_member_count 2 - elif [ "$DATASTORE_TYPE" = "mongodb" ]; then + elif [[ "$DATASTORE_TYPE" = "mongodb" ]]; then setup_cluster_configs cluster_member_count 2 # Decrease the number of required config servers per cluster to save resources. iniset $TROVE_CONF $DATASTORE_TYPE num_config_servers_per_cluster 1 @@ -747,7 +705,7 @@ function cmd_test_init() { local DATASTORE_TYPE=$1 local DATASTORE_VERSION=$2 - if [ -z "${DATASTORE_TYPE}" ]; then + if [[ -z "${DATASTORE_TYPE}" ]]; then exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" exit 1 fi @@ -768,73 +726,60 @@ function cmd_test_init() { pip3 install -U git+https://opendev.org/openstack/python-troveclient@master#egg=python-troveclient } +# Build trove guest image function cmd_build_image() { exclaim "Params for cmd_build_image function: $@" - local IMAGE_DATASTORE_TYPE=${1:-'mysql'} - local IMAGE_GUEST_OS=${2:-'ubuntu'} - local IMAGE_GUEST_RELEASE=${3:-'xenial'} - local DEV_MODE=${4:-'true'} - local guest_username=${5:-'ubuntu'} - local output=$6 + local image_guest_os=${1:-'ubuntu'} + local image_guest_release=${2:-'bionic'} + local dev_mode=${3:-'true'} + local guest_username=${4:-'ubuntu'} + local output=$5 if [[ -z "$output" ]]; then - image_name="trove-datastore-${IMAGE_GUEST_OS}-${IMAGE_GUEST_RELEASE}-${IMAGE_DATASTORE_TYPE}" + image_name="trove-guest-${image_guest_os}-${image_guest_release}" + if [[ ${dev_mode} == "true" ]]; then + image_name="${image_name}-dev" + fi image_folder=$HOME/images - output="${image_folder}/${image_name}" + output="${image_folder}/${image_name}.qcow2" fi + # Always rebuild the image. - sudo rm -f $output - sudo mkdir -p $(dirname $output); sudo chmod 777 -R $(dirname $output) + sudo rm -rf ${output} + sudo mkdir -p $(dirname ${output}); sudo chmod 777 -R $(dirname ${output}) echo "Ensuring we have all packages needed to build image." sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS update - sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap + sudo $HTTP_PROXY $PKG_MGR $PKG_GET_ARGS install qemu git kpartx debootstrap squashfs-tools sudo -H $HTTP_PROXY pip install diskimage-builder - exclaim "Use diskimage-builder to actually build the Trove Guest Agent Image." - build_guest_image $IMAGE_DATASTORE_TYPE $IMAGE_GUEST_OS $IMAGE_GUEST_RELEASE $DEV_MODE ${guest_username} $output + build_guest_image ${image_guest_os} ${image_guest_release} ${dev_mode} ${guest_username} ${output} } # Build guest image and upload to Glance, register the datastore and configuration parameters. -# We could skip the image build and upload by: -# 1. MYSQL_IMAGE_ID is passed, or -# 2. There is an image in Glance contains the datastore name function cmd_build_and_upload_image() { - local datastore_type=$1 - local guest_os=${2:-"ubuntu"} - local guest_release=${3:-"xenial"} - local dev_mode=${4:-"true"} - local guest_username=${5:-"ubuntu"} - local output_dir=${6:-"$HOME/images"} - - if [ -z "${datastore_type}" ]; then - exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" - exit 1 - fi - - image_var="${datastore_type^^}_IMAGE_ID" - glance_imageid=`eval echo '$'"$image_var"` - - if [[ -z $glance_imageid ]]; then - # Find the first image id with the name contains datastore_type. - glance_imageid=$(openstack $CLOUD_ADMIN_ARG image list | grep "$datastore_type" | awk 'NR==1 {print}' | awk '{print $2}') - - if [[ -z $glance_imageid ]]; then - mkdir -p ${output_dir} - name=trove-datastore-${guest_os}-${guest_release}-${datastore_type} - output=${output_dir}/$name.qcow2 - cmd_build_image ${datastore_type} ${guest_os} ${guest_release} ${dev_mode} ${guest_username} $output - - glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create $name --public --disk-format qcow2 --container-format bare --file $output --property hw_rng_model='virtio' -c id -f value) - [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1 - fi + local guest_os=${1:-"ubuntu"} + local guest_release=${2:-"bionic"} + local dev_mode=${3:-"true"} + local guest_username=${4:-"ubuntu"} + local output_dir=${5:-"$HOME/images"} + + name=trove-guest-${guest_os}-${guest_release} + glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image list --name $name -f value -c ID) + if [[ -z ${glance_imageid} ]]; then + mkdir -p ${output_dir} + output=${output_dir}/${name} + cmd_build_image ${guest_os} ${guest_release} ${dev_mode} ${guest_username} ${output} + + glance_imageid=$(openstack ${CLOUD_ADMIN_ARG} image create ${name} --public --disk-format qcow2 --container-format bare --file ${output} --property hw_rng_model='virtio' --tag trove -c id -f value) + [[ -z "$glance_imageid" ]] && echo "Glance upload failed!" && exit 1 fi exclaim "Using Glance image ID: $glance_imageid" exclaim "Updating Datastores" - cmd_set_datastore "${glance_imageid}" "${datastore_type}" + cmd_set_datastore "${glance_imageid}" } @@ -991,11 +936,11 @@ function cmd_stop() { function cmd_int_tests() { exclaim "Running Trove Integration Tests..." - if [ ! $USAGE_ENDPOINT ]; then + if [[ ! $USAGE_ENDPOINT ]]; then export USAGE_ENDPOINT=trove.tests.util.usage.FakeVerifier fi cd $TROVESTACK_SCRIPTS - if [ $# -lt 1 ]; then + if [[ $# -lt 1 ]]; then args="--group=mysql" else args="$@" @@ -1203,7 +1148,7 @@ function cmd_kick_start() { local DATASTORE_TYPE=$1 local DATASTORE_VERSION=$2 - if [ -z "${DATASTORE_TYPE}" ]; then + if [[ -z "${DATASTORE_TYPE}" ]]; then exclaim "${COLOR_RED}Datastore argument was not specified.${COLOR_NONE}" exit 1 fi @@ -1220,10 +1165,13 @@ function cmd_kick_start() { function cmd_gate_tests() { local DATASTORE_TYPE=${1:-'mysql'} local TEST_GROUP=${2:-${DATASTORE_TYPE}} - local DATASTORE_VERSION=${3:-'5.7'} + local DATASTORE_VERSION=${3:-'5.7.29'} local HOST_SCP_USERNAME=${4:-$(whoami)} local GUEST_USERNAME=${5:-'ubuntu'} + export DATASTORE_TYPE=${DATASTORE_TYPE} + export DATASTORE_VERSION=${DATASTORE_VERSION} + exclaim "Running cmd_gate_tests ..." export REPORT_DIRECTORY=${REPORT_DIRECTORY:=$HOME/gate-tests-report/} @@ -1238,7 +1186,7 @@ function cmd_gate_tests() { cd $TROVESTACK_SCRIPTS # Build and upload guest image, register datastore version. - cmd_build_and_upload_image ${DATASTORE_TYPE} + cmd_build_and_upload_image cmd_kick_start "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" |