Use Neutron by default

nova-network is being deprecated, and devstack is moving to using
Neutron by default. This patch makes redstack use Neutron by default and
fixed a number of issues related to using Neutron.

Change-Id: If216688ef26b118c6d7598b2e086ba446b03978a
Closes-bug: 1604019

8 files changed, 37 insertions, 39 deletions

diff --git a/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd b/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd
index a6ea6e1..960342e 100755
--- a/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd
+++ b/scripts/files/elements/fedora-guest/extra-data.d/20-guest-systemd
@@ -13,9 +13,9 @@ source $_LIB/die
 
 [ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
 [ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
-[ -n "${NETWORK_GATEWAY}" ] || die "NETWORK_GATEWAY needs to be set to the user for the host instance"
+[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
 [ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host"
 [ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir"
 [ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
 
-sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/NETWORK_GATEWAY/${NETWORK_GATEWAY}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
+sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.systemd.conf > ${TMP_HOOKS_PATH}/trove-guest.service
diff --git a/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart b/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart
index 032c19c..2b22159 100755
--- a/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart
+++ b/scripts/files/elements/ubuntu-guest/extra-data.d/20-guest-upstart
@@ -13,9 +13,9 @@ source $_LIB/die
 
 [ -n "${GUEST_USERNAME}" ] || die "GUEST_USERNAME needs to be set to the user for the guest image"
 [ -n "${HOST_SCP_USERNAME}" ] || die "HOST_SCP_USERNAME needs to be set to the user for the host instance"
-[ -n "${NETWORK_GATEWAY}" ] || die "NETWORK_GATEWAY needs to be set to the user for the host instance"
+[ -n "${CONTROLLER_IP}" ] || die "CONTROLLER_IP needs to be set to the ip address that guests will use to contact the controller"
 [ -n "${ESCAPED_PATH_TROVE}" ] || die "ESCAPED_PATH_TROVE needs to be set to the path to the trove directory on the redstack host"
 [ -n "${REDSTACK_SCRIPTS}" ] || die "REDSTACK_SCRIPTS needs to be set to the trove-integration scripts dir"
 [ -n "${ESCAPED_GUEST_LOGDIR}" ] || die "ESCAPED_GUEST_LOGDIR must be set to the escaped guest log dir"
 
-sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/NETWORK_GATEWAY/${NETWORK_GATEWAY}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf
+sed "s/GUEST_USERNAME/${GUEST_USERNAME}/g;s/GUEST_LOGDIR/${ESCAPED_GUEST_LOGDIR}/g;s/HOST_SCP_USERNAME/${HOST_SCP_USERNAME}/g;s/CONTROLLER_IP/${CONTROLLER_IP}/g;s/PATH_TROVE/${ESCAPED_PATH_TROVE}/g" ${REDSTACK_SCRIPTS}/files/trove-guest.upstart.conf > ${TMP_HOOKS_PATH}/trove-guest.conf
diff --git a/scripts/files/trove-guest.systemd.conf b/scripts/files/trove-guest.systemd.conf
index 0f8cc41..47bc2cf 100644
--- a/scripts/files/trove-guest.systemd.conf
+++ b/scripts/files/trove-guest.systemd.conf
@@ -12,11 +12,11 @@ ExecStartPre=/bin/bash -c "sudo mkdir -p GUEST_LOGDIR ; sudo chown GUEST_USERNAM
 
 # If ~/trove-installed does not exist, copy the trove source from
 # the user's development environment, then touch the sentinel file
-ExecStartPre=/bin/bash -c "test -d /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@NETWORK_GATEWAY:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
+ExecStartPre=/bin/bash -c "test -d /home/GUEST_USERNAME/trove-installed || sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove && touch /home/GUEST_USERNAME/trove-installed"
 
 # If /etc/trove does not exist, create it and then copy the trove-guestagent.conf
 # from /etc/trove on the user's development environment,
-ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@NETWORK_GATEWAY:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf"
+ExecStartPre=/bin/bash -c "test -d /etc/trove/conf.d || sudo mkdir -p /etc/trove/conf.d && sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/ && sudo mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/conf.d/trove-guestagent.conf"
 
 ExecStartPre=/bin/bash -c "sudo chown -R GUEST_USERNAME:root /etc/trove"
 
diff --git a/scripts/files/trove-guest.upstart.conf b/scripts/files/trove-guest.upstart.conf
index 6ba8a48..2e20d6c 100644
--- a/scripts/files/trove-guest.upstart.conf
+++ b/scripts/files/trove-guest.upstart.conf
@@ -16,13 +16,12 @@ pre-start script
 
     # Copy the trove source from the user's development environment
     if [ ! -d /home/GUEST_USERNAME/trove ]; then
-        sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@NETWORK_GATEWAY:PATH_TROVE/ /home/GUEST_USERNAME/trove
+        sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:PATH_TROVE/ /home/GUEST_USERNAME/trove
     fi
 
     # Ensure conf dir exists and is readable
     mkdir -p /etc/trove/conf.d
     chmod -R +r /etc/trove
-
 end script
 
 script
@@ -31,7 +30,7 @@ script
     if [ ! -f /etc/trove/conf.d/guest_info ] && [ ! -f /etc/trove/conf.d/trove-guestagent.conf ]; then
 
         chmod +r /etc/guest_info
-        sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@NETWORK_GATEWAY:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/
+        sudo -u GUEST_USERNAME rsync -e 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' -avz --exclude='.*' HOST_SCP_USERNAME@CONTROLLER_IP:/etc/trove/trove-guestagent.conf ~GUEST_USERNAME/
         mv ~GUEST_USERNAME/trove-guestagent.conf /etc/trove/trove-guestagent.conf
         TROVE_CONFIG="--config-file=/etc/guest_info --config-file=/etc/trove/trove-guestagent.conf"
 
diff --git a/scripts/functions_qemu b/scripts/functions_qemu
index 81954a6..62875ee 100644
--- a/scripts/functions_qemu
+++ b/scripts/functions_qemu
@@ -35,7 +35,7 @@ function build_vm() {
     export HOST_USERNAME
     export HOST_SCP_USERNAME
     export GUEST_USERNAME
-    export NETWORK_GATEWAY
+    export CONTROLLER_IP
     export REDSTACK_SCRIPTS
     export SERVICE_TYPE
     export PATH_TROVE
@@ -50,6 +50,7 @@ function build_vm() {
     export DATASTORE_PKG_LOCATION
     export BRANCH_OVERRIDE
     export DIB_APT_CONF_DIR=/etc/apt/apt.conf.d
+    export DIB_CLOUD_INIT_ETC_HOSTS=true
     local QEMU_IMG_OPTIONS=$(! $(qemu-img | grep -q 'version 1') && echo "--qemu-img-options compat=0.10")
     ${PATH_DISKIMAGEBUILDER}/bin/disk-image-create -a amd64 -o "${VM}" \
         -x ${QEMU_IMG_OPTIONS} ${DISTRO} ${EXTRA_ELEMENTS} vm heat-cfntools \
diff --git a/scripts/localrc.rc b/scripts/localrc.rc
index ef45ba4..8549373 100644
--- a/scripts/localrc.rc
+++ b/scripts/localrc.rc
@@ -22,6 +22,9 @@ ENABLED_SERVICES+=,-n-novnc,-n-xvnc
 if [[ $ENABLE_NEUTRON = true ]]; then
     ENABLED_SERVICES+=,neutron,q-svc,q-agt,q-dhcp,q-l3,q-meta
     disable_service n-net
+else
+    enable_service n-net
+    disable_service neutron q-svc q-agt q-dhcp q-l3 q-meta
 fi
 # OSprofiler depends on Ceilometer
 if [[ $ENABLE_CEILOMETER = true ]] || [[ $ENABLE_PROFILER = true ]]; then
diff --git a/scripts/redstack b/scripts/redstack
index 75eec90..7ac423a 100755
--- a/scripts/redstack
+++ b/scripts/redstack
@@ -75,11 +75,11 @@ export TROVE_API_PASTE_INI=$TROVE_CONF_DIR/api-paste.ini
 export TEST_CONF=$TROVE_CONF_DIR/test.conf
 
 # Public facing bits
-[[ $ENABLE_NEUTRON = true ]] && BRIDGE_IP=172.24.4.1 || BRIDGE_IP=10.0.0.1
 SERVICE_PROTOCOL=${SERVICE_PROTOCOL:-http}
-NETWORK_GATEWAY=${NETWORK_GATEWAY:-$BRIDGE_IP}
 NETWORK_INTERFACE=${NETWORK_INTERFACE:-eth0}
 NETWORK_SUBNET=${NETWORK_SUBNET:-10.0.0.0/24}
+NETWORK_GATEWAY=${NETWORK_GATEWAY:-10.0.0.1}
+BRIDGE_IP=${BRIDGE_IP:-172.24.4.1}
 KEYSTONE_AUTH_HOST=${KEYSTONE_AUTH_HOST:-$SERVICE_HOST}
 KEYSTONE_AUTH_PROTOCOL=${KEYSTONE_AUTH_PROTOCOL:-$SERVICE_PROTOCOL}
 KEYSTONE_AUTH_PORT=${KEYSTONE_AUTH_PORT:-35357}
@@ -87,6 +87,10 @@ GLANCE_HOSTPORT=${GLANCE_HOSTPORT:-$SERVICE_HOST:9292}
 GLANCE_SERVICE_PROTOCOL=${GLANCE_SERVICE_PROTOCOL:-http}
 TROVE_AUTH_ENDPOINT=$KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:$KEYSTONE_AUTH_PORT/v2.0
 
+# The following depends on whether neutron is used or nova-network
+# neutron uses a bridge, nova-network does not
+[[ $ENABLE_NEUTRON = true ]] && CONTROLLER_IP=$BRIDGE_IP || CONTROLLER_IP=$NETWORK_GATEWAY
+
 # PATH_TROVE more than likely has file separators, which sed does not like
 # This will escape them
 ESCAPED_PATH_TROVE=$(echo $PATH_TROVE | sed 's/\//\\\//g')
@@ -697,24 +701,11 @@ function mod_confs() {
     # If neutron is enabled, we create a shared network and write this info to the
     # confs so that the integration tests can use it.
     if [[ $ENABLE_NEUTRON = true ]]; then
-        # customer_network_id=$(neutron net-list | awk '/customer/ {print $2}')
-        # if [[ -z "$customer_network_id" ]]; then
-        #    customer_network_id=$(neutron net-create customer | awk '/ id / {print $4}')
-        #    customer_subnet_id=$(neutron subnet-create customer 10.0.1.0/24 --no-gateway | awk '/ id / {print $4}')
-        # fi
-        management_network_id=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/private/ {print $2}')
-        management_subnet=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/private/ {print $7}')
+        management_network_id=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT net-list | awk '/ alt-private / {print $2}')
+        management_subnet=$(neutron --os-username=admin --os-password=$ADMIN_PASSWORD --os-tenant-name=admin --os-auth-url=$TROVE_AUTH_ENDPOINT subnet-list | awk '/ alt-private-subnet / {print $2}')
+        echo "Using neutron network $management_network_id and subnet $management_subnet"
         sed -i "s,%shared_network%,$management_network_id,g" $TEST_CONF
         sed -i "s,%shared_network_subnet%,$management_subnet,g" $TEST_CONF
-        # sed -i "s,%private_network%,$customer_network_id,g" $TEST_CONF
-        # sed -i "s,%private_network_subnet%,10.0.1.0/24,g" $TEST_CONF
-        iniset $TROVE_CONF DEFAULT network_label_regex .*
-        iniset $TROVE_CONF DEFAULT ip_regex .*
-        iniset $TROVE_CONF DEFAULT black_list_regex ^10.0.1.*
-        iniset $TROVE_CONF DEFAULT default_neutron_networks $management_network_id
-        iniset $TROVE_CONF DEFAULT network_driver trove.network.neutron.NeutronDriver
-        iniset $TROVE_TASKMANAGER_CONF DEFAULT network_driver trove.network.neutron.NeutronDriver
-        iniset $TROVE_TASKMANAGER_CONF mysql tcp_ports 22,3306
     else
         # do not leave invalid keys in the configuration when using Nova for networking
         sed -i "/%shared_network%/d" $TEST_CONF
@@ -1189,7 +1180,7 @@ function cmd_dsvm_gate_tests() {
     TEST_GROUP=${2:-${DATASTORE_TYPE}}
     HOST_SCP_USERNAME=${3:-'jenkins'}
     GUEST_USERNAME=${4:-'ubuntu'}
-    NETWORK_GATEWAY=${5:-'10.1.0.1'}
+    CONTROLLER_IP=${5:-'10.1.0.1'}
     ESCAPED_PATH_TROVE=${6:-'\/opt\/stack\/new\/trove'}
 
     if [[ $BRANCH_OVERRIDE == "stable/liberty" ]]; then
@@ -1208,28 +1199,28 @@ function cmd_dsvm_gate_tests() {
 
     sudo chown -R $(whoami) /etc/trove
     sudo chown -R $(whoami) $DEST/trove-integration
-    iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $NETWORK_GATEWAY
-    iniset $TROVE_GUESTAGENT_CONF oslo_messaging_rabbit rabbit_hosts $NETWORK_GATEWAY
+    iniset $TROVE_GUESTAGENT_CONF DEFAULT rabbit_host $CONTROLLER_IP
+    iniset $TROVE_GUESTAGENT_CONF oslo_messaging_rabbit rabbit_hosts $CONTROLLER_IP
     cd $DEST/trove-integration/scripts
     sudo -H $HTTP_PROXY pip install --upgrade pip dib-utils
 
     cmd_kick_start $DATASTORE_TYPE
 
-    # Update the local swift endpoint in the catalog to use the NETWORK_GATEWAY instead of 127.0.0.1
+    # Update the local swift endpoint in the catalog to use the CONTROLLER_IP instead of 127.0.0.1
     source $DEST/devstack/accrc/admin/admin
     # NOTE(mriedem): We have to treat stable branches before liberty special
     # due to constraints with older versions of python-openstackclient.
     if [[ $BRANCH_OVERRIDE == "stable/juno" || $BRANCH_OVERRIDE == "stable/kilo" ]]; then
         SWIFT_ENDPOINT=$(openstack endpoint list | grep 'swift' | get_field 1)
-        openstack endpoint create swift --region RegionOne --publicurl 'http://'$NETWORK_GATEWAY':8080/v1/AUTH_$(tenant_id)s' \
-            --internalurl 'http://'$NETWORK_GATEWAY':8080/v1/AUTH_$(tenant_id)s' --adminurl 'http://'$NETWORK_GATEWAY':8080'
+        openstack endpoint create swift --region RegionOne --publicurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' \
+            --internalurl 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --adminurl 'http://'$CONTROLLER_IP':8080'
         openstack endpoint delete $SWIFT_ENDPOINT
     else
         OS_CLIENT_ARGS="--os-auth-type v3password --os-auth-url $KEYSTONE_AUTH_PROTOCOL://$KEYSTONE_AUTH_HOST:5000/v3 --os-identity-api-version=3"
         SWIFT_ENDPOINTS=$(openstack endpoint list $OS_CLIENT_ARGS --service swift -c ID -f value)
-        openstack endpoint create $OS_CLIENT_ARGS swift public 'http://'$NETWORK_GATEWAY':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
-        openstack endpoint create $OS_CLIENT_ARGS swift internal 'http://'$NETWORK_GATEWAY':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
-        openstack endpoint create $OS_CLIENT_ARGS swift admin 'http://'$NETWORK_GATEWAY':8080' --region RegionOne
+        openstack endpoint create $OS_CLIENT_ARGS swift public 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
+        openstack endpoint create $OS_CLIENT_ARGS swift internal 'http://'$CONTROLLER_IP':8080/v1/AUTH_$(tenant_id)s' --region RegionOne
+        openstack endpoint create $OS_CLIENT_ARGS swift admin 'http://'$CONTROLLER_IP':8080' --region RegionOne
         echo $SWIFT_ENDPOINTS | xargs -n 1 openstack endpoint delete $OS_CLIENT_ARGS
     fi
 
diff --git a/scripts/redstack.rc b/scripts/redstack.rc
index 2c694ea..4207d28 100644
--- a/scripts/redstack.rc
+++ b/scripts/redstack.rc
@@ -1,7 +1,7 @@
 # Paths inside the VM.
 [ -z $SERVICE_HOST ] && SERVICE_HOST=`get_default_host_ip`
 [ -z $DEST ] && DEST=/opt/stack
-[ -z $BRIDGE_IP ] && BRIDGE_IP=10.0.0.1
+[ -z $BRIDGE_IP ] && BRIDGE_IP=172.24.4.1
 [ -z $PATH_DEVSTACK_SRC ] && PATH_DEVSTACK_SRC=~/devstack
 [ -z $TROVE_CONF_DIR ] && TROVE_CONF_DIR=/etc/trove
 [ -z $MYSQL_HOST ] && MYSQL_HOST=$SERVICE_HOST
@@ -11,7 +11,11 @@
 REGION_NAME=${REGION_NAME:-${OS_REGION_NAME:-RegionOne}}
 
 # Enable neutron instead of nova-network
-ENABLE_NEUTRON=$(get_bool ENABLE_NEUTRON false)
+NEUTRON_DEFAULT=true
+if [[ $BRANCH_OVERRIDE == "stable/liberty" || $BRANCH_OVERRIDE == "stable/mitaka" ]]; then
+    NEUTRON_DEFAULT=false
+fi
+ENABLE_NEUTRON=$(get_bool ENABLE_NEUTRON $NEUTRON_DEFAULT)
 
 # Enable osprofiler - note: Enables Ceilometer as well
 ENABLE_PROFILER=$(get_bool ENABLE_PROFILER false)