Add tool for updating datasets

Crude copy of migration script for updating datasets for testing.

Change-Id: Ic92809c2b9f963af94ca43872835a7e43a9c6d35

1 files changed, 278 insertions, 0 deletions

diff --git a/tools/update_datasets.sh b/tools/update_datasets.sh
new file mode 100644
index 0000000..b635b84
--- /dev/null
+++ b/tools/update_datasets.sh
@@ -0,0 +1,278 @@
+#!/bin/bash
+#
+# Copyright 2014 Rackspace Australia
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# A tool to update a given dataset to a given version. Used to keep datasets
+# somewhat fresh rather than over-exercising old migrations.
+
+
+# Usage: ./update_datasets.sh VENV_NAME WORKING_DIR GIT_PATH DB_USER DB_PASS DB_NAME SEED_DATA OUTPUT_DATA
+# apt-get install git virtualenvwrapper python-pip mysql-server python-lxml build-essential libffi-dev
+
+# $1 is the unique job id
+# $2 is the working dir path
+# $3 is the path to the git repo path
+# $4 is the nova db user
+# $5 is the nova db password
+# $6 is the nova db name
+# $7 is the path to the seed dataset to test against
+# $8 is the logging.conf for openstack
+# $9 is the pip cache dir
+
+UNIQUE_ID=$1
+WORKING_DIR_PATH=`realpath $2`
+GIT_REPO_PATH=`realpath $3`
+DB_USER=$4
+DB_PASS=$5
+DB_NAME=$6
+DATASET_SEED_SQL=`realpath $7`
+DATASET_OUTPUT_SQL=$8
+
+SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+# We also support the following environment variables to tweak our behavour:
+#   NOCLEANUP: if set to anything, don't cleanup at the end of the run
+
+pip_requires() {
+  pip install -q mysql-python
+  pip install -q eventlet
+  requires="tools/pip-requires"
+  if [ ! -e $requires ]
+  then
+    requires="requirements.txt"
+  fi
+  echo "Install pip requirements from $requires"
+  pip install -q -r $requires
+  echo "Requirements installed"
+}
+
+db_sync() {
+  # $1 is the test target (ie branch name)
+  # $2 is an (optional) destination version number
+
+  # Create a nova.conf file
+  cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF
+[DEFAULT]
+sql_connection = mysql://$DB_USER:$DB_PASS@localhost/$DB_NAME?charset=utf8
+#log_config = $LOG_CONF_FILE
+EOF
+
+  # Silently return git to a known good state (delete untracked files)
+  git clean -xfdq
+
+  echo "***** Start DB upgrade to state of $1 *****"
+  echo "HEAD of branch under test is:"
+  git log -n 1
+
+  echo "Setting up the nova-manage entry point"
+  python setup.py -q clean
+  python setup.py -q develop
+  python setup.py -q install
+
+  # Log the migrations present
+  echo "Migrations present:"
+  ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_"
+
+  # Flush innodb's caches
+  echo "Restarting mysql"
+  sudo service mysql stop
+  sudo service mysql start
+
+  start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+
+  if [ "%$2%" == "%%" ]
+  then
+    end_version=`ls $GIT_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"`
+  else
+    end_version=$2
+  fi
+
+  echo "Test will migrate from $start_version to $end_version"
+  if [ $end_version -lt $start_version ]
+  then
+    increment=-1
+    end_version=$(( $end_version + 1 ))
+  else
+    increment=1
+    start_version=$(( $start_version + 1))
+  fi
+
+  for i in `seq $start_version $increment $end_version`
+  do
+    set -x
+    $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i
+    manage_exit=$?
+    set +x
+
+    echo "nova-manage returned exit code $manage_exit"
+    if [ $manage_exit -gt 0 ]
+    then
+      echo "Aborting early"
+      exit $manage_exit
+    fi
+  done
+
+  echo "***** Finished DB upgrade to state of $1 *****"
+}
+
+stable_release_db_sync() {
+  version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+
+  # Some databases are from Folsom
+  echo "Schema version is $version"
+  if [ $version -lt "161" ]
+  then
+    echo "Database is from Folsom! Upgrade via Grizzly"
+    git branch -D eol/grizzly || true
+    git remote update
+    git checkout -b eol/grizzly
+    # Use tag
+    git reset --hard grizzly-eol
+    pip_requires
+    db_sync "grizzly"
+  fi
+
+  version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+  # Some databases are from Grizzly
+  echo "Schema version is $version"
+  if [ $version -lt "216" ]
+  then
+    echo "Database is from Grizzly! Upgrade via Havana"
+    git branch -D eol/havana || true
+    git remote update
+    git checkout -b eol/havana
+    # Use tag
+    git reset --hard havana-eol
+    pip_requires
+    db_sync "havana"
+  fi
+
+  version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+  # Some databases are from Havana
+  echo "Schema version is $version"
+  if [ $version -lt "234" ]
+  then
+    echo "Database is from Havana! Upgrade via Icehouse"
+    git branch -D eol/icehouse || true
+    git remote update
+    git checkout -b eol/icehouse
+    # Use tag
+    git reset --hard icehouse-eol
+    pip_requires
+    db_sync "icehouse"
+  fi
+
+  version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+  # Some databases are from Icehouse
+  echo "Schema version is $version"
+  if [ $version -lt "254" ]
+  then
+    echo "Database is from Icehouse! Upgrade via Juno"
+    git branch -D stable/juno || true
+    git remote update
+    git checkout -b stable/juno
+    git reset --hard remotes/origin/stable/juno
+    pip_requires
+    db_sync "juno"
+  fi
+
+  version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+  # Some databases are from Juno
+  echo "Schema version is $version"
+  if [ $version -lt "280" ]
+  then
+    echo "Database is from Juno! Upgrade via Kilo"
+    git branch -D stable/kilo || true
+    git remote update
+    git checkout -b stable/kilo
+    git reset --hard remotes/origin/stable/kilo
+    pip_requires
+    db_sync "kilo"
+
+    # TODO(jhesketh): This is a bit of a hack until we update our datasets to
+    # have the flavour data migrated. We have to do this before upgrading from
+    set -x
+    $SCRIPT_DIR/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-kilo.conf --verbose db migrate_flavor_data --force
+    set +x
+  fi
+
+  # TODO(jhesketh): Add in Liberty here once released
+
+  # TODO(jhesketh): Make this more DRY and/or automatically match migration
+  # numbers to releases.
+}
+
+echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)"
+echo "To execute this script manually, run this:"
+echo "$0 $@"
+
+# Setup the environment
+set -x
+export PATH=/usr/lib/ccache:$PATH
+#export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror"
+export PIP_INDEX_URL="http://pypi.openstack.org/simple/"
+export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/"
+which pip
+pip --version
+which virtualenv
+virtualenv --version
+which mkvirtualenv
+set +x
+
+# Restore database to known good state
+echo "Restoring test database $DB_NAME"
+set -x
+mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME"
+mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME"
+mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL
+set +x
+
+echo "Build test environment"
+cd $GIT_REPO_PATH
+
+echo "Setting up virtual env"
+source ~/.bashrc
+export WORKON_HOME=`pwd`/envs
+mkdir -p $WORKON_HOME
+VENV_PATH=$WORKON_HOME/$UNIQUE_ID
+rm -rf $VENV_PATH
+source /usr/local/bin/virtualenvwrapper.sh
+source /etc/bash_completion.d/virtualenvwrapper
+mkvirtualenv --no-site-packages $UNIQUE_ID
+#toggleglobalsitepackages
+export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH
+
+if [ ! -e $VENV_PATH ]
+then
+  echo "Error: making the virtual env failed"
+  exit 1
+fi
+
+stable_release_db_sync
+
+# Determine the final schema version
+version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
+echo "Final schema version is $version"
+
+if [ "%$NOCLEANUP%" == "%%" ]
+then
+  # Cleanup virtual env
+  echo "Cleaning up virtual env"
+  deactivate
+  rmvirtualenv $UNIQUE_ID
+fi
+
+cd $SCRIPT_DIR
+mysqldump -u $DB_USER --password=$DB_PASS $DB_NAME > $DATASET_OUTPUT_SQL