blob: 765277128a6cf680dee31ef28f68a26c2dff82e0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
|
#!/bin/bash
#
# Copyright 2013 Rackspace Australia
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# $1 is the unique job id
# $2 is the working dir path
# $3 is the path to the git repo path
# $4 is the nova db user
# $5 is the nova db password
# $6 is the nova db name
# $7 is the path to the dataset to test against
# $8 is the logging.conf for openstack
# $9 is the pip cache dir
UNIQUE_ID=$1
WORKING_DIR_PATH=$2
GIT_REPO_PATH=$3
DB_USER=$4
DB_PASS=$5
DB_NAME=$6
DATASET_SEED_SQL=$7
LOG_CONF_FILE=$8
PIP_CACHE_DIR=$9
# We also support the following environment variables to tweak our behavour:
# NOCLEANUP: if set to anything, don't cleanup at the end of the run
pip_requires() {
pip install -q mysql-python
pip install -q eventlet
requires="tools/pip-requires"
if [ ! -e $requires ]
then
requires="requirements.txt"
fi
echo "Install pip requirements from $requires"
pip install -q -r $requires
echo "Requirements installed"
}
db_sync() {
# $1 is the test target (ie branch name)
# $2 is an (optional) destination version number
# Create a nova.conf file
cat - > $WORKING_DIR_PATH/nova-$1.conf <<EOF
[DEFAULT]
sql_connection = mysql://$DB_USER:$DB_PASS@172.16.0.1/$DB_NAME?charset=utf8
log_config = $LOG_CONF_FILE
EOF
# Silently return git to a known good state (delete untracked files)
git clean -xfdq
echo "***** Start DB upgrade to state of $1 *****"
echo "HEAD of branch under test is:"
git log -n 1
echo "Setting up the nova-manage entry point"
python setup.py -q clean
python setup.py -q develop
python setup.py -q install
# Log the migrations present
echo "Migrations present:"
ls $GIR_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_"
# Flush innodb's caches
echo "Restarting mysql"
sudo service mysql stop
sudo service mysql start
echo "MySQL counters before upgrade:"
mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "show status like 'innodb%';"
start_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
if [ "%$2%" == "%%" ]
then
end_version=`ls $GIR_REPO_PATH/nova/db/sqlalchemy/migrate_repo/versions/*.py | sed 's/.*\///' | egrep "^[0-9]+_" | tail -1 | cut -f 1 -d "_"`
else
end_version=$2
fi
echo "Test will migrate from $start_version to $end_version"
if [ $end_version -lt $start_version ]
then
increment=-1
end_version=$(( $end_version + 1 ))
else
increment=1
start_version=$(( $start_version + 1))
fi
for i in `seq $start_version $increment $end_version`
do
set -x
sudo /sbin/ip netns exec nonet `dirname $0`/nova-manage-wrapper.sh $VENV_PATH --config-file $WORKING_DIR_PATH/nova-$1.conf --verbose db sync --version $i
manage_exit=$?
set +x
echo "MySQL counters after upgrade:"
mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "show status like 'innodb%';"
echo "nova-manage returned exit code $manage_exit"
if [ $manage_exit -gt 0 ]
then
echo "Aborting early"
exit $manage_exit
fi
done
echo "***** Finished DB upgrade to state of $1 *****"
}
stable_release_db_sync() {
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
# Some databases are from Folsom
echo "Schema version is $version"
if [ $version -le "133" ]
then
echo "Database is from Folsom! Upgrade via Grizzly"
git branch -D eol/grizzly || true
git remote update
git checkout -b eol/grizzly
# Use tag
git reset --hard grizzly-eol
pip_requires
db_sync "grizzly"
fi
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
# Some databases are from Grizzly
echo "Schema version is $version"
if [ $version -le "161" ]
then
echo "Database is from Grizzly! Upgrade via Havana"
git branch -D eol/havana || true
git remote update
git checkout -b eol/havana
# Use tag
git reset --hard havana-eol
pip_requires
db_sync "havana"
fi
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
# Some databases are from Havana
echo "Schema version is $version"
if [ $version -le "216" ]
then
echo "Database is from Grizzly! Upgrade via Icehouse"
git branch -D stable/icehouse || true
git remote update
git checkout -b stable/icehouse
git reset --hard remotes/origin/stable/icehouse
pip_requires
db_sync "icehouse"
fi
# TODO(jhesketh): Add in Juno here once released
# TODO(jhesketh): Make this more DRY and/or automatically match migration
# numbers to releases.
}
echo "Test running on "`hostname`" as "`whoami`" ("`echo ~`", $HOME)"
echo "To execute this script manually, run this:"
echo "$0 $@"
# Setup the environment
set -x
export PATH=/usr/lib/ccache:$PATH
export PIP_DOWNLOAD_CACHE=$PIP_CACHE_DIR
#export PIP_INDEX_URL="http://www.rcbops.com/pypi/mirror"
export PIP_INDEX_URL="http://pypi.openstack.org/simple/"
export PIP_EXTRA_INDEX_URL="https://pypi.python.org/simple/"
which pip
pip --version
which virtualenv
virtualenv --version
which mkvirtualenv
set +x
# Restore database to known good state
echo "Restoring test database $DB_NAME"
set -x
mysql -u $DB_USER --password=$DB_PASS -e "drop database $DB_NAME"
mysql -u $DB_USER --password=$DB_PASS -e "create database $DB_NAME"
mysql -u $DB_USER --password=$DB_PASS $DB_NAME < $DATASET_SEED_SQL
set +x
echo "Build test environment"
cd $GIT_REPO_PATH
echo "Setting up virtual env"
source ~/.bashrc
export WORKON_HOME=/var/lib/turbo-hipster/envs
VENV_PATH=$WORKON_HOME/$UNIQUE_ID
rm -rf $VENV_PATH
source /usr/local/bin/virtualenvwrapper.sh
mkvirtualenv --no-site-packages $UNIQUE_ID
#toggleglobalsitepackages
export PYTHONPATH=$PYTHONPATH:$GIT_REPO_PATH
if [ ! -e $VENV_PATH ]
then
echo "Error: making the virtual env failed"
exit 1
fi
stable_release_db_sync
last_stable_version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
echo "Schema after stable_release_db_sync version is $last_stable_version"
# Make sure the test DB is up to date with trunk
if [ `git show | grep "^\-\-\-" | grep "migrate_repo/versions" | wc -l` -gt 0 ]
then
echo "This change alters an existing migration, skipping trunk updates."
else
echo "Update database to current state of trunk"
git checkout master
pip_requires
db_sync "trunk" $WORKING_DIR_PATH $GIT_REPO_PATH $DB_USER $DB_PASS $DB_NAME $LOG_CONF_FILE
git checkout working
fi
# Now run the patchset
echo "Now test the patchset"
pip_requires
db_sync "patchset" $WORKING_DIR_PATH $GIT_REPO_PATH $DB_USER $DB_PASS $DB_NAME $LOG_CONF_FILE
# Determine the schema version
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
echo "Schema version is $version"
echo "Now downgrade all the way back to the last stable version (v$last_stable_version)"
db_sync "downgrade" $WORKING_DIR_PATH $GIT_REPO_PATH $DB_USER $DB_PASS $DB_NAME $LOG_CONF_FILE $last_stable_version
# Determine the schema version
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
echo "Schema version is $version"
echo "And now back up to head from the start of trunk"
db_sync "patchset" $WORKING_DIR_PATH $GIT_REPO_PATH $DB_USER $DB_PASS $DB_NAME $LOG_CONF_FILE
# Determine the final schema version
version=`mysql -u $DB_USER --password=$DB_PASS $DB_NAME -e "select * from migrate_version \G" | grep version | sed 's/.*: //'`
echo "Final schema version is $version"
if [ "%$NOCLEANUP%" == "%%" ]
then
# Cleanup virtual env
echo "Cleaning up virtual env"
deactivate
rmvirtualenv $UNIQUE_ID
fi
|