Merge "db: Final cleanups"

6 files changed, 6 insertions, 500 deletions

diff --git a/lower-constraints.txt b/lower-constraints.txt
index 5ccbfa7738..b57532ced6 100644
--- a/lower-constraints.txt
+++ b/lower-constraints.txt
@@ -1,4 +1,4 @@
-alembic==0.9.8
+alembic==1.5.0
 amqp==2.5.0
 appdirs==1.4.3
 asn1crypto==0.24.0
@@ -137,7 +137,7 @@ simplejson==3.13.2
 six==1.15.0
 smmap2==2.0.3
 sortedcontainers==2.1.0
-SQLAlchemy==1.2.19
+SQLAlchemy==1.4.13
 sqlalchemy-migrate==0.13.0
 sqlparse==0.2.4
 statsd==3.2.2
diff --git a/requirements.txt b/requirements.txt
index b4eb37f5d6..a8bed744fb 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,5 @@
-# The order of packages is significant, because pip processes them in the order
-# of appearance. Changing the order has an impact on the overall integration
-# process, which may cause wedges in the gate later.
-
 pbr>=5.5.1 # Apache-2.0
-SQLAlchemy>=1.2.19 # MIT
+SQLAlchemy>=1.4.13 # MIT
 decorator>=4.1.0 # BSD
 eventlet>=0.30.1 # MIT
 Jinja2>=2.10 # BSD License (3 clause)
@@ -19,6 +15,7 @@ PasteDeploy>=1.5.0 # MIT
 Paste>=2.0.2 # MIT
 PrettyTable>=0.7.1 # BSD
 sqlalchemy-migrate>=0.13.0 # Apache-2.0
+alembic>=1.5.0 # MIT
 netaddr>=0.7.18 # BSD
 netifaces>=0.10.4 # MIT
 paramiko>=2.7.1 # LGPLv2.1+
diff --git a/tools/db/schema_diff.py b/tools/db/schema_diff.py
deleted file mode 100755
index f6a4d5e6d7..0000000000
--- a/tools/db/schema_diff.py
+++ /dev/null
@@ -1,283 +0,0 @@
-#!/usr/bin/env python
-# Copyright 2012 OpenStack Foundation
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-"""
-Utility for diff'ing two versions of the DB schema.
-
-Each release cycle the plan is to compact all of the migrations from that
-release into a single file. This is a manual and, unfortunately, error-prone
-process. To ensure that the schema doesn't change, this tool can be used to
-diff the compacted DB schema to the original, uncompacted form.
-
-The database is specified by providing a SQLAlchemy connection URL WITHOUT the
-database-name portion (that will be filled in automatically with a temporary
-database name).
-
-The schema versions are specified by providing a git ref (a branch name or
-commit hash) and a SQLAlchemy-Migrate version number:
-
-Run like:
-
-    MYSQL:
-
-    ./tools/db/schema_diff.py mysql+pymysql://root@localhost \
-                              master:latest my_branch:82
-
-    POSTGRESQL:
-
-    ./tools/db/schema_diff.py postgresql://localhost \
-                              master:latest my_branch:82
-
-"""
-
-import datetime
-import glob
-import os
-import subprocess
-import sys
-
-from nova.i18n import _
-
-
-# Dump
-
-
-def dump_db(db_driver, db_name, db_url, migration_version, dump_filename):
-    if not db_url.endswith('/'):
-        db_url += '/'
-
-    db_url += db_name
-
-    db_driver.create(db_name)
-    try:
-        _migrate(db_url, migration_version)
-        db_driver.dump(db_name, dump_filename)
-    finally:
-        db_driver.drop(db_name)
-
-
-# Diff
-
-
-def diff_files(filename1, filename2):
-    pipeline = ['diff -U 3 %(filename1)s %(filename2)s'
-                % {'filename1': filename1, 'filename2': filename2}]
-
-    # Use colordiff if available
-    if subprocess.call(['which', 'colordiff']) == 0:
-        pipeline.append('colordiff')
-
-    pipeline.append('less -R')
-
-    cmd = ' | '.join(pipeline)
-    subprocess.check_call(cmd, shell=True)
-
-
-# Database
-
-
-class Mysql(object):
-    def create(self, name):
-        subprocess.check_call(['mysqladmin', '-u', 'root', 'create', name])
-
-    def drop(self, name):
-        subprocess.check_call(['mysqladmin', '-f', '-u', 'root', 'drop', name])
-
-    def dump(self, name, dump_filename):
-        subprocess.check_call(
-                'mysqldump -u root %(name)s > %(dump_filename)s'
-                % {'name': name, 'dump_filename': dump_filename},
-                shell=True)
-
-
-class Postgresql(object):
-    def create(self, name):
-        subprocess.check_call(['createdb', name])
-
-    def drop(self, name):
-        subprocess.check_call(['dropdb', name])
-
-    def dump(self, name, dump_filename):
-        subprocess.check_call(
-                'pg_dump %(name)s > %(dump_filename)s'
-                % {'name': name, 'dump_filename': dump_filename},
-                shell=True)
-
-
-def _get_db_driver_class(db_url):
-    try:
-        return globals()[db_url.split('://')[0].capitalize()]
-    except KeyError:
-        raise Exception(_("database %s not supported") % db_url)
-
-
-# Migrate
-
-
-MIGRATE_REPO = os.path.join(os.getcwd(), "nova/db/main/legacy_migrations")
-
-
-def _migrate(db_url, migration_version):
-    earliest_version = _migrate_get_earliest_version()
-
-    # NOTE(sirp): sqlalchemy-migrate currently cannot handle the skipping of
-    # migration numbers.
-    _migrate_cmd(
-            db_url, 'version_control', str(earliest_version - 1))
-
-    upgrade_cmd = ['upgrade']
-    if migration_version != 'latest':
-        upgrade_cmd.append(str(migration_version))
-
-    _migrate_cmd(db_url, *upgrade_cmd)
-
-
-def _migrate_cmd(db_url, *cmd):
-    manage_py = os.path.join(MIGRATE_REPO, 'manage.py')
-
-    args = ['python', manage_py]
-    args += cmd
-    args += ['--repository=%s' % MIGRATE_REPO,
-             '--url=%s' % db_url]
-
-    subprocess.check_call(args)
-
-
-def _migrate_get_earliest_version():
-    versions_glob = os.path.join(MIGRATE_REPO, 'versions', '???_*.py')
-
-    versions = []
-    for path in glob.iglob(versions_glob):
-        filename = os.path.basename(path)
-        prefix = filename.split('_', 1)[0]
-        try:
-            version = int(prefix)
-        except ValueError:
-            pass
-        versions.append(version)
-
-    versions.sort()
-    return versions[0]
-
-
-# Git
-
-
-def git_current_branch_name():
-    ref_name = git_symbolic_ref('HEAD', quiet=True)
-    current_branch_name = ref_name.replace('refs/heads/', '')
-    return current_branch_name
-
-
-def git_symbolic_ref(ref, quiet=False):
-    args = ['git', 'symbolic-ref', ref]
-    if quiet:
-        args.append('-q')
-    proc = subprocess.Popen(args, stdout=subprocess.PIPE)
-    stdout, stderr = proc.communicate()
-    return stdout.strip()
-
-
-def git_checkout(branch_name):
-    subprocess.check_call(['git', 'checkout', branch_name])
-
-
-def git_has_uncommited_changes():
-    return subprocess.call(['git', 'diff', '--quiet', '--exit-code']) == 1
-
-
-# Command
-
-
-def die(msg):
-    print("ERROR: %s" % msg, file=sys.stderr)
-    sys.exit(1)
-
-
-def usage(msg=None):
-    if msg:
-        print("ERROR: %s" % msg, file=sys.stderr)
-
-    prog = "schema_diff.py"
-    args = ["<db-url>", "<orig-branch:orig-version>",
-            "<new-branch:new-version>"]
-
-    print("usage: %s %s" % (prog, ' '.join(args)), file=sys.stderr)
-    sys.exit(1)
-
-
-def parse_options():
-    try:
-        db_url = sys.argv[1]
-    except IndexError:
-        usage("must specify DB connection url")
-
-    try:
-        orig_branch, orig_version = sys.argv[2].split(':')
-    except IndexError:
-        usage('original branch and version required (e.g. master:82)')
-
-    try:
-        new_branch, new_version = sys.argv[3].split(':')
-    except IndexError:
-        usage('new branch and version required (e.g. master:82)')
-
-    return db_url, orig_branch, orig_version, new_branch, new_version
-
-
-def main():
-    timestamp = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S")
-
-    ORIG_DB = 'orig_db_%s' % timestamp
-    NEW_DB = 'new_db_%s' % timestamp
-
-    ORIG_DUMP = ORIG_DB + ".dump"
-    NEW_DUMP = NEW_DB + ".dump"
-
-    options = parse_options()
-    db_url, orig_branch, orig_version, new_branch, new_version = options
-
-    # Since we're going to be switching branches, ensure user doesn't have any
-    # uncommitted changes
-    if git_has_uncommited_changes():
-        die("You have uncommitted changes. Please commit them before running "
-            "this command.")
-
-    db_driver = _get_db_driver_class(db_url)()
-
-    users_branch = git_current_branch_name()
-    git_checkout(orig_branch)
-
-    try:
-        # Dump Original Schema
-        dump_db(db_driver, ORIG_DB, db_url, orig_version, ORIG_DUMP)
-
-        # Dump New Schema
-        git_checkout(new_branch)
-        dump_db(db_driver, NEW_DB, db_url, new_version, NEW_DUMP)
-
-        diff_files(ORIG_DUMP, NEW_DUMP)
-    finally:
-        git_checkout(users_branch)
-
-        if os.path.exists(ORIG_DUMP):
-            os.unlink(ORIG_DUMP)
-
-        if os.path.exists(NEW_DUMP):
-            os.unlink(NEW_DUMP)
-
-
-if __name__ == "__main__":
-    main()
diff --git a/tools/generate-schemas b/tools/generate-schemas
deleted file mode 100755
index 4df396bf8c..0000000000
--- a/tools/generate-schemas
+++ /dev/null
@@ -1,123 +0,0 @@
-#!/usr/bin/env bash
-#
-# Script to generate schemas for the various versions.
-#
-# Some setup is required, similar to the opportunistic tests.
-#
-# MySQL ->
-#
-# $ mysql -uroot
-# MariaDB [(none)]> CREATE DATABASE nova
-# MariaDB [(none)]> GRANT ALL PRIVILEGES ON nova.* TO 'nova'@'localhost' IDENTIFIED BY 'password';
-# MariaDB [(none)]> quit;
-#
-# Postgres ->
-#
-# $ sudo -u postgres psql
-# postgres=# create user nova with createdb login password 'password';
-# postgres=# create database nova with owner nova;
-# postgres=# quit;
-#
-# Note that you may also have to configure 'pg_hba.conf' to use password-based
-# auth instead of "ident", if you haven't done so already. You can locate this
-# with 'locate pg_hba.conf'. More details at
-# https://ubuntu.com/server/docs/databases-postgresql
-
-set -o xtrace
-set -e
-
-source .tox/py36/bin/activate
-pushd nova/db/main/legacy_migrations
-
-INIT_VERSION=$(ls -1 versions/ | head -1 | awk -F_ '{print $1}')
-INIT_VERSION=$(($INIT_VERSION-1))
-
-echo "Detected init version of $INIT_VERSION"
-
-mkdir -p schemas
-rm -f "schemas/$INIT_VERSION-*.sql"
-
-#
-# sqlite
-#
-
-# cleanup from previous runs
-
-rm -f nova.db
-
-# sync schema
-
-python manage.py version_control \
-    --database 'sqlite:///nova.db' \
-    --version $INIT_VERSION
-
-python manage.py upgrade \
-    --database 'sqlite:///nova.db'
-
-# dump the schema
-
-sqlite3 nova.db << EOF
-.output "schemas/${INIT_VERSION}-sqlite.sql"
-.schema
-.quit
-EOF
-
-rm -f nova.db
-
-#
-# mysql
-#
-
-# cleanup from previous runs
-
-mysql -u nova -ppassword << EOF
-DROP DATABASE IF EXISTS nova;
-CREATE DATABASE nova;
-EOF
-
-# sync schema
-
-python manage.py version_control \
-    --database 'mysql+pymysql://nova:password@localhost/nova' \
-    --version "$INIT_VERSION"
-
-python manage.py upgrade \
-    --database 'mysql+pymysql://nova:password@localhost/nova'
-
-# dump the schema
-
-mysqldump --no-data --skip-comments -u nova -ppassword \
-    nova > "schemas/${INIT_VERSION}-mysql.sql"
-
-mysql -u nova -ppassword << EOF
-DROP DATABASE IF EXISTS nova;
-EOF
-
-
-#
-# postgres
-#
-
-# cleanup from previous runs
-
-sudo -u postgres dropdb --if-exists nova
-sudo -u postgres createdb --owner=nova nova
-
-# sync to initial version
-
-python manage.py version_control \
-    --database 'postgresql://nova:password@localhost/nova' \
-    --version "$INIT_VERSION"
-
-python manage.py upgrade \
-    --database 'postgresql://nova:password@localhost/nova'
-
-# dump the schema
-
-pg_dump postgresql://nova:password@localhost/nova \
-    --schema-only > "schemas/${INIT_VERSION}-postgres.sql"
-
-sudo -u postgres dropdb --if-exists nova
-
-popd
-deactivate
diff --git a/tools/reserve-migrations.py b/tools/reserve-migrations.py
deleted file mode 100755
index 373c574b69..0000000000
--- a/tools/reserve-migrations.py
+++ /dev/null
@@ -1,75 +0,0 @@
-#!/usr/bin/env python
-
-import argparse
-import glob
-import os
-import subprocess
-
-BASE = 'nova/db/main/legacy_migrations/versions'.split('/')
-API_BASE = 'nova/db/api/legacy_migrations/versions'.split('/')
-
-STUB = \
-"""#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-# This is a placeholder for backports.
-# Do not use this number for new work.  New work starts after
-# all the placeholders.
-#
-# See this for more information:
-# http://lists.openstack.org/pipermail/openstack-dev/2013-March/006827.html
-
-
-def upgrade(migrate_engine):
-    pass
-"""
-
-
-def get_last_migration(base):
-    path = os.path.join(*tuple(base + ['[0-9]*.py']))
-    migrations = sorted([os.path.split(fn)[-1] for fn in glob.glob(path)])
-    return int(migrations[-1].split('_')[0])
-
-
-def reserve_migrations(base, number, git_add):
-    last = get_last_migration(base)
-    for i in range(last + 1, last + number + 1):
-        name = '%03i_placeholder.py' % i
-        path = os.path.join(*tuple(base + [name]))
-        with open(path, 'w') as f:
-            f.write(STUB)
-        print('Created %s' % path)
-        if git_add:
-            subprocess.call('git add %s' % path, shell=True)
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument('-n', '--number', default=10,
-                        type=int,
-                        help='Number of migrations to reserve')
-    parser.add_argument('-g', '--git-add', action='store_const',
-                        const=True, default=False,
-                        help='Automatically git-add new migrations')
-    parser.add_argument('-a', '--api', action='store_const',
-                        const=True, default=False,
-                        help='Reserve migrations for the API database')
-    args = parser.parse_args()
-    if args.api:
-        base = API_BASE
-    else:
-        base = BASE
-    reserve_migrations(base, args.number, args.git_add)
-
-
-if __name__ == '__main__':
-    main()
diff --git a/tox.ini b/tox.ini
index 694ecce506..6dfde9afbb 100644
--- a/tox.ini
+++ b/tox.ini
@@ -32,11 +32,7 @@ passenv =
 # there is also secret magic in subunit-trace which lets you run in a fail only
 # mode. To do this define the TRACE_FAILONLY environmental variable.
 commands =
-# NOTE(gibi): The group-regex runs the matching tests in the same executor.
-# These tests runs against a real mysql instance and in an IO deprived CI VM they tend to time out.
-# See bug https://launchpad.net/bugs/1823251 for details.
-# By running them in the same executor we can spread the IO load of these tests in time.
-  stestr --group-regex=nova\.tests\.unit\.db\.test_migrations\.TestNovaMigrationsMySQL run {posargs}
+  stestr run {posargs}
   env TEST_OSPROFILER=1 stestr run --combine --no-discover 'nova.tests.unit.test_profiler'
   stestr slowest
 
@@ -96,13 +92,7 @@ deps =
   {[testenv]deps}
   openstack-placement>=1.0.0
 commands =
-# NOTE(gibi): The group-regex runs the matching tests in the same executor.
-# These tests runs against a real db instance and in an IO deprived CI VM they tend to time out.
-# See bug https://launchpad.net/bugs/1823251 for details.
-# By running them in the same executor we can spread the IO load of these tests in time.
-# NOTE(gibi): I was not able to group only the mysql tests this way as regex
-# TestNovaAPIMigrations.*MySQL does not do what I expect
-  stestr --group-regex=nova\.tests\.functional\.db\.api\.test_migrations\.TestNovaAPIMigrations --test-path=./nova/tests/functional run {posargs}
+  stestr --test-path=./nova/tests/functional run {posargs}
   stestr slowest
 
 [testenv:functional-py36]