diff options
| -rw-r--r-- | oslo.db/locale/en_GB/LC_MESSAGES/oslo.db-log-info.po | 18 | ||||
| -rw-r--r-- | oslo.db/locale/oslo.db-log-info.pot | 12 | ||||
| -rw-r--r-- | oslo/db/exception.py | 8 | ||||
| -rw-r--r-- | oslo/db/sqlalchemy/provision.py | 522 | ||||
| -rw-r--r-- | oslo/db/sqlalchemy/session.py | 3 | ||||
| -rw-r--r-- | oslo/db/sqlalchemy/test_base.py | 87 | ||||
| -rw-r--r-- | oslo/db/sqlalchemy/test_migrations.py | 75 | ||||
| -rw-r--r-- | oslo/db/sqlalchemy/utils.py | 34 | ||||
| -rw-r--r-- | requirements.txt | 2 | ||||
| -rw-r--r-- | test-requirements-py2.txt | 4 | ||||
| -rw-r--r-- | test-requirements-py3.txt | 4 | ||||
| -rw-r--r-- | tests/sqlalchemy/test_migrations.py | 12 | ||||
| -rw-r--r-- | tests/sqlalchemy/test_utils.py | 59 | ||||
| -rw-r--r-- | tox.ini | 1 |
14 files changed, 648 insertions, 193 deletions
diff --git a/oslo.db/locale/en_GB/LC_MESSAGES/oslo.db-log-info.po b/oslo.db/locale/en_GB/LC_MESSAGES/oslo.db-log-info.po index fc898eb..071fc4f 100644 --- a/oslo.db/locale/en_GB/LC_MESSAGES/oslo.db-log-info.po +++ b/oslo.db/locale/en_GB/LC_MESSAGES/oslo.db-log-info.po @@ -8,9 +8,9 @@ msgid "" msgstr "" "Project-Id-Version: oslo.db\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2014-09-12 06:00+0000\n" -"PO-Revision-Date: 2014-09-02 09:18+0000\n" -"Last-Translator: Andi Chandler <andi@gowling.com>\n" +"POT-Creation-Date: 2014-11-10 06:02+0000\n" +"PO-Revision-Date: 2014-11-08 14:37+0000\n" +"Last-Translator: openstackjenkins <jenkins@openstack.org>\n" "Language-Team: English (United Kingdom) (http://www.transifex.com/projects/p/" "oslodb/language/en_GB/)\n" "Language: en_GB\n" @@ -20,12 +20,12 @@ msgstr "" "Generated-By: Babel 1.3\n" "Plural-Forms: nplurals=2; plural=(n != 1);\n" -#: oslo/db/sqlalchemy/utils.py:433 +#: oslo/db/sqlalchemy/provision.py:151 oslo/db/sqlalchemy/provision.py:162 #, python-format -msgid "Deleting duplicated row with id: %(id)s from table: %(table)s" -msgstr "Deleting duplicated row with id: %(id)s from table: %(table)s" +msgid "The %(dbapi)s backend is unavailable: %(err)s" +msgstr "" -#: oslo/db/sqlalchemy/utils.py:694 +#: oslo/db/sqlalchemy/utils.py:415 #, python-format -msgid "The %(backend)s backend is unavailable: %(exception)s" -msgstr "The %(backend)s backend is unavailable: %(exception)s" +msgid "Deleting duplicated row with id: %(id)s from table: %(table)s" +msgstr "Deleting duplicated row with id: %(id)s from table: %(table)s" diff --git a/oslo.db/locale/oslo.db-log-info.pot b/oslo.db/locale/oslo.db-log-info.pot index 66f8976..aee920a 100644 --- a/oslo.db/locale/oslo.db-log-info.pot +++ b/oslo.db/locale/oslo.db-log-info.pot @@ -6,9 +6,9 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: oslo.db 0.3.0.70.g69f16bf\n" +"Project-Id-Version: oslo.db 1.0.1.49.g04235f9\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2014-08-05 06:03+0000\n" +"POT-Creation-Date: 2014-11-10 06:02+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language-Team: LANGUAGE <LL@li.org>\n" @@ -17,13 +17,13 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 1.3\n" -#: oslo/db/sqlalchemy/utils.py:428 +#: oslo/db/sqlalchemy/provision.py:151 oslo/db/sqlalchemy/provision.py:162 #, python-format -msgid "Deleting duplicated row with id: %(id)s from table: %(table)s" +msgid "The %(dbapi)s backend is unavailable: %(err)s" msgstr "" -#: oslo/db/sqlalchemy/utils.py:689 +#: oslo/db/sqlalchemy/utils.py:415 #, python-format -msgid "The %(backend)s backend is unavailable: %(exception)s" +msgid "Deleting duplicated row with id: %(id)s from table: %(table)s" msgstr "" diff --git a/oslo/db/exception.py b/oslo/db/exception.py index e549bc3..a96ad76 100644 --- a/oslo/db/exception.py +++ b/oslo/db/exception.py @@ -163,3 +163,11 @@ class InvalidSortKey(Exception): class ColumnError(Exception): """Error raised when no column or an invalid column is found.""" + + +class BackendNotAvailable(Exception): + """Error raised when a particular database backend is not available + + within a test suite. + + """ diff --git a/oslo/db/sqlalchemy/provision.py b/oslo/db/sqlalchemy/provision.py index f1aa2cd..260cb5c 100644 --- a/oslo/db/sqlalchemy/provision.py +++ b/oslo/db/sqlalchemy/provision.py @@ -15,111 +15,459 @@ """Provision test environment for specific DB backends""" +import abc import argparse -import copy import logging import os import random +import re import string +import six from six import moves import sqlalchemy +from sqlalchemy.engine import url as sa_url -from oslo.db import exception as exc - +from oslo.db._i18n import _LI +from oslo.db import exception +from oslo.db.sqlalchemy import session +from oslo.db.sqlalchemy import utils LOG = logging.getLogger(__name__) -def get_engine(uri): - """Engine creation +class ProvisionedDatabase(object): + """Represent a single database node that can be used for testing in + + a serialized fashion. + + ``ProvisionedDatabase`` includes features for full lifecycle management + of a node, in a way that is context-specific. Depending on how the + test environment runs, ``ProvisionedDatabase`` should know if it needs + to create and drop databases or if it is making use of a database that + is maintained by an external process. - Call the function without arguments to get admin connection. Admin - connection required to create temporary database for each - particular test. Otherwise use existing connection to recreate - connection to the temporary database. """ - return sqlalchemy.create_engine(uri, poolclass=sqlalchemy.pool.NullPool) + def __init__(self, database_type): + self.backend = Backend.backend_for_database_type(database_type) + self.db_token = _random_ident() -def _execute_sql(engine, sql, driver): - """Initialize connection, execute sql query and close it.""" - try: - with engine.connect() as conn: - if driver == 'postgresql': - conn.connection.set_isolation_level(0) - for s in sql: - conn.execute(s) - except sqlalchemy.exc.OperationalError: - msg = ('%s does not match database admin ' - 'credentials or database does not exist.') - LOG.exception(msg, engine.url) - raise exc.DBConnectionError(msg % engine.url) - - -def create_database(engine): - """Provide temporary database for each particular test.""" - driver = engine.name - - database = ''.join(random.choice(string.ascii_lowercase) - for i in moves.range(10)) - - if driver == 'sqlite': - database = '/tmp/%s' % database - elif driver in ['mysql', 'postgresql']: - sql = 'create database %s;' % database - _execute_sql(engine, [sql], driver) - else: - raise ValueError('Unsupported RDBMS %s' % driver) - - # Both shallow and deep copies may lead to surprising behaviour - # without knowing the implementation of sqlalchemy.engine.url. - # Use a shallow copy here, since we're only overriding a single - # property, invoking __str__ and then discarding our copy. This - # is currently safe and _should_ remain safe into the future. - new_url = copy.copy(engine.url) - - new_url.database = database - return str(new_url) - - -def drop_database(admin_engine, current_uri): - """Drop temporary database after each particular test.""" - - engine = get_engine(current_uri) - driver = engine.name - - if driver == 'sqlite': + self.backend.create_named_database(self.db_token) + self.engine = self.backend.provisioned_engine(self.db_token) + + def dispose(self): + self.engine.dispose() + self.backend.drop_named_database(self.db_token) + + +class Backend(object): + """Represent a particular database backend that may be provisionable. + + The ``Backend`` object maintains a database type (e.g. database without + specific driver type, such as "sqlite", "postgresql", etc.), + a target URL, a base ``Engine`` for that URL object that can be used + to provision databases and a ``BackendImpl`` which knows how to perform + operations against this type of ``Engine``. + + """ + + backends_by_database_type = {} + + def __init__(self, database_type, url): + self.database_type = database_type + self.url = url + self.verified = False + self.engine = None + self.impl = BackendImpl.impl(database_type) + Backend.backends_by_database_type[database_type] = self + + @classmethod + def backend_for_database_type(cls, database_type): + """Return and verify the ``Backend`` for the given database type. + + Creates the engine if it does not already exist and raises + ``BackendNotAvailable`` if it cannot be produced. + + :return: a base ``Engine`` that allows provisioning of databases. + + :raises: ``BackendNotAvailable``, if an engine for this backend + cannot be produced. + + """ + try: + backend = cls.backends_by_database_type[database_type] + except KeyError: + raise exception.BackendNotAvailable(database_type) + else: + return backend._verify() + + @classmethod + def all_viable_backends(cls): + """Return an iterator of all ``Backend`` objects that are present + + and provisionable. + + """ + + for backend in cls.backends_by_database_type.values(): + try: + yield backend._verify() + except exception.BackendNotAvailable: + pass + + def _verify(self): + """Verify that this ``Backend`` is available and provisionable. + + :return: this ``Backend`` + + :raises: ``BackendNotAvailable`` if the backend is not available. + + """ + + if not self.verified: + try: + eng = self._ensure_backend_available(self.url) + except exception.BackendNotAvailable: + raise + else: + self.engine = eng + finally: + self.verified = True + if self.engine is None: + raise exception.BackendNotAvailable(self.database_type) + return self + + @classmethod + def _ensure_backend_available(cls, url): + url = sa_url.make_url(str(url)) try: - os.remove(engine.url.database) - except OSError: - pass - elif driver in ['mysql', 'postgresql']: - sql = 'drop database %s;' % engine.url.database - _execute_sql(admin_engine, [sql], driver) - else: - raise ValueError('Unsupported RDBMS %s' % driver) + eng = sqlalchemy.create_engine(url) + except ImportError as i_e: + # SQLAlchemy performs an "import" of the DBAPI module + # within create_engine(). So if ibm_db_sa, cx_oracle etc. + # isn't installed, we get an ImportError here. + LOG.info( + _LI("The %(dbapi)s backend is unavailable: %(err)s"), + dict(dbapi=url.drivername, err=i_e)) + raise exception.BackendNotAvailable("No DBAPI installed") + else: + try: + conn = eng.connect() + except sqlalchemy.exc.DBAPIError as d_e: + # upon connect, SQLAlchemy calls dbapi.connect(). This + # usually raises OperationalError and should always at + # least raise a SQLAlchemy-wrapped DBAPI Error. + LOG.info( + _LI("The %(dbapi)s backend is unavailable: %(err)s"), + dict(dbapi=url.drivername, err=d_e) + ) + raise exception.BackendNotAvailable("Could not connect") + else: + conn.close() + return eng + + def create_named_database(self, ident): + """Create a database with the given name.""" + + self.impl.create_named_database(self.engine, ident) + + def drop_named_database(self, ident, conditional=False): + """Drop a database with the given name.""" + + self.impl.drop_named_database( + self.engine, ident, + conditional=conditional) + + def database_exists(self, ident): + """Return True if a database of the given name exists.""" + + return self.impl.database_exists(self.engine, ident) + + def provisioned_engine(self, ident): + """Given the URL of a particular database backend and the string + + name of a particular 'database' within that backend, return + an Engine instance whose connections will refer directly to the + named database. + + For hostname-based URLs, this typically involves switching just the + 'database' portion of the URL with the given name and creating + an engine. + + For URLs that instead deal with DSNs, the rules may be more custom; + for example, the engine may need to connect to the root URL and + then emit a command to switch to the named database. + + """ + return self.impl.provisioned_engine(self.url, ident) + + @classmethod + def _setup(cls): + """Initial startup feature will scan the environment for configured + + URLs and place them into the list of URLs we will use for provisioning. + + This searches through OS_TEST_DBAPI_ADMIN_CONNECTION for URLs. If + not present, we set up URLs based on the "opportunstic" convention, + e.g. username+password = "openstack_citest". + + The provisioning system will then use or discard these URLs as they + are requested, based on whether or not the target database is actually + found to be available. + + """ + configured_urls = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', None) + if configured_urls: + configured_urls = configured_urls.split(";") + else: + configured_urls = [ + impl.create_opportunistic_driver_url() + for impl in BackendImpl.all_impls() + ] + + for url_str in configured_urls: + url = sa_url.make_url(url_str) + m = re.match(r'([^+]+?)(?:\+(.+))?$', url.drivername) + database_type, drivertype = m.group(1, 2) + Backend(database_type, url) + + +@six.add_metaclass(abc.ABCMeta) +class BackendImpl(object): + """Provide database-specific implementations of key provisioning + + functions. + + ``BackendImpl`` is owned by a ``Backend`` instance which delegates + to it for all database-specific features. + + """ + + @classmethod + def all_impls(cls): + """Return an iterator of all possible BackendImpl objects. + + These are BackendImpls that are implemented, but not + necessarily provisionable. + + """ + for database_type in cls.impl.reg: + if database_type == '*': + continue + yield BackendImpl.impl(database_type) + @utils.dispatch_for_dialect("*") + def impl(drivername): + """Return a ``BackendImpl`` instance corresponding to the -def main(): - """Controller to handle commands + given driver name. + + This is a dispatched method which will refer to the constructor + of implementing subclasses. + + """ + raise NotImplementedError( + "No provision impl available for driver: %s" % drivername) + + def __init__(self, drivername): + self.drivername = drivername + + @abc.abstractmethod + def create_opportunistic_driver_url(self): + """Produce a string url known as the 'opportunistic' URL. + + This URL is one that corresponds to an established Openstack + convention for a pre-established database login, which, when + detected as available in the local environment, is automatically + used as a test platform for a specific type of driver. + + """ + + @abc.abstractmethod + def create_named_database(self, engine, ident): + """Create a database with the given name.""" + + @abc.abstractmethod + def drop_named_database(self, engine, ident, conditional=False): + """Drop a database with the given name.""" + + def provisioned_engine(self, base_url, ident): + """Return a provisioned engine. + + Given the URL of a particular database backend and the string + name of a particular 'database' within that backend, return + an Engine instance whose connections will refer directly to the + named database. + + For hostname-based URLs, this typically involves switching just the + 'database' portion of the URL with the given name and creating + an engine. + + For URLs that instead deal with DSNs, the rules may be more custom; + for example, the engine may need to connect to the root URL and + then emit a command to switch to the named database. + + """ + + url = sa_url.make_url(str(base_url)) + url.database = ident + return session.create_engine( + url, + logging_name="%s@%s" % (self.drivername, ident)) + + +@BackendImpl.impl.dispatch_for("mysql") +class MySQLBackendImpl(BackendImpl): + def create_opportunistic_driver_url(self): + return "mysql://openstack_citest:openstack_citest@localhost/" + + def create_named_database(self, engine, ident): + with engine.connect() as conn: + conn.execute("CREATE DATABASE %s" % ident) + + def drop_named_database(self, engine, ident, conditional=False): + with engine.connect() as conn: + if not conditional or self.database_exists(conn, ident): + conn.execute("DROP DATABASE %s" % ident) + + def database_exists(self, engine, ident): + return bool(engine.scalar("SHOW DATABASES LIKE '%s'" % ident)) + + +@BackendImpl.impl.dispatch_for("sqlite") +class SQLiteBackendImpl(BackendImpl): + def create_opportunistic_driver_url(self): + return "sqlite://" + + def create_named_database(self, engine, ident): + url = self._provisioned_database_url(engine.url, ident) + eng = sqlalchemy.create_engine(url) + eng.connect().close() + + def provisioned_engine(self, base_url, ident): + return session.create_engine( + self._provisioned_database_url(base_url, ident)) + + def drop_named_database(self, engine, ident, conditional=False): + url = self._provisioned_database_url(engine.url, ident) + filename = url.database + if filename and (not conditional or os.access(filename, os.F_OK)): + os.remove(filename) + + def database_exists(self, engine, ident): + url = self._provisioned_database_url(engine.url, ident) + filename = url.database + return not filename or os.access(filename, os.F_OK) + + def _provisioned_database_url(self, base_url, ident): + if base_url.database: + return sa_url.make_url("sqlite:////tmp/%s.db" % ident) + else: + return base_url + + +@BackendImpl.impl.dispatch_for("postgresql") +class PostgresqlBackendImpl(BackendImpl): + def create_opportunistic_driver_url(self): + return "postgresql://openstack_citest:openstack_citest"\ + "@localhost/postgres" + + def create_named_database(self, engine, ident): + with engine.connect().execution_options( + isolation_level="AUTOCOMMIT") as conn: + conn.execute("CREATE DATABASE %s" % ident) + + def drop_named_database(self, engine, ident, conditional=False): + with engine.connect().execution_options( + isolation_level="AUTOCOMMIT") as conn: + self._close_out_database_users(conn, ident) + if conditional: + conn.execute("DROP DATABASE IF EXISTS %s" % ident) + else: + conn.execute("DROP DATABASE %s" % ident) + + def database_exists(self, engine, ident): + return bool( + engine.scalar( + sqlalchemy.text( + "select datname from pg_database " + "where datname=:name"), name=ident) + ) + + def _close_out_database_users(self, conn, ident): + """Attempt to guarantee a database can be dropped. + + Optional feature which guarantees no connections with our + username are attached to the DB we're going to drop. + + This method has caveats; for one, the 'pid' column was named + 'procpid' prior to Postgresql 9.2. But more critically, + prior to 9.2 this operation required superuser permissions, + even if the connections we're closing are under the same username + as us. In more recent versions this restriction has been + lifted for same-user connections. + + """ + if conn.dialect.server_version_info >= (9, 2): + conn.execute( + sqlalchemy.text( + "select pg_terminate_backend(pid) " + "from pg_stat_activity " + "where usename=current_user and " + "pid != pg_backend_pid() " + "and datname=:dname" + ), dname=ident) + + +def _random_ident(): + return ''.join( + random.choice(string.ascii_lowercase) + for i in moves.range(10)) + + +def _echo_cmd(args): + idents = [_random_ident() for i in moves.range(args.instances_count)] + print("\n".join(idents)) + + +def _create_cmd(args): + idents = [_random_ident() for i in moves.range(args.instances_count)] + + for backend in Backend.all_viable_backends(): + for ident in idents: + backend.create_named_database(ident) + + print("\n".join(idents)) + + +def _drop_cmd(args): + for backend in Backend.all_viable_backends(): + for ident in args.instances: + backend.drop_named_database(ident, args.conditional) + +Backend._setup() + + +def main(argv=None): + """Command line interface to create/drop databases. ::create: Create test database with random names. ::drop: Drop database created by previous command. + ::echo: create random names and display them; don't create. """ parser = argparse.ArgumentParser( description='Controller to handle database creation and dropping' ' commands.', - epilog='Under normal circumstances is not used directly.' - ' Used in .testr.conf to automate test database creation' - ' and dropping processes.') + epilog='Typically called by the test runner, e.g. shell script, ' + 'testr runner via .testr.conf, or other system.') subparsers = parser.add_subparsers( help='Subcommands to manipulate temporary test databases.') create = subparsers.add_parser( 'create', help='Create temporary test databases.') - create.set_defaults(which='create') + create.set_defaults(which=_create_cmd) create.add_argument( 'instances_count', type=int, @@ -128,25 +476,31 @@ def main(): drop = subparsers.add_parser( 'drop', help='Drop temporary test databases.') - drop.set_defaults(which='drop') + drop.set_defaults(which=_drop_cmd) drop.add_argument( 'instances', nargs='+', help='List of databases uri to be dropped.') + drop.add_argument( + '--conditional', + action="store_true", + help="Check if database exists first before dropping" + ) - args = parser.parse_args() + echo = subparsers.add_parser( + 'echo', + help="Create random database names and display only." + ) + echo.set_defaults(which=_echo_cmd) + echo.add_argument( + 'instances_count', + type=int, + help='Number of identifiers to create.') - connection_string = os.getenv('OS_TEST_DBAPI_ADMIN_CONNECTION', - 'sqlite://') - engine = get_engine(connection_string) - which = args.which + args = parser.parse_args(argv) - if which == "create": - for i in range(int(args.instances_count)): - print(create_database(engine)) - elif which == "drop": - for db in args.instances: - drop_database(engine, db) + cmd = args.which + cmd(args) if __name__ == "__main__": diff --git a/oslo/db/sqlalchemy/session.py b/oslo/db/sqlalchemy/session.py index b24e06b..937494c 100644 --- a/oslo/db/sqlalchemy/session.py +++ b/oslo/db/sqlalchemy/session.py @@ -368,7 +368,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, connection_debug=0, max_pool_size=None, max_overflow=None, pool_timeout=None, sqlite_synchronous=True, connection_trace=False, max_retries=10, retry_interval=10, - thread_checkin=True): + thread_checkin=True, logging_name=None): """Return a new SQLAlchemy engine.""" url = sqlalchemy.engine.url.make_url(sql_connection) @@ -377,6 +377,7 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, "pool_recycle": idle_timeout, 'convert_unicode': True, 'connect_args': {}, + 'logging_name': logging_name } _setup_logging(connection_debug) diff --git a/oslo/db/sqlalchemy/test_base.py b/oslo/db/sqlalchemy/test_base.py index 02a356a..d483fad 100644 --- a/oslo/db/sqlalchemy/test_base.py +++ b/oslo/db/sqlalchemy/test_base.py @@ -13,9 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import abc -import os - import fixtures try: @@ -24,9 +21,10 @@ except ImportError: raise NameError('Oslotest is not installed. Please add oslotest in your' ' test-requirements') + import six -import testtools +from oslo.db import exception from oslo.db.sqlalchemy import provision from oslo.db.sqlalchemy import session from oslo.db.sqlalchemy import utils @@ -41,8 +39,12 @@ class DbFixture(fixtures.Fixture): credentials for specific backend. """ - def _get_uri(self): - return os.getenv('OS_TEST_DBAPI_CONNECTION', 'sqlite://') + DRIVER = "sqlite" + + # these names are deprecated, and are not used by DbFixture. + # they are here for backwards compatibility with test suites that + # are referring to them directly. + DBNAME = PASSWORD = USERNAME = 'openstack_citest' def __init__(self, test): super(DbFixture, self).__init__() @@ -52,9 +54,17 @@ class DbFixture(fixtures.Fixture): def setUp(self): super(DbFixture, self).setUp() - self.test.engine = session.create_engine(self._get_uri()) - self.addCleanup(self.test.engine.dispose) - self.test.sessionmaker = session.get_maker(self.test.engine) + try: + self.provision = provision.ProvisionedDatabase(self.DRIVER) + self.addCleanup(self.provision.dispose) + except exception.BackendNotAvailable: + msg = '%s backend is not available.' % self.DRIVER + return self.test.skip(msg) + else: + self.test.engine = self.provision.engine + self.addCleanup(setattr, self.test, 'engine', None) + self.test.sessionmaker = session.get_maker(self.test.engine) + self.addCleanup(setattr, self.test, 'sessionmaker', None) class DbTestCase(test_base.BaseTestCase): @@ -72,6 +82,9 @@ class DbTestCase(test_base.BaseTestCase): self.useFixture(self.FIXTURE(self)) +class OpportunisticTestCase(DbTestCase): + """Placeholder for backwards compatibility.""" + ALLOWED_DIALECTS = ['sqlite', 'mysql', 'postgresql'] @@ -98,64 +111,12 @@ def backend_specific(*dialects): return wrap -@six.add_metaclass(abc.ABCMeta) -class OpportunisticFixture(DbFixture): - """Base fixture to use default CI databases. - - The databases exist in OpenStack CI infrastructure. But for the - correct functioning in local environment the databases must be - created manually. - """ - - DRIVER = abc.abstractproperty(lambda: None) - DBNAME = PASSWORD = USERNAME = 'openstack_citest' - _uri = None - - def _get_uri(self): - if self._uri is not None: - return self._uri - - credentials = { - 'backend': self.DRIVER, - 'user': self.USERNAME, - 'passwd': self.PASSWORD, - 'database': self.DBNAME} - if self.DRIVER and not utils.is_backend_avail(**credentials): - msg = '%s backend is not available.' % self.DRIVER - raise testtools.testcase.TestSkipped(msg) - - self._provisioning_engine = provision.get_engine( - utils.get_connect_string(backend=self.DRIVER, - user=self.USERNAME, - passwd=self.PASSWORD, - database=self.DBNAME) - ) - self._uri = provision.create_database(self._provisioning_engine) - self.addCleanup( - provision.drop_database, self._provisioning_engine, self._uri) - self.addCleanup(setattr, self, '_uri', None) - return self._uri - - -@six.add_metaclass(abc.ABCMeta) -class OpportunisticTestCase(DbTestCase): - """Base test case to use default CI databases. - - The subclasses of the test case are running only when openstack_citest - database is available otherwise tests will be skipped. - """ - - FIXTURE = abc.abstractproperty(lambda: None) - - -class MySQLOpportunisticFixture(OpportunisticFixture): +class MySQLOpportunisticFixture(DbFixture): DRIVER = 'mysql' - DBNAME = '' # connect to MySQL server, but not to the openstack_citest db -class PostgreSQLOpportunisticFixture(OpportunisticFixture): +class PostgreSQLOpportunisticFixture(DbFixture): DRIVER = 'postgresql' - DBNAME = 'postgres' # PostgreSQL requires the db name here,use service one class MySQLOpportunisticTestCase(OpportunisticTestCase): diff --git a/oslo/db/sqlalchemy/test_migrations.py b/oslo/db/sqlalchemy/test_migrations.py index 28b613f..c6bd146 100644 --- a/oslo/db/sqlalchemy/test_migrations.py +++ b/oslo/db/sqlalchemy/test_migrations.py @@ -15,6 +15,7 @@ # under the License. import abc +import collections import logging import pprint @@ -510,6 +511,73 @@ class ModelsMigrationsSync(object): for table in tbs: conn.execute(schema.DropTable(table)) + FKInfo = collections.namedtuple('fk_info', ['constrained_columns', + 'referred_table', + 'referred_columns']) + + def check_foreign_keys(self, metadata, bind): + """Compare foreign keys between model and db table. + + :returns: a list that contains information about: + + * should be a new key added or removed existing, + * name of that key, + * source table, + * referred table, + * constrained columns, + * referred columns + + Output:: + + [('drop_key', + 'testtbl_fk_check_fkey', + 'testtbl', + fk_info(constrained_columns=(u'fk_check',), + referred_table=u'table', + referred_columns=(u'fk_check',)))] + + """ + + diff = [] + insp = sqlalchemy.engine.reflection.Inspector.from_engine(bind) + # Get all tables from db + db_tables = insp.get_table_names() + # Get all tables from models + model_tables = metadata.tables + for table in db_tables: + if table not in model_tables: + continue + # Get all necessary information about key of current table from db + fk_db = dict((self._get_fk_info_from_db(i), i['name']) + for i in insp.get_foreign_keys(table)) + fk_db_set = set(fk_db.keys()) + # Get all necessary information about key of current table from + # models + fk_models = dict((self._get_fk_info_from_model(fk), fk) + for fk in model_tables[table].foreign_keys) + fk_models_set = set(fk_models.keys()) + for key in (fk_db_set - fk_models_set): + diff.append(('drop_key', fk_db[key], table, key)) + LOG.info(("Detected removed foreign key %(fk)r on " + "table %(table)r"), {'fk': fk_db[key], + 'table': table}) + for key in (fk_models_set - fk_db_set): + diff.append(('add_key', fk_models[key], key)) + LOG.info(( + "Detected added foreign key for column %(fk)r on table " + "%(table)r"), {'fk': fk_models[key].column.name, + 'table': table}) + return diff + + def _get_fk_info_from_db(self, fk): + return self.FKInfo(tuple(fk['constrained_columns']), + fk['referred_table'], + tuple(fk['referred_columns'])) + + def _get_fk_info_from_model(self, fk): + return self.FKInfo((fk.parent.name,), fk.column.table.name, + (fk.column.name,)) + def test_models_sync(self): # recent versions of sqlalchemy and alembic are needed for running of # this test, but we already have them in requirements @@ -534,8 +602,11 @@ class ModelsMigrationsSync(object): mc = alembic.migration.MigrationContext.configure(conn, opts=opts) # compare schemas and fail with diff, if it's not empty - diff = alembic.autogenerate.compare_metadata(mc, - self.get_metadata()) + diff1 = alembic.autogenerate.compare_metadata(mc, + self.get_metadata()) + diff2 = self.check_foreign_keys(self.get_metadata(), + self.get_engine()) + diff = diff1 + diff2 if diff: msg = pprint.pformat(diff, indent=2, width=20) self.fail( diff --git a/oslo/db/sqlalchemy/utils.py b/oslo/db/sqlalchemy/utils.py index e2537a6..ccac7cd 100644 --- a/oslo/db/sqlalchemy/utils.py +++ b/oslo/db/sqlalchemy/utils.py @@ -649,6 +649,11 @@ def get_connect_string(backend, database, user=None, passwd=None, Try to get a connection with a very specific set of values, if we get these then we'll run the tests, otherwise they are skipped + + DEPRECATED: this function is deprecated and will be removed from oslo.db + in a few releases. Please use the provisioning system for dealing + with URLs and database provisioning. + """ args = {'backend': backend, 'user': user, @@ -663,22 +668,25 @@ def get_connect_string(backend, database, user=None, passwd=None, def is_backend_avail(backend, database, user=None, passwd=None): + """Return True if the given backend is available. + + + DEPRECATED: this function is deprecated and will be removed from oslo.db + in a few releases. Please use the provisioning system to access + databases based on backend availability. + + """ + from oslo.db.sqlalchemy import provision + + connect_uri = get_connect_string(backend=backend, + database=database, + user=user, + passwd=passwd) try: - connect_uri = get_connect_string(backend=backend, - database=database, - user=user, - passwd=passwd) - engine = sqlalchemy.create_engine(connect_uri) - connection = engine.connect() - except Exception as e: - # intentionally catch all to handle exceptions even if we don't - # have any backend code loaded. - msg = _LI("The %(backend)s backend is unavailable: %(exception)s") - LOG.info(msg, {"backend": backend, "exception": e}) + provision.Backend._ensure_backend_available(connect_uri) + except exception.BackendNotAvailable: return False else: - connection.close() - engine.dispose() return True diff --git a/requirements.txt b/requirements.txt index 24d3452..cc50660 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,5 +10,5 @@ oslo.config>=1.4.0 # Apache-2.0 oslo.utils>=1.0.0 # Apache-2.0 SQLAlchemy>=0.8.4,<=0.8.99,>=0.9.7,<=0.9.99 sqlalchemy-migrate>=0.9.1,!=0.9.2 -stevedore>=1.0.0 # Apache-2.0 +stevedore>=1.1.0 # Apache-2.0 six>=1.7.0 diff --git a/test-requirements-py2.txt b/test-requirements-py2.txt index efb80d4..13cea90 100644 --- a/test-requirements-py2.txt +++ b/test-requirements-py2.txt @@ -13,7 +13,7 @@ psycopg2 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.2.0 # Apache-2.0 -oslotest>=1.1.0 # Apache-2.0 +oslotest>=1.2.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 -testtools>=0.9.34 +testtools>=0.9.36 diff --git a/test-requirements-py3.txt b/test-requirements-py3.txt index 3d33d0e..4f195da 100644 --- a/test-requirements-py3.txt +++ b/test-requirements-py3.txt @@ -12,10 +12,10 @@ psycopg2 python-subunit>=0.0.18 sphinx>=1.1.2,!=1.2.0,!=1.3b1,<1.3 oslosphinx>=2.2.0 # Apache-2.0 -oslotest>=1.1.0 # Apache-2.0 +oslotest>=1.2.0 # Apache-2.0 testrepository>=0.0.18 testscenarios>=0.4 -testtools>=0.9.34 +testtools>=0.9.36 # TODO(harlowja): add in pymysql when able to... # https://review.openstack.org/#/c/123737 diff --git a/tests/sqlalchemy/test_migrations.py b/tests/sqlalchemy/test_migrations.py index abb5f8a..db6c9c8 100644 --- a/tests/sqlalchemy/test_migrations.py +++ b/tests/sqlalchemy/test_migrations.py @@ -194,6 +194,7 @@ class ModelsMigrationSyncMixin(test.BaseTestCase): sa.Column('defaulttest4', sa.Enum('first', 'second', name='testenum'), server_default="first"), + sa.Column('fk_check', sa.String(36), nullable=False), sa.UniqueConstraint('spam', 'eggs', name='uniq_cons'), ) @@ -210,6 +211,7 @@ class ModelsMigrationSyncMixin(test.BaseTestCase): eggs = sa.Column('eggs', sa.DateTime) foo = sa.Column('foo', sa.Boolean, server_default=sa.sql.expression.true()) + fk_check = sa.Column('fk_check', sa.String(36), nullable=False) bool_wo_default = sa.Column('bool_wo_default', sa.Boolean) defaulttest = sa.Column('defaulttest', sa.Integer, server_default='5') @@ -243,6 +245,12 @@ class ModelsMigrationSyncMixin(test.BaseTestCase): def _test_models_not_sync(self): self.metadata_migrations.clear() sa.Table( + 'table', self.metadata_migrations, + sa.Column('fk_check', sa.String(36), nullable=False), + sa.PrimaryKeyConstraint('fk_check'), + mysql_engine='InnoDB' + ) + sa.Table( 'testtbl', self.metadata_migrations, sa.Column('id', sa.Integer, primary_key=True), sa.Column('spam', sa.String(8), nullable=True), @@ -257,7 +265,10 @@ class ModelsMigrationSyncMixin(test.BaseTestCase): sa.Column('defaulttest4', sa.Enum('first', 'second', name='testenum'), server_default="first"), + sa.Column('fk_check', sa.String(36), nullable=False), sa.UniqueConstraint('spam', 'foo', name='uniq_cons'), + sa.ForeignKeyConstraint(['fk_check'], ['table.fk_check']), + mysql_engine='InnoDB' ) msg = six.text_type(self.assertRaises(AssertionError, @@ -276,6 +287,7 @@ class ModelsMigrationSyncMixin(test.BaseTestCase): self.assertIn('bool_wo_default', msg) self.assertIn('defaulttest', msg) self.assertIn('defaulttest3', msg) + self.assertIn('drop_key', msg) class ModelsMigrationsSyncMysql(ModelsMigrationSyncMixin, diff --git a/tests/sqlalchemy/test_utils.py b/tests/sqlalchemy/test_utils.py index acf31d0..c13fe9d 100644 --- a/tests/sqlalchemy/test_utils.py +++ b/tests/sqlalchemy/test_utils.py @@ -26,13 +26,15 @@ from sqlalchemy.dialects import mysql from sqlalchemy import Boolean, Index, Integer, DateTime, String, SmallInteger from sqlalchemy import MetaData, Table, Column, ForeignKey from sqlalchemy.engine import reflection -from sqlalchemy.exc import ResourceClosedError +from sqlalchemy.engine import url as sa_url +from sqlalchemy.exc import OperationalError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.sql import select from sqlalchemy.types import UserDefinedType, NullType from oslo.db import exception from oslo.db.sqlalchemy import models +from oslo.db.sqlalchemy import provision from oslo.db.sqlalchemy import session from oslo.db.sqlalchemy import test_base as db_test_base from oslo.db.sqlalchemy import utils @@ -537,19 +539,58 @@ class TestConnectionUtils(test_utils.BaseTestCase): def test_is_backend_unavail(self): log = self.useFixture(fixtures.FakeLogger()) - error_cause = ('This result object does not return rows. It has been' - 'closed automatically.') - error_msg = ("The %s backend is unavailable: %s\n" % - ('mysql', error_cause)) - + err = OperationalError("Can't connect to database", None, None) + error_msg = "The mysql backend is unavailable: %s\n" % err self.mox.StubOutWithMock(sqlalchemy.engine.base.Engine, 'connect') - sqlalchemy.engine.base.Engine.connect().AndRaise( - ResourceClosedError(error_cause)) + sqlalchemy.engine.base.Engine.connect().AndRaise(err) self.mox.ReplayAll() - self.assertFalse(utils.is_backend_avail(**self.full_credentials)) self.assertEqual(error_msg, log.output) + def test_ensure_backend_available(self): + self.mox.StubOutWithMock(sqlalchemy.engine.base.Engine, 'connect') + fake_connection = self.mox.CreateMockAnything() + fake_connection.close() + sqlalchemy.engine.base.Engine.connect().AndReturn(fake_connection) + self.mox.ReplayAll() + + eng = provision.Backend._ensure_backend_available(self.connect_string) + self.assertIsInstance(eng, sqlalchemy.engine.base.Engine) + self.assertEqual(self.connect_string, str(eng.url)) + + def test_ensure_backend_available_no_connection_raises(self): + log = self.useFixture(fixtures.FakeLogger()) + err = OperationalError("Can't connect to database", None, None) + self.mox.StubOutWithMock(sqlalchemy.engine.base.Engine, 'connect') + sqlalchemy.engine.base.Engine.connect().AndRaise(err) + self.mox.ReplayAll() + + exc = self.assertRaises( + exception.BackendNotAvailable, + provision.Backend._ensure_backend_available, self.connect_string + ) + self.assertEqual("Could not connect", str(exc)) + self.assertEqual( + "The mysql backend is unavailable: %s" % err, + log.output.strip()) + + def test_ensure_backend_available_no_dbapi_raises(self): + log = self.useFixture(fixtures.FakeLogger()) + self.mox.StubOutWithMock(sqlalchemy, 'create_engine') + sqlalchemy.create_engine( + sa_url.make_url(self.connect_string)).AndRaise( + ImportError("Can't import DBAPI module foobar")) + self.mox.ReplayAll() + + exc = self.assertRaises( + exception.BackendNotAvailable, + provision.Backend._ensure_backend_available, self.connect_string + ) + self.assertEqual("No DBAPI installed", str(exc)) + self.assertEqual( + "The mysql backend is unavailable: Can't import " + "DBAPI module foobar", log.output.strip()) + def test_get_db_connection_info(self): conn_pieces = parse.urlparse(self.connect_string) self.assertEqual(utils.get_db_connection_info(conn_pieces), @@ -55,7 +55,6 @@ commands = show-source = True ignore = E123,E125,H305,H307,H803,H904 -builtins = _ exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,build [hacking] |