diff options
31 files changed, 590 insertions, 56 deletions
@@ -18,4 +18,4 @@ ChangeLog .project .pydevproject oslo.db.egg-info/ -doc/source/api +doc/source/reference/api diff --git a/doc/source/conf.py b/doc/source/conf.py index 00f65cb..576e270 100755 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -23,9 +23,16 @@ extensions = [ 'sphinx.ext.autodoc', #'sphinx.ext.intersphinx', 'oslo_config.sphinxext', - 'oslosphinx', + 'openstackdocstheme', 'stevedore.sphinxext' ] +# openstackdocstheme options +repository_name = 'openstack/oslo.db' +bug_project = 'oslo.db' +bug_tag = '' + +# Must set this variable to include year, month, day, hours, and minutes. +html_last_updated_fmt = '%Y-%m-%d %H:%M' # autodoc generation is a bit aggressive and a nuisance when doing heavy # text edit cycles. @@ -66,7 +73,7 @@ modindex_common_prefix = ['oslo_db.'] # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. # html_theme_path = ["."] -# html_theme = '_theme' +html_theme = 'openstackdocs' # html_static_path = ['static'] # Output file base name for HTML help builder. diff --git a/doc/source/contributing.rst b/doc/source/contributing.rst deleted file mode 100644 index ac7b6bc..0000000 --- a/doc/source/contributing.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../CONTRIBUTING.rst diff --git a/doc/source/contributor/index.rst b/doc/source/contributor/index.rst new file mode 100644 index 0000000..b1cd2f3 --- /dev/null +++ b/doc/source/contributor/index.rst @@ -0,0 +1 @@ +.. include:: ../../../CONTRIBUTING.rst diff --git a/doc/source/history.rst b/doc/source/history.rst deleted file mode 100644 index 69ed4fe..0000000 --- a/doc/source/history.rst +++ /dev/null @@ -1 +0,0 @@ -.. include:: ../../ChangeLog diff --git a/doc/source/index.rst b/doc/source/index.rst index de4d909..3876cdd 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -9,19 +9,10 @@ utils. .. toctree:: :maxdepth: 2 - installation - opts - usage - contributing - history - -API Documentation -================= - -.. toctree:: - :maxdepth: 1 - - api/autoindex + install/index + contributor/index + user/index + reference/index Indices and tables ================== diff --git a/doc/source/installation.rst b/doc/source/install/index.rst index 1262160..1262160 100644 --- a/doc/source/installation.rst +++ b/doc/source/install/index.rst diff --git a/doc/source/reference/index.rst b/doc/source/reference/index.rst new file mode 100644 index 0000000..9eb4867 --- /dev/null +++ b/doc/source/reference/index.rst @@ -0,0 +1,18 @@ +.. _using: + +========= +Reference +========= + +.. toctree:: + :maxdepth: 2 + + opts + +API +=== + +.. toctree:: + :maxdepth: 1 + + api/autoindex diff --git a/doc/source/opts.rst b/doc/source/reference/opts.rst index aa6f145..aa6f145 100644 --- a/doc/source/opts.rst +++ b/doc/source/reference/opts.rst diff --git a/doc/source/user/history.rst b/doc/source/user/history.rst new file mode 100644 index 0000000..f69be70 --- /dev/null +++ b/doc/source/user/history.rst @@ -0,0 +1 @@ +.. include:: ../../../ChangeLog diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst new file mode 100644 index 0000000..8cf52c9 --- /dev/null +++ b/doc/source/user/index.rst @@ -0,0 +1,9 @@ +============== + Using oslo.db +============== + +.. toctree:: + :maxdepth: 2 + + usage + history diff --git a/doc/source/usage.rst b/doc/source/user/usage.rst index 1b2d996..2fe883c 100644 --- a/doc/source/usage.rst +++ b/doc/source/user/usage.rst @@ -9,7 +9,7 @@ Session Handling Session handling is achieved using the :mod:`oslo_db.sqlalchemy.enginefacade` system. This module presents a function decorator as well as a -context manager approach to delivering :class:`.Session` as well as +context manager approach to delivering :class:`.session.Session` as well as :class:`.Connection` objects to a function or block. Both calling styles require the use of a context object. This object may @@ -76,7 +76,7 @@ decorator. Each function must receive the context argument: some_writer_api_function(context, 5, 10) -``connection`` modifier can be used when a :class:`.Session` object is not +``connection`` modifier can be used when a :class:`.session.Session` object is not needed, e.g. when `SQLAlchemy Core <http://docs.sqlalchemy.org/en/latest/core/>`_ is preferred: diff --git a/oslo_db/exception.py b/oslo_db/exception.py index e66fe98..c8da996 100644 --- a/oslo_db/exception.py +++ b/oslo_db/exception.py @@ -243,6 +243,10 @@ class DBDataError(DBError): """ +class DBNotSupportedError(DBError): + """Raised when a database backend has raised sqla.exc.NotSupportedError""" + + class InvalidSortKey(Exception): """A sort key destined for database query usage is invalid.""" diff --git a/oslo_db/options.py b/oslo_db/options.py index 1ce2381..824661f 100644 --- a/oslo_db/options.py +++ b/oslo_db/options.py @@ -44,6 +44,10 @@ database_opts = [ 'server-set SQL mode. To use whatever SQL mode ' 'is set by the server configuration, ' 'set this to no value. Example: mysql_sql_mode='), + cfg.BoolOpt('mysql_enable_ndb', + default=False, + help='If True, transparently enables support for handling ' + 'MySQL Cluster (NDB).'), cfg.IntOpt('idle_timeout', default=3600, deprecated_opts=[cfg.DeprecatedOpt('sql_idle_timeout', diff --git a/oslo_db/sqlalchemy/enginefacade.py b/oslo_db/sqlalchemy/enginefacade.py index 82ebbb1..752c6ca 100644 --- a/oslo_db/sqlalchemy/enginefacade.py +++ b/oslo_db/sqlalchemy/enginefacade.py @@ -135,6 +135,7 @@ class _TransactionFactory(object): self._engine_cfg = { 'sqlite_fk': _Default(False), 'mysql_sql_mode': _Default('TRADITIONAL'), + 'mysql_enable_ndb': _Default(False), 'idle_timeout': _Default(3600), 'connection_debug': _Default(0), 'max_pool_size': _Default(), @@ -206,6 +207,8 @@ class _TransactionFactory(object): :param mysql_sql_mode: MySQL SQL mode, defaults to TRADITIONAL + :param mysql_enable_ndb: enable MySQL Cluster (NDB) support + :param idle_timeout: connection pool recycle time, defaults to 3600. Note the connection does not actually have to be "idle" to be recycled. @@ -578,24 +581,27 @@ class _TransactionContext(object): self.flush_on_subtransaction = kw['flush_on_subtransaction'] @contextlib.contextmanager - def _connection(self, savepoint=False): + def _connection(self, savepoint=False, context=None): if self.connection is None: try: if self.session is not None: # use existing session, which is outer to us self.connection = self.session.connection() if savepoint: - with self.connection.begin_nested(): + with self.connection.begin_nested(), \ + self._add_context(self.connection, context): yield self.connection else: - yield self.connection + with self._add_context(self.connection, context): + yield self.connection else: # is outermost self.connection = self.factory._create_connection( mode=self.mode) self.transaction = self.connection.begin() try: - yield self.connection + with self._add_context(self.connection, context): + yield self.connection self._end_connection_transaction(self.transaction) except Exception: self.transaction.rollback() @@ -613,19 +619,22 @@ class _TransactionContext(object): else: # use existing connection, which is outer to us if savepoint: - with self.connection.begin_nested(): + with self.connection.begin_nested(), \ + self._add_context(self.connection, context): yield self.connection else: - yield self.connection + with self._add_context(self.connection, context): + yield self.connection @contextlib.contextmanager - def _session(self, savepoint=False): + def _session(self, savepoint=False, context=None): if self.session is None: self.session = self.factory._create_session( bind=self.connection, mode=self.mode) try: self.session.begin() - yield self.session + with self._add_context(self.session, context): + yield self.session self._end_session_transaction(self.session) except Exception: with excutils.save_and_reraise_exception(): @@ -637,12 +646,21 @@ class _TransactionContext(object): # use existing session, which is outer to us if savepoint: with self.session.begin_nested(): - yield self.session + with self._add_context(self.session, context): + yield self.session else: - yield self.session + with self._add_context(self.session, context): + yield self.session if self.flush_on_subtransaction: self.session.flush() + @contextlib.contextmanager + def _add_context(self, connection, context): + restore_context = connection.info.get('using_context') + connection.info['using_context'] = context + yield connection + connection.info['using_context'] = restore_context + def _end_session_transaction(self, session): if self.mode is _WRITER: session.commit() @@ -660,7 +678,8 @@ class _TransactionContext(object): else: transaction.rollback() - def _produce_block(self, mode, connection, savepoint, allow_async=False): + def _produce_block(self, mode, connection, savepoint, allow_async=False, + context=None): if mode is _WRITER: self._writer() elif mode is _ASYNC_READER: @@ -668,9 +687,9 @@ class _TransactionContext(object): else: self._reader(allow_async) if connection: - return self._connection(savepoint) + return self._connection(savepoint, context=context) else: - return self._session(savepoint) + return self._session(savepoint, context=context) def _writer(self): if self.mode is None: @@ -1005,7 +1024,8 @@ class _TransactionContextManager(object): mode=self._mode, connection=self._connection, savepoint=self._savepoint, - allow_async=self._allow_async) as resource: + allow_async=self._allow_async, + context=context) as resource: yield resource else: yield @@ -1173,6 +1193,9 @@ class LegacyEngineFacade(object): :keyword mysql_sql_mode: the SQL mode to be used for MySQL sessions. (defaults to TRADITIONAL) + :keyword mysql_enable_ndb: If True, transparently enables support for + handling MySQL Cluster (NDB). + (defaults to False) :keyword idle_timeout: timeout before idle sql connections are reaped (defaults to 3600) :keyword connection_debug: verbosity of SQL debugging information. diff --git a/oslo_db/sqlalchemy/engines.py b/oslo_db/sqlalchemy/engines.py index 601be76..a54cb23 100644 --- a/oslo_db/sqlalchemy/engines.py +++ b/oslo_db/sqlalchemy/engines.py @@ -32,6 +32,7 @@ from sqlalchemy.sql.expression import select from oslo_db import exception from oslo_db.sqlalchemy import exc_filters +from oslo_db.sqlalchemy import ndb from oslo_db.sqlalchemy import utils LOG = logging.getLogger(__name__) @@ -100,7 +101,26 @@ def _setup_logging(connection_debug=0): logger.setLevel(logging.WARNING) +def _vet_url(url): + if "+" not in url.drivername and not url.drivername.startswith("sqlite"): + if url.drivername.startswith("mysql"): + LOG.warning( + "URL %r does not contain a '+drivername' portion, " + "and will make use of a default driver. " + "A full dbname+drivername:// protocol is recommended. " + "For MySQL, it is strongly recommended that mysql+pymysql:// " + "be specified for maximum service compatibility", url + ) + else: + LOG.warning( + "URL %r does not contain a '+drivername' portion, " + "and will make use of a default driver. " + "A full dbname+drivername:// protocol is recommended.", url + ) + + def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, + mysql_enable_ndb=False, idle_timeout=3600, connection_debug=0, max_pool_size=None, max_overflow=None, pool_timeout=None, sqlite_synchronous=True, @@ -112,6 +132,8 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, url = sqlalchemy.engine.url.make_url(sql_connection) + _vet_url(url) + engine_args = { "pool_recycle": idle_timeout, 'convert_unicode': True, @@ -132,6 +154,9 @@ def create_engine(sql_connection, sqlite_fk=False, mysql_sql_mode=None, engine = sqlalchemy.create_engine(url, **engine_args) + if mysql_enable_ndb: + ndb.enable_ndb_support(engine) + _init_events( engine, mysql_sql_mode=mysql_sql_mode, @@ -265,6 +290,9 @@ def _init_events(engine, mysql_sql_mode=None, **kw): "consider enabling TRADITIONAL or STRICT_ALL_TABLES", realmode) + if ndb.ndb_status(engine): + ndb.init_ndb_events(engine) + @_init_events.dispatch_for("sqlite") def _init_events(engine, sqlite_synchronous=True, sqlite_fk=False, **kw): diff --git a/oslo_db/sqlalchemy/exc_filters.py b/oslo_db/sqlalchemy/exc_filters.py index 5743e89..2f575d8 100644 --- a/oslo_db/sqlalchemy/exc_filters.py +++ b/oslo_db/sqlalchemy/exc_filters.py @@ -395,6 +395,11 @@ def _is_db_connection_error(operational_error, match, engine_name, raise exception.DBConnectionError(operational_error) +@filters("*", sqla_exc.NotSupportedError, r".*") +def _raise_for_NotSupportedError(error, match, engine_name, is_disconnect): + raise exception.DBNotSupportedError(error) + + @filters("*", sqla_exc.DBAPIError, r".*") def _raise_for_remaining_DBAPIError(error, match, engine_name, is_disconnect): """Filter for remaining DBAPIErrors. diff --git a/oslo_db/sqlalchemy/migration.py b/oslo_db/sqlalchemy/migration.py index f6a4d55..457dd35 100644 --- a/oslo_db/sqlalchemy/migration.py +++ b/oslo_db/sqlalchemy/migration.py @@ -75,7 +75,10 @@ def db_sync(engine, abs_path, version=None, init_version=0, sanity_check=True): if sanity_check: _db_schema_sanity_check(engine) if version is None or version > current_version: - migration = versioning_api.upgrade(engine, repository, version) + try: + migration = versioning_api.upgrade(engine, repository, version) + except Exception as ex: + raise exception.DbMigrationError(ex) else: migration = versioning_api.downgrade(engine, repository, version) diff --git a/oslo_db/sqlalchemy/ndb.py b/oslo_db/sqlalchemy/ndb.py new file mode 100644 index 0000000..c7a3de9 --- /dev/null +++ b/oslo_db/sqlalchemy/ndb.py @@ -0,0 +1,137 @@ +# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Core functions for MySQL Cluster (NDB) Support.""" + +import re + +from sqlalchemy import String, event, schema +from sqlalchemy.ext.compiler import compiles +from sqlalchemy.types import VARCHAR + +engine_regex = re.compile("engine=innodb", re.IGNORECASE) +trans_regex = re.compile("savepoint|rollback|release savepoint", re.IGNORECASE) + + +def enable_ndb_support(engine): + """Enable NDB Support. + + Function to flag the MySQL engine dialect to support features specific + to MySQL Cluster (NDB). + """ + engine.dialect._oslodb_enable_ndb_support = True + + +def ndb_status(engine_or_compiler): + """Test if NDB Support is enabled. + + Function to test if NDB support is enabled or not. + """ + return getattr(engine_or_compiler.dialect, + '_oslodb_enable_ndb_support', + False) + + +def init_ndb_events(engine): + """Initialize NDB Events. + + Function starts NDB specific events. + """ + @event.listens_for(engine, "before_cursor_execute", retval=True) + def before_cursor_execute(conn, cursor, statement, parameters, context, + executemany): + """Listen for specific SQL strings and replace automatically. + + Function will intercept any raw execute calls and automatically + convert InnoDB to NDBCLUSTER, drop SAVEPOINT requests, drop + ROLLBACK requests, and drop RELEASE SAVEPOINT requests. + """ + if ndb_status(engine): + statement = engine_regex.sub("ENGINE=NDBCLUSTER", statement) + if re.match(trans_regex, statement): + statement = "SET @oslo_db_ndb_savepoint_rollback_disabled = 0;" + + return statement, parameters + + +@compiles(schema.CreateTable, "mysql") +def prefix_inserts(create_table, compiler, **kw): + """Replace InnoDB with NDBCLUSTER automatically. + + Function will intercept CreateTable() calls and automatically + convert InnoDB to NDBCLUSTER. Targets compiler events. + """ + existing = compiler.visit_create_table(create_table, **kw) + if ndb_status(compiler): + existing = engine_regex.sub("ENGINE=NDBCLUSTER", existing) + + return existing + + +class AutoStringTinyText(String): + """Class definition for AutoStringTinyText. + + Class is used by compiler function _auto-string_tiny_text(). + """ + + pass + + +@compiles(AutoStringTinyText, 'mysql') +def _auto_string_tiny_text(element, compiler, **kw): + if ndb_status(compiler): + return "TINYTEXT" + else: + return compiler.visit_string(element, **kw) + + +class AutoStringText(String): + """Class definition for AutoStringText. + + Class is used by compiler function _auto_string_text(). + """ + + pass + + +@compiles(AutoStringText, 'mysql') +def _auto_string_text(element, compiler, **kw): + if ndb_status(compiler): + return "TEXT" + else: + return compiler.visit_string(element, **kw) + + +class AutoStringSize(String): + """Class definition for AutoStringSize. + + Class is used by the compiler function _auto_string_size(). + """ + + def __init__(self, length, ndb_size, **kw): + """Initialize and extend the String arguments. + + Function adds the innodb_size and ndb_size arguments to the + function String(). + """ + super(AutoStringSize, self).__init__(length=length, **kw) + self.ndb_size = ndb_size + self.length = length + + +@compiles(AutoStringSize, 'mysql') +def _auto_string_size(element, compiler, **kw): + if ndb_status(compiler): + return compiler.process(VARCHAR(element.ndb_size), **kw) + else: + return compiler.visit_string(element, **kw) diff --git a/oslo_db/sqlalchemy/session.py b/oslo_db/sqlalchemy/session.py index bb3616f..5e37363 100644 --- a/oslo_db/sqlalchemy/session.py +++ b/oslo_db/sqlalchemy/session.py @@ -90,8 +90,8 @@ Recommended ways to use sessions within this framework: .. code-block:: sql - UPDATE bar SET bar = ${newbar} - WHERE id=(SELECT bar_id FROM foo WHERE id = ${foo_id} LIMIT 1); + UPDATE bar SET bar = '${newbar}' + WHERE id=(SELECT bar_id FROM foo WHERE id = '${foo_id}' LIMIT 1); .. note:: `create_duplicate_foo` is a trivially simple example of catching an exception while using a savepoint. Here we create two duplicate diff --git a/oslo_db/sqlalchemy/test_migrations.py b/oslo_db/sqlalchemy/test_migrations.py index 09e61f8..d650025 100644 --- a/oslo_db/sqlalchemy/test_migrations.py +++ b/oslo_db/sqlalchemy/test_migrations.py @@ -45,7 +45,7 @@ class WalkVersionsMixin(object): abstract class mixin. `INIT_VERSION`, `REPOSITORY` and `migration_api` attributes must be implemented in subclasses. - .. _auxiliary-dynamic-methods: Auxiliary Methods + .. _auxiliary-dynamic-methods: Auxiliary Methods: @@ -214,7 +214,7 @@ class WalkVersionsMixin(object): :type version: str :keyword with_data: Whether to verify the absence of changes from migration(s) being downgraded, see - :ref:`auxiliary-dynamic-methods <Auxiliary Methods>`. + :ref:`Auxiliary Methods <auxiliary-dynamic-methods>`. :type with_data: Bool """ @@ -246,7 +246,7 @@ class WalkVersionsMixin(object): :param version: id of revision to upgrade. :type version: str :keyword with_data: Whether to verify the applied changes with data, - see :ref:`auxiliary-dynamic-methods <Auxiliary Methods>`. + see :ref:`Auxiliary Methods <auxiliary-dynamic-methods>`. :type with_data: Bool """ # NOTE(sdague): try block is here because it's impossible to debug diff --git a/oslo_db/sqlalchemy/utils.py b/oslo_db/sqlalchemy/utils.py index 120b129..ac027ca 100644 --- a/oslo_db/sqlalchemy/utils.py +++ b/oslo_db/sqlalchemy/utils.py @@ -1139,6 +1139,36 @@ def get_non_innodb_tables(connectable, skip_tables=('migrate_version', return [i[0] for i in noninnodb] +def get_non_ndbcluster_tables(connectable, skip_tables=None): + """Get a list of tables which don't use MySQL Cluster (NDB) storage engine. + + :param connectable: a SQLAlchemy Engine or Connection instance + :param skip_tables: a list of tables which might have a different + storage engine + """ + query_str = """ + SELECT table_name + FROM information_schema.tables + WHERE table_schema = :database AND + engine != 'ndbcluster' + """ + + params = {} + if skip_tables: + params = dict( + ('skip_%s' % i, table_name) + for i, table_name in enumerate(skip_tables) + ) + + placeholders = ', '.join(':' + p for p in params) + query_str += ' AND table_name NOT IN (%s)' % placeholders + + params['database'] = connectable.engine.url.database + query = text(query_str) + nonndbcluster = connectable.execute(query, **params) + return [i[0] for i in nonndbcluster] + + class NonCommittingConnectable(object): """A ``Connectable`` substitute which rolls all operations back. diff --git a/oslo_db/tests/sqlalchemy/test_enginefacade.py b/oslo_db/tests/sqlalchemy/test_enginefacade.py index 8524683..303a8dd 100644 --- a/oslo_db/tests/sqlalchemy/test_enginefacade.py +++ b/oslo_db/tests/sqlalchemy/test_enginefacade.py @@ -57,6 +57,7 @@ class SingletonConnection(SingletonOnName): def __init__(self, **kw): super(SingletonConnection, self).__init__( "connection", **kw) + self.info = {} class SingletonEngine(SingletonOnName): @@ -115,14 +116,16 @@ class MockFacadeTest(oslo_test_base.BaseTestCase): writer_conn = SingletonConnection() writer_engine = SingletonEngine(writer_conn) writer_session = mock.Mock( - connection=mock.Mock(return_value=writer_conn)) + connection=mock.Mock(return_value=writer_conn), + info={}) writer_maker = mock.Mock(return_value=writer_session) if self.slave_uri: async_reader_conn = SingletonConnection() async_reader_engine = SingletonEngine(async_reader_conn) async_reader_session = mock.Mock( - connection=mock.Mock(return_value=async_reader_conn)) + connection=mock.Mock(return_value=async_reader_conn), + info={}) async_reader_maker = mock.Mock(return_value=async_reader_session) else: @@ -772,6 +775,20 @@ class MockFacadeTest(oslo_test_base.BaseTestCase): with self._assert_reader_session(makers) as session: session.execute("test1") + def test_using_context_present_in_session_info(self): + context = oslo_context.RequestContext() + + with enginefacade.reader.using(context) as session: + self.assertEqual(context, session.info['using_context']) + self.assertIsNone(session.info['using_context']) + + def test_using_context_present_in_connection_info(self): + context = oslo_context.RequestContext() + + with enginefacade.writer.connection.using(context) as connection: + self.assertEqual(context, connection.info['using_context']) + self.assertIsNone(connection.info['using_context']) + def test_using_reader_rollback_reader_session(self): enginefacade.configure(rollback_reader_sessions=True) diff --git a/oslo_db/tests/sqlalchemy/test_migration_common.py b/oslo_db/tests/sqlalchemy/test_migration_common.py index 95d031e..870f208 100644 --- a/oslo_db/tests/sqlalchemy/test_migration_common.py +++ b/oslo_db/tests/sqlalchemy/test_migration_common.py @@ -192,6 +192,22 @@ class TestMigrationCommon(test_base.DbTestCase): self.assertRaises(db_exception.DBMigrationError, migration.db_sync, self.engine, self.path, 'foo') + @mock.patch.object(versioning_api, 'upgrade') + def test_db_sync_script_not_present(self, upgrade): + # For non existent migration script file sqlalchemy-migrate will raise + # VersionNotFoundError which will be wrapped in DbMigrationError. + upgrade.side_effect = migrate_exception.VersionNotFoundError + self.assertRaises(db_exception.DbMigrationError, + migration.db_sync, self.engine, self.path, + self.test_version + 1) + + @mock.patch.object(versioning_api, 'upgrade') + def test_db_sync_known_error_raised(self, upgrade): + upgrade.side_effect = migrate_exception.KnownError + self.assertRaises(db_exception.DbMigrationError, + migration.db_sync, self.engine, self.path, + self.test_version + 1) + def test_db_sync_upgrade(self): init_ver = 55 with test_utils.nested( diff --git a/oslo_db/tests/sqlalchemy/test_models.py b/oslo_db/tests/sqlalchemy/test_models.py index 893d96f..0b0f6d8 100644 --- a/oslo_db/tests/sqlalchemy/test_models.py +++ b/oslo_db/tests/sqlalchemy/test_models.py @@ -211,7 +211,7 @@ class SoftDeleteMixinTest(test_base.DbTestCase): self.session.add(m) self.session.commit() self.assertEqual(0, m.deleted) - self.assertIs(None, m.deleted_at) + self.assertIsNone(m.deleted_at) m.soft_delete(self.session) self.assertEqual(123456, m.deleted) diff --git a/oslo_db/tests/sqlalchemy/test_ndb.py b/oslo_db/tests/sqlalchemy/test_ndb.py new file mode 100644 index 0000000..dc5dc62 --- /dev/null +++ b/oslo_db/tests/sqlalchemy/test_ndb.py @@ -0,0 +1,176 @@ +# Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Tests for MySQL Cluster (NDB) Support.""" + +import logging + +import mock + +from oslo_db import exception +from oslo_db.sqlalchemy import enginefacade +from oslo_db.sqlalchemy import engines +from oslo_db.sqlalchemy import ndb +from oslo_db.sqlalchemy import test_fixtures +from oslo_db.sqlalchemy import utils + +from oslotest import base as test_base + +from sqlalchemy import Column +from sqlalchemy import Integer +from sqlalchemy import MetaData +from sqlalchemy import String +from sqlalchemy import Table +from sqlalchemy import Text + +from sqlalchemy import create_engine +from sqlalchemy import schema + +from sqlalchemy.dialects.mysql import TINYTEXT + +LOG = logging.getLogger(__name__) + +_MOCK_CONNECTION = 'mysql+pymysql://' +_TEST_TABLE = Table("test_ndb", MetaData(), + Column('id', Integer, primary_key=True), + Column('test1', ndb.AutoStringTinyText(255)), + Column('test2', ndb.AutoStringText(4096)), + Column('test3', ndb.AutoStringSize(255, 64)), + mysql_engine='InnoDB') + + +class NDBMockTestBase(test_base.BaseTestCase): + def setUp(self): + super(NDBMockTestBase, self).setUp() + mock_dbapi = mock.Mock() + self.test_engine = test_engine = create_engine( + _MOCK_CONNECTION, module=mock_dbapi) + test_engine.dialect._oslodb_enable_ndb_support = True + ndb.init_ndb_events(test_engine) + + +class NDBEventTestCase(NDBMockTestBase): + + def test_ndb_createtable_override(self): + test_engine = self.test_engine + self.assertRegex( + str(schema.CreateTable(_TEST_TABLE).compile( + dialect=test_engine.dialect)), + "ENGINE=NDBCLUSTER") + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_engine_override(self): + test_engine = self.test_engine + statement = "ENGINE=InnoDB" + for fn in test_engine.dispatch.before_cursor_execute: + statement, dialect = fn( + mock.Mock(), mock.Mock(), statement, {}, mock.Mock(), False) + self.assertEqual(statement, "ENGINE=NDBCLUSTER") + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_savepoint_override(self): + test_engine = self.test_engine + statement = "SAVEPOINT xyx" + for fn in test_engine.dispatch.before_cursor_execute: + statement, dialect = fn( + mock.Mock(), mock.Mock(), statement, {}, mock.Mock(), False) + self.assertEqual(statement, + "SET @oslo_db_ndb_savepoint_rollback_disabled = 0;") + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_rollback_override(self): + test_engine = self.test_engine + statement = "ROLLBACK TO SAVEPOINT xyz" + for fn in test_engine.dispatch.before_cursor_execute: + statement, dialect = fn( + mock.Mock(), mock.Mock(), statement, {}, mock.Mock(), False) + self.assertEqual(statement, + "SET @oslo_db_ndb_savepoint_rollback_disabled = 0;") + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_rollback_release_override(self): + test_engine = self.test_engine + statement = "RELEASE SAVEPOINT xyz" + for fn in test_engine.dispatch.before_cursor_execute: + statement, dialect = fn( + mock.Mock(), mock.Mock(), statement, {}, mock.Mock(), False) + self.assertEqual(statement, + "SET @oslo_db_ndb_savepoint_rollback_disabled = 0;") + test_engine.dialect._oslodb_enable_ndb_support = False + + +class NDBDatatypesTestCase(NDBMockTestBase): + def test_ndb_autostringtinytext(self): + test_engine = self.test_engine + self.assertEqual("TINYTEXT", + str(ndb.AutoStringTinyText(255).compile( + dialect=test_engine.dialect))) + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_autostringtext(self): + test_engine = self.test_engine + self.assertEqual("TEXT", + str(ndb.AutoStringText(4096).compile( + dialect=test_engine.dialect))) + test_engine.dialect._oslodb_enable_ndb_support = False + + def test_ndb_autostringsize(self): + test_engine = self.test_engine + self.assertEqual('VARCHAR(64)', + str(ndb.AutoStringSize(255, 64).compile( + dialect=test_engine.dialect))) + test_engine.dialect._oslodb_enable_ndb_support = False + + +class NDBOpportunisticTestCase( + test_fixtures.OpportunisticDBTestMixin, test_base.BaseTestCase): + + FIXTURE = test_fixtures.MySQLOpportunisticFixture + + def init_db(self, use_ndb): + # get the MySQL engine created by the opportunistic + # provisioning system + self.engine = enginefacade.writer.get_engine() + if use_ndb: + # if we want NDB, make a new local engine that uses the + # URL / database / schema etc. of the provisioned engine, + # since NDB-ness is a per-table thing + self.engine = engines.create_engine( + self.engine.url, mysql_enable_ndb=True + ) + self.addCleanup(self.engine.dispose) + self.test_table = _TEST_TABLE + try: + self.test_table.create(self.engine) + except exception.DBNotSupportedError: + self.skip("MySQL NDB Cluster not available") + + def test_ndb_enabled(self): + self.init_db(True) + self.assertTrue(ndb.ndb_status(self.engine)) + self.assertIsInstance(self.test_table.c.test1.type, TINYTEXT) + self.assertIsInstance(self.test_table.c.test2.type, Text) + self.assertIsInstance(self.test_table.c.test3.type, String) + self.assertEqual(64, self.test_table.c.test3.type.length) + self.assertEqual([], utils.get_non_ndbcluster_tables(self.engine)) + + def test_ndb_disabled(self): + self.init_db(False) + self.assertFalse(ndb.ndb_status(self.engine)) + self.assertIsInstance(self.test_table.c.test1.type, String) + self.assertEqual(255, self.test_table.c.test1.type.length) + self.assertIsInstance(self.test_table.c.test2.type, String) + self.assertEqual(4096, self.test_table.c.test2.type.length) + self.assertIsInstance(self.test_table.c.test3.type, String) + self.assertEqual(255, self.test_table.c.test3.type.length) + self.assertEqual([], utils.get_non_innodb_tables(self.engine)) diff --git a/oslo_db/tests/sqlalchemy/test_sqlalchemy.py b/oslo_db/tests/sqlalchemy/test_sqlalchemy.py index a37cdc6..c9f3d7d 100644 --- a/oslo_db/tests/sqlalchemy/test_sqlalchemy.py +++ b/oslo_db/tests/sqlalchemy/test_sqlalchemy.py @@ -341,6 +341,7 @@ class EngineFacadeTestCase(oslo_test.BaseTestCase): connection_debug=100, max_pool_size=10, mysql_sql_mode='TRADITIONAL', + mysql_enable_ndb=False, sqlite_fk=False, idle_timeout=mock.ANY, retry_interval=mock.ANY, @@ -664,6 +665,52 @@ class CreateEngineTest(oslo_test.BaseTestCase): engines._thread_yield ) + def test_warn_on_missing_driver(self): + + warnings = mock.Mock() + + def warn_interpolate(msg, args): + # test the interpolation itself to ensure the password + # is concealed + warnings.warning(msg % args) + + with mock.patch( + "oslo_db.sqlalchemy.engines.LOG.warning", + warn_interpolate): + + engines._vet_url( + url.make_url("mysql://scott:tiger@some_host/some_db")) + engines._vet_url(url.make_url( + "mysql+mysqldb://scott:tiger@some_host/some_db")) + engines._vet_url(url.make_url( + "mysql+pymysql://scott:tiger@some_host/some_db")) + engines._vet_url(url.make_url( + "postgresql+psycopg2://scott:tiger@some_host/some_db")) + engines._vet_url(url.make_url( + "postgresql://scott:tiger@some_host/some_db")) + + self.assertEqual( + [ + mock.call.warning( + "URL mysql://scott:***@some_host/some_db does not contain " + "a '+drivername' portion, " + "and will make use of a default driver. " + "A full dbname+drivername:// protocol is recommended. " + "For MySQL, it is strongly recommended that " + "mysql+pymysql:// " + "be specified for maximum service compatibility", + + ), + mock.call.warning( + "URL postgresql://scott:***@some_host/some_db does not " + "contain a '+drivername' portion, " + "and will make use of a default driver. " + "A full dbname+drivername:// protocol is recommended." + ) + ], + warnings.mock_calls + ) + class ProcessGuardTest(test_base.DbTestCase): def test_process_guard(self): diff --git a/releasenotes/notes/warn-incomplete-url-c44cd03baf630c7c.yaml b/releasenotes/notes/warn-incomplete-url-c44cd03baf630c7c.yaml new file mode 100644 index 0000000..ee53555 --- /dev/null +++ b/releasenotes/notes/warn-incomplete-url-c44cd03baf630c7c.yaml @@ -0,0 +1,14 @@ +--- +upgrade: + - | + oslo.db now logs a warning when the connection URL does not + explicitly mention a driver. The default driver is still used, but + in some cases, such as MySQL, the default is incompatible with the + concurrency library eventlet. + - | + It is strongly recommended to use the `PyMySQL + <https://pypi.python.org/pypi/PyMySQL>`__ driver when connecting + to a MySQL-compatible database to ensure the best compatibility + with the concurrency library eventlet. To use PyMySQL, ensure the + connection URL is specified with ``mysql+pymysql://`` as the + scheme. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index e5cc22e..eb70f78 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -38,9 +38,13 @@ # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'oslosphinx', + 'openstackdocstheme', 'reno.sphinxext', ] +# openstackdocstheme options +repository_name = 'openstack/oslo.db' +bug_project = 'oslo.db' +bug_tag = '' # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -112,7 +116,7 @@ pygments_style = 'sphinx' # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +html_theme = 'openstackdocs' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -150,7 +154,7 @@ html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -# html_last_updated_fmt = '%b %d, %Y' +html_last_updated_fmt = '%Y-%m-%d %H:%M' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. diff --git a/requirements.txt b/requirements.txt index d868a68..b6f7af4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,11 +5,10 @@ pbr!=2.1.0,>=2.0.0 # Apache-2.0 alembic>=0.8.10 # MIT debtcollector>=1.2.0 # Apache-2.0 -oslo.i18n>=2.1.0 # Apache-2.0 -oslo.config>=3.22.0 # Apache-2.0 -oslo.context>=2.12.0 # Apache-2.0 +oslo.i18n!=3.15.2,>=2.1.0 # Apache-2.0 +oslo.config!=4.3.0,!=4.4.0,>=4.0.0 # Apache-2.0 oslo.utils>=3.20.0 # Apache-2.0 SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT -sqlalchemy-migrate>=0.9.6 # Apache-2.0 +sqlalchemy-migrate>=0.11.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 six>=1.9.0 # MIT @@ -31,19 +31,20 @@ postgresql = # Dependencies for testing oslo.db itself. test = hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 - coverage>=4.0 # Apache-2.0 + coverage!=4.4,>=4.0 # Apache-2.0 doc8 # Apache-2.0 - eventlet!=0.18.3,>=0.18.2 # MIT + eventlet!=0.18.3,!=0.20.1,<0.21.0,>=0.18.2 # MIT fixtures>=3.0.0 # Apache-2.0/BSD mock>=2.0 # BSD python-subunit>=0.0.18 # Apache-2.0/BSD - sphinx>=1.5.1 # BSD - oslosphinx>=4.7.0 # Apache-2.0 + sphinx>=1.6.2 # BSD + openstackdocstheme>=1.11.0 # Apache-2.0 oslotest>=1.10.0 # Apache-2.0 + oslo.context>=2.14.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD testtools>=1.4.0 # MIT os-testr>=0.8.0 # Apache-2.0 - reno>=1.8.0 # Apache-2.0 + reno!=2.3.1,>=1.8.0 # Apache-2.0 fixtures = testresources>=0.2.4 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD @@ -70,6 +71,7 @@ universal = 1 source-dir = doc/source build-dir = doc/build all_files = 1 +warning-is-error = 1 [upload_sphinx] upload-dir = doc/build/html @@ -89,7 +91,7 @@ mapping_file = babel.cfg output_file = oslo_db/locale/oslo_db.pot [pbr] -warnerrors = True autodoc_index_modules = True +api_doc_dir = reference/api autodoc_exclude_modules = oslo_db.tests.* |