diff options
91 files changed, 1119 insertions, 1373 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..0ae225c54 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +--- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: mixed-line-ending + args: ['--fix', 'lf'] + exclude: '.*\.(svg)$' + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + exclude: 'rally-scenarios/heat-fakevirt.yaml' + - repo: local + hooks: + - id: flake8 + name: flake8 + additional_dependencies: + - hacking>=3.1.0,<3.2.0 + language: python + entry: flake8 + files: '^.*\.py$' + exclude: '^(doc|releasenotes|tools)/.*$' diff --git a/api-ref/source/v1/events.inc b/api-ref/source/v1/events.inc index a0f72ece2..ef3c08da2 100644 --- a/api-ref/source/v1/events.inc +++ b/api-ref/source/v1/events.inc @@ -169,7 +169,7 @@ Shows details for an event. .. rest_status_code:: success status.yaml - - 200 + - 200 .. rest_status_code:: error status.yaml diff --git a/api-ref/source/v1/services.inc b/api-ref/source/v1/services.inc index 67b344cd5..3ce6b94ab 100644 --- a/api-ref/source/v1/services.inc +++ b/api-ref/source/v1/services.inc @@ -11,7 +11,7 @@ Show orchestration engine status Enables administrative users to view details for all orchestration engines. -Orchestration engine details include engine id, binary, topic name, host, +Orchestration engine details include engine id, binary, topic name, host, report interval, last updated time, health status, and host name. Response Codes diff --git a/bin/heat-db-setup b/bin/heat-db-setup index b0e39ac74..8df02aa08 100755 --- a/bin/heat-db-setup +++ b/bin/heat-db-setup @@ -289,7 +289,7 @@ rm $log_conf # Do a final sanity check on the database. -echo "SELECT * FROM migrate_version;" | mysql -u heat --password=${MYSQL_HEAT_PW} heat > /dev/null +echo "SELECT * FROM alembic_version;" | mysql -u heat --password=${MYSQL_HEAT_PW} heat > /dev/null if ! [ $? -eq 0 ] then echo "Final sanity check failed." >&2 diff --git a/doc/source/conf.py b/doc/source/conf.py index 913262c95..2bbceb0a8 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -178,7 +178,7 @@ apidoc_separate_modules = True apidoc_excluded_paths = [ 'cmd', 'cloudinit', - 'db/sqlalchemy/migrate_repo/versions', + 'db/sqlalchemy/migrations/versions', 'engine/resources/aws', 'engine/resources/openstack', 'hacking', diff --git a/heat/cmd/manage.py b/heat/cmd/manage.py index c853d9ac2..870c95d1d 100644 --- a/heat/cmd/manage.py +++ b/heat/cmd/manage.py @@ -25,18 +25,18 @@ from heat.common import exception from heat.common.i18n import _ from heat.common import messaging from heat.common import service_utils -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api +from heat.db import migration as db_migration from heat.objects import service as service_objects from heat.rpc import client as rpc_client from heat import version - CONF = cfg.CONF def do_db_version(): """Print database's current migration level.""" - print(db_api.db_version(db_api.get_engine())) + print(db_migration.db_version()) def do_db_sync(): @@ -44,7 +44,7 @@ def do_db_sync(): Creating first if necessary. """ - db_api.db_sync(db_api.get_engine(), CONF.command.version) + db_migration.db_sync(CONF.command.version) class ServiceManageCommand(object): diff --git a/heat/common/context.py b/heat/common/context.py index c72c8367b..854c13467 100644 --- a/heat/common/context.py +++ b/heat/common/context.py @@ -31,12 +31,11 @@ from heat.common import endpoint_utils from heat.common import exception from heat.common import policy from heat.common import wsgi -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine import clients LOG = logging.getLogger(__name__) - cfg.CONF.import_opt('client_retry_limit', 'heat.common.config') # Note, we yield the options via list_opts to enable generation of the diff --git a/heat/db/alembic.ini b/heat/db/alembic.ini new file mode 100644 index 000000000..dbee79a3b --- /dev/null +++ b/heat/db/alembic.ini @@ -0,0 +1,72 @@ +[alembic] +# path to migration scripts +script_location = %(here)s/migrations + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# version location specification; This defaults +# to heat/db/migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:heat/db/migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +sqlalchemy.url = sqlite:///heat.db + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/heat/db/sqlalchemy/api.py b/heat/db/api.py index df627d43c..8b2c99967 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/api.py @@ -38,10 +38,9 @@ from sqlalchemy.orm import aliased as orm_aliased from heat.common import crypt from heat.common import exception from heat.common.i18n import _ -from heat.db.sqlalchemy import filters as db_filters -from heat.db.sqlalchemy import migration -from heat.db.sqlalchemy import models -from heat.db.sqlalchemy import utils as db_utils +from heat.db import filters as db_filters +from heat.db import models +from heat.db import utils as db_utils from heat.engine import environment as heat_environment from heat.rpc import api as rpc_api @@ -1622,19 +1621,6 @@ def sync_point_update_input_data(context, entity_id, return rows_updated -def db_sync(engine, version=None): - """Migrate the database to `version` or the most recent version.""" - if version is not None and int(version) < db_version(engine): - raise exception.Error(_("Cannot migrate to lower schema version.")) - - return migration.db_sync(engine, version=version) - - -def db_version(engine): - """Display the current database version.""" - return migration.db_version(engine) - - def _crypt_action(encrypt): if encrypt: return _('encrypt') diff --git a/heat/db/sqlalchemy/filters.py b/heat/db/filters.py index 6c7b8cf24..6c7b8cf24 100644 --- a/heat/db/sqlalchemy/filters.py +++ b/heat/db/filters.py diff --git a/heat/db/migration.py b/heat/db/migration.py new file mode 100644 index 000000000..910656ca1 --- /dev/null +++ b/heat/db/migration.py @@ -0,0 +1,117 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from alembic import command as alembic_api +from alembic import config as alembic_config +from alembic import migration as alembic_migration +from oslo_log import log as logging +import sqlalchemy as sa + +from heat.db import api as db_api + +LOG = logging.getLogger(__name__) + +ALEMBIC_INIT_VERSION = 'c6214ca60943' + + +def _migrate_legacy_database(engine, connection, config): + """Check if database is a legacy sqlalchemy-migrate-managed database. + + If it is, migrate it by "stamping" the initial alembic schema. + """ + # If the database doesn't have the sqlalchemy-migrate legacy migration + # table, we don't have anything to do + if not sa.inspect(engine).has_table('migrate_version'): + return + + # Likewise, if we've already migrated to alembic, we don't have anything to + # do + context = alembic_migration.MigrationContext.configure(connection) + if context.get_current_revision(): + return + + # We have legacy migrations but no alembic migration. Stamp (dummy apply) + # the initial alembic migration. + + LOG.info( + 'The database is still under sqlalchemy-migrate control; ' + 'fake applying the initial alembic migration' + ) + alembic_api.stamp(config, ALEMBIC_INIT_VERSION) + + +def _find_alembic_conf(): + """Get the project's alembic configuration + + :returns: An instance of ``alembic.config.Config`` + """ + path = os.path.join( + os.path.abspath(os.path.dirname(__file__)), + 'alembic.ini', + ) + config = alembic_config.Config(os.path.abspath(path)) + # we don't want to use the logger configuration from the file, which is + # only really intended for the CLI + # https://stackoverflow.com/a/42691781/613428 + config.attributes['configure_logger'] = False + return config + + +def _upgrade_alembic(engine, config, version): + # re-use the connection rather than creating a new one + with engine.begin() as connection: + config.attributes['connection'] = connection + _migrate_legacy_database(engine, connection, config) + alembic_api.upgrade(config, version or 'head') + + +def db_sync(version=None, engine=None): + """Migrate the database to `version` or the most recent version.""" + # if the user requested a specific version, check if it's an integer: if + # so, we're almost certainly in sqlalchemy-migrate land and won't support + # that + if version is not None and version.isdigit(): + raise ValueError( + 'You requested an sqlalchemy-migrate database version; this is ' + 'no longer supported' + ) + + if engine is None: + engine = db_api.get_engine() + + config = _find_alembic_conf() + + # discard the URL encoded in alembic.ini in favour of the URL configured + # for the engine by the database fixtures, casting from + # 'sqlalchemy.engine.url.URL' to str in the process. This returns a + # RFC-1738 quoted URL, which means that a password like "foo@" will be + # turned into "foo%40". This in turns causes a problem for + # set_main_option() because that uses ConfigParser.set, which (by design) + # uses *python* interpolation to write the string out ... where "%" is the + # special python interpolation character! Avoid this mismatch by quoting + # all %'s for the set below. + engine_url = str(engine.url).replace('%', '%%') + config.set_main_option('sqlalchemy.url', str(engine_url)) + + LOG.info('Applying migration(s)') + _upgrade_alembic(engine, config, version) + LOG.info('Migration(s) applied') + + +def db_version(): + """Get database version.""" + engine = db_api.get_engine() + with engine.connect() as connection: + m_context = alembic_migration.MigrationContext.configure(connection) + return m_context.get_current_revision() diff --git a/heat/db/migrations/README.rst b/heat/db/migrations/README.rst new file mode 100644 index 000000000..b2283fc82 --- /dev/null +++ b/heat/db/migrations/README.rst @@ -0,0 +1,15 @@ +Database migrations +=================== + +This directory contains migrations for the database. These are implemented +using `alembic`__, a lightweight database migration tool designed for usage +with `SQLAlchemy`__. + +The best place to start understanding Alembic is with its own `tutorial`__. You +can also play around with the :command:`alembic` command:: + + $ alembic --help + +.. __: https://alembic.sqlalchemy.org/en/latest/ +.. __: https://www.sqlalchemy.org/ +.. __: https://alembic.sqlalchemy.org/en/latest/tutorial.html diff --git a/heat/db/migrations/env.py b/heat/db/migrations/env.py new file mode 100644 index 000000000..932b16a97 --- /dev/null +++ b/heat/db/migrations/env.py @@ -0,0 +1,94 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from heat.db import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.attributes.get('configure_logger', True): + fileConfig(config.config_file_name) + +# this is the MetaData object for the various models in the database +target_metadata = models.BASE.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL and not an Engine, though an + Engine is acceptable here as well. By skipping the Engine creation we + don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine and associate a connection + with the context. + + This is modified from the default based on the below, since we want to + share an engine when unit testing so in-memory database testing actually + works. + + https://alembic.sqlalchemy.org/en/latest/cookbook.html#connection-sharing + """ + connectable = config.attributes.get('connection', None) + + if connectable is None: + # only create Engine if we don't have a Connection from the outside + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + # when connectable is already a Connection object, calling connect() gives + # us a *branched connection* + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/heat/db/migrations/script.py.mako b/heat/db/migrations/script.py.mako new file mode 100644 index 000000000..120937fbc --- /dev/null +++ b/heat/db/migrations/script.py.mako @@ -0,0 +1,20 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} diff --git a/heat/db/migrations/versions/c6214ca60943_initial_revision.py b/heat/db/migrations/versions/c6214ca60943_initial_revision.py new file mode 100644 index 000000000..d3e9d757d --- /dev/null +++ b/heat/db/migrations/versions/c6214ca60943_initial_revision.py @@ -0,0 +1,392 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Initial revision + +Revision ID: c6214ca60943 +Revises: +Create Date: 2023-03-22 18:04:02.387269 +""" + +from alembic import op +import sqlalchemy as sa + +import heat.db.types + +# revision identifiers, used by Alembic. +revision = 'c6214ca60943' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'raw_template_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('files', heat.db.types.Json(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'resource_properties_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('data', heat.db.types.Json(), nullable=True), + sa.Column('encrypted', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'service', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('engine_id', sa.String(length=36), nullable=False), + sa.Column('host', sa.String(length=255), nullable=False), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('binary', sa.String(length=255), nullable=False), + sa.Column('topic', sa.String(length=255), nullable=False), + sa.Column('report_interval', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'software_config', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('group', sa.String(length=255), nullable=True), + sa.Column('config', heat.db.types.Json(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_software_config_tenant'), + 'software_config', + ['tenant'], + unique=False, + ) + op.create_table( + 'user_creds', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('username', sa.String(length=255), nullable=True), + sa.Column('password', sa.String(length=255), nullable=True), + sa.Column('region_name', sa.String(length=255), nullable=True), + sa.Column('decrypt_method', sa.String(length=64), nullable=True), + sa.Column('tenant', sa.String(length=1024), nullable=True), + sa.Column('auth_url', sa.Text(), nullable=True), + sa.Column('tenant_id', sa.String(length=256), nullable=True), + sa.Column('trust_id', sa.String(length=255), nullable=True), + sa.Column('trustor_user_id', sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'raw_template', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('template', heat.db.types.Json(), nullable=True), + sa.Column('files', heat.db.types.Json(), nullable=True), + sa.Column( + 'environment', heat.db.types.Json(), nullable=True + ), + sa.Column('files_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['files_id'], + ['raw_template_files.id'], + name='raw_tmpl_files_fkey_ref', + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'software_deployment', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('server_id', sa.String(length=36), nullable=False), + sa.Column('config_id', sa.String(length=36), nullable=False), + sa.Column( + 'input_values', heat.db.types.Json(), nullable=True + ), + sa.Column( + 'output_values', heat.db.types.Json(), nullable=True + ), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.Column( + 'stack_user_project_id', sa.String(length=64), nullable=True + ), + sa.ForeignKeyConstraint( + ['config_id'], + ['software_config.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'ix_software_deployment_created_at', + 'software_deployment', + ['created_at'], + unique=False, + ) + op.create_index( + op.f('ix_software_deployment_server_id'), + 'software_deployment', + ['server_id'], + unique=False, + ) + op.create_index( + op.f('ix_software_deployment_tenant'), + 'software_deployment', + ['tenant'], + unique=False, + ) + op.create_table( + 'stack', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('raw_template_id', sa.Integer(), nullable=False), + sa.Column('prev_raw_template_id', sa.Integer(), nullable=True), + sa.Column('user_creds_id', sa.Integer(), nullable=True), + sa.Column('username', sa.String(length=256), nullable=True), + sa.Column('owner_id', sa.String(length=36), nullable=True), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('timeout', sa.Integer(), nullable=True), + sa.Column('tenant', sa.String(length=256), nullable=True), + sa.Column('disable_rollback', sa.Boolean(), nullable=False), + sa.Column( + 'stack_user_project_id', sa.String(length=64), nullable=True + ), + sa.Column('backup', sa.Boolean(), nullable=True), + sa.Column('nested_depth', sa.Integer(), nullable=True), + sa.Column('convergence', sa.Boolean(), nullable=True), + sa.Column('current_traversal', sa.String(length=36), nullable=True), + sa.Column( + 'current_deps', heat.db.types.Json(), nullable=True + ), + sa.Column( + 'parent_resource_name', sa.String(length=255), nullable=True + ), + sa.ForeignKeyConstraint( + ['prev_raw_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['raw_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['user_creds_id'], + ['user_creds.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'ix_stack_name', 'stack', ['name'], unique=False, mysql_length=255 + ) + op.create_index( + 'ix_stack_tenant', 'stack', ['tenant'], unique=False, mysql_length=255 + ) + op.create_index( + op.f('ix_stack_owner_id'), 'stack', ['owner_id'], unique=False + ) + op.create_table( + 'event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('resource_action', sa.String(length=255), nullable=True), + sa.Column('resource_status', sa.String(length=255), nullable=True), + sa.Column('resource_name', sa.String(length=255), nullable=True), + sa.Column( + 'physical_resource_id', sa.String(length=255), nullable=True + ), + sa.Column( + 'resource_status_reason', sa.String(length=255), nullable=True + ), + sa.Column('resource_type', sa.String(length=255), nullable=True), + sa.Column('resource_properties', sa.PickleType(), nullable=True), + sa.Column('rsrc_prop_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['rsrc_prop_data_id'], + ['resource_properties_data.id'], + name='ev_rsrc_prop_data_ref', + ), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid'), + mysql_engine='InnoDB', + ) + op.create_table( + 'resource', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('nova_instance', sa.String(length=255), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column( + 'rsrc_metadata', heat.db.types.Json(), nullable=True + ), + sa.Column( + 'properties_data', heat.db.types.Json(), nullable=True + ), + sa.Column('engine_id', sa.String(length=36), nullable=True), + sa.Column('atomic_key', sa.Integer(), nullable=True), + sa.Column('needed_by', heat.db.types.List(), nullable=True), + sa.Column('requires', heat.db.types.List(), nullable=True), + sa.Column('replaces', sa.Integer(), nullable=True), + sa.Column('replaced_by', sa.Integer(), nullable=True), + sa.Column('current_template_id', sa.Integer(), nullable=True), + sa.Column('properties_data_encrypted', sa.Boolean(), nullable=True), + sa.Column('root_stack_id', sa.String(length=36), nullable=True), + sa.Column('rsrc_prop_data_id', sa.Integer(), nullable=True), + sa.Column('attr_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['attr_data_id'], + ['resource_properties_data.id'], + name='rsrc_attr_data_ref', + ), + sa.ForeignKeyConstraint( + ['current_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['rsrc_prop_data_id'], + ['resource_properties_data.id'], + name='rsrc_rsrc_prop_data_ref', + ), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_resource_root_stack_id'), + 'resource', + ['root_stack_id'], + unique=False, + ) + op.create_table( + 'snapshot', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.String(length=255), nullable=True), + sa.Column('data', heat.db.types.Json(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_snapshot_tenant'), 'snapshot', ['tenant'], unique=False + ) + op.create_table( + 'stack_lock', + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('engine_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('stack_id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'stack_tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('tag', sa.Unicode(length=80), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'sync_point', + sa.Column('entity_id', sa.String(length=36), nullable=False), + sa.Column('traversal_id', sa.String(length=36), nullable=False), + sa.Column('is_update', sa.Boolean(), nullable=False), + sa.Column('atomic_key', sa.Integer(), nullable=False), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column( + 'input_data', heat.db.types.Json(), nullable=True + ), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('entity_id', 'traversal_id', 'is_update'), + ) + op.create_table( + 'resource_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('key', sa.String(length=255), nullable=True), + sa.Column('value', sa.Text(), nullable=True), + sa.Column('redact', sa.Boolean(), nullable=True), + sa.Column('decrypt_method', sa.String(length=64), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ['resource_id'], + ['resource.id'], + name='fk_resource_id', + ondelete='CASCADE', + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) diff --git a/heat/db/sqlalchemy/models.py b/heat/db/models.py index ca208bef0..ebc235f5d 100644 --- a/heat/db/sqlalchemy/models.py +++ b/heat/db/models.py @@ -21,7 +21,7 @@ from sqlalchemy.ext import declarative from sqlalchemy.orm import backref from sqlalchemy.orm import relationship -from heat.db.sqlalchemy import types +from heat.db import types BASE = declarative.declarative_base() diff --git a/heat/db/sqlalchemy/__init__.py b/heat/db/sqlalchemy/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/heat/db/sqlalchemy/__init__.py +++ /dev/null diff --git a/heat/db/sqlalchemy/migrate_repo/README b/heat/db/sqlalchemy/migrate_repo/README deleted file mode 100644 index 131117104..000000000 --- a/heat/db/sqlalchemy/migrate_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -https://opendev.org/openstack/sqlalchemy-migrate diff --git a/heat/db/sqlalchemy/migrate_repo/__init__.py b/heat/db/sqlalchemy/migrate_repo/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/heat/db/sqlalchemy/migrate_repo/__init__.py +++ /dev/null diff --git a/heat/db/sqlalchemy/migrate_repo/manage.py b/heat/db/sqlalchemy/migrate_repo/manage.py deleted file mode 100755 index 41cba1adb..000000000 --- a/heat/db/sqlalchemy/migrate_repo/manage.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/heat/db/sqlalchemy/migrate_repo/migrate.cfg b/heat/db/sqlalchemy/migrate_repo/migrate.cfg deleted file mode 100644 index 134fc065d..000000000 --- a/heat/db/sqlalchemy/migrate_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=heat - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/heat/db/sqlalchemy/migrate_repo/versions/073_newton.py b/heat/db/sqlalchemy/migrate_repo/versions/073_newton.py deleted file mode 100644 index 3293b91ce..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/073_newton.py +++ /dev/null @@ -1,387 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import uuid - -import sqlalchemy - -from heat.db.sqlalchemy import types - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData() - meta.bind = migrate_engine - - raw_template_files = sqlalchemy.Table( - 'raw_template_files', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, - primary_key=True, - nullable=False), - sqlalchemy.Column('files', types.Json), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - raw_template = sqlalchemy.Table( - 'raw_template', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('template', types.LongText), - sqlalchemy.Column('files', types.Json), - sqlalchemy.Column('environment', types.Json), - sqlalchemy.Column('files_id', sqlalchemy.Integer(), - sqlalchemy.ForeignKey( - 'raw_template_files.id', - name='raw_tmpl_files_fkey_ref')), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - user_creds = sqlalchemy.Table( - 'user_creds', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, - primary_key=True, nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('username', sqlalchemy.String(255)), - sqlalchemy.Column('password', sqlalchemy.String(255)), - sqlalchemy.Column('region_name', sqlalchemy.String(length=255)), - sqlalchemy.Column('decrypt_method', sqlalchemy.String(length=64)), - sqlalchemy.Column('tenant', sqlalchemy.String(1024)), - sqlalchemy.Column('auth_url', sqlalchemy.Text), - sqlalchemy.Column('tenant_id', sqlalchemy.String(256)), - sqlalchemy.Column('trust_id', sqlalchemy.String(255)), - sqlalchemy.Column('trustor_user_id', sqlalchemy.String(64)), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - stack = sqlalchemy.Table( - 'stack', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), - primary_key=True, nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - sqlalchemy.Column('name', sqlalchemy.String(255)), - sqlalchemy.Column('raw_template_id', - sqlalchemy.Integer, - sqlalchemy.ForeignKey('raw_template.id'), - nullable=False), - sqlalchemy.Column('prev_raw_template_id', - sqlalchemy.Integer, - sqlalchemy.ForeignKey('raw_template.id')), - sqlalchemy.Column('user_creds_id', sqlalchemy.Integer, - sqlalchemy.ForeignKey('user_creds.id')), - sqlalchemy.Column('username', sqlalchemy.String(256)), - sqlalchemy.Column('owner_id', sqlalchemy.String(36)), - sqlalchemy.Column('action', sqlalchemy.String(255)), - sqlalchemy.Column('status', sqlalchemy.String(255)), - sqlalchemy.Column('status_reason', sqlalchemy.Text), - sqlalchemy.Column('timeout', sqlalchemy.Integer), - sqlalchemy.Column('tenant', sqlalchemy.String(256)), - sqlalchemy.Column('disable_rollback', sqlalchemy.Boolean, - nullable=False), - sqlalchemy.Column('stack_user_project_id', - sqlalchemy.String(length=64)), - sqlalchemy.Column('backup', sqlalchemy.Boolean, default=False), - sqlalchemy.Column('nested_depth', sqlalchemy.Integer, default=0), - sqlalchemy.Column('convergence', sqlalchemy.Boolean, default=False), - sqlalchemy.Column('current_traversal', sqlalchemy.String(36)), - sqlalchemy.Column('current_deps', types.Json), - sqlalchemy.Column('parent_resource_name', sqlalchemy.String(255)), - sqlalchemy.Index('ix_stack_name', 'name', mysql_length=255), - sqlalchemy.Index('ix_stack_tenant', 'tenant', mysql_length=255), - sqlalchemy.Index('ix_stack_owner_id', 'owner_id', mysql_length=36), - - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - resource = sqlalchemy.Table( - 'resource', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('uuid', sqlalchemy.String(36), unique=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('nova_instance', sqlalchemy.String(255)), - sqlalchemy.Column('name', sqlalchemy.String(255)), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('action', sqlalchemy.String(255)), - sqlalchemy.Column('status', sqlalchemy.String(255)), - sqlalchemy.Column('status_reason', sqlalchemy.Text), - sqlalchemy.Column('stack_id', sqlalchemy.String(36), - sqlalchemy.ForeignKey('stack.id'), nullable=False), - sqlalchemy.Column('rsrc_metadata', types.LongText), - sqlalchemy.Column('properties_data', types.Json), - sqlalchemy.Column('engine_id', sqlalchemy.String(length=36)), - sqlalchemy.Column('atomic_key', sqlalchemy.Integer), - sqlalchemy.Column('needed_by', types.List), - sqlalchemy.Column('requires', types.List), - sqlalchemy.Column('replaces', sqlalchemy.Integer), - sqlalchemy.Column('replaced_by', sqlalchemy.Integer), - sqlalchemy.Column('current_template_id', sqlalchemy.Integer, - sqlalchemy.ForeignKey('raw_template.id')), - sqlalchemy.Column('properties_data_encrypted', - sqlalchemy.Boolean, - default=False), - sqlalchemy.Column('root_stack_id', sqlalchemy.String(36)), - sqlalchemy.Index('ix_resource_root_stack_id', - 'root_stack_id', - mysql_length=36), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - resource_data = sqlalchemy.Table( - 'resource_data', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('key', sqlalchemy.String(255)), - sqlalchemy.Column('value', sqlalchemy.Text), - sqlalchemy.Column('redact', sqlalchemy.Boolean), - sqlalchemy.Column('decrypt_method', sqlalchemy.String(length=64)), - sqlalchemy.Column('resource_id', - sqlalchemy.Integer, - sqlalchemy.ForeignKey('resource.id', - name='fk_resource_id', - ondelete='CASCADE'), - nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - event = sqlalchemy.Table( - 'event', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('uuid', sqlalchemy.String(36), - default=lambda: str(uuid.uuid4()), unique=True), - sqlalchemy.Column('stack_id', sqlalchemy.String(36), - sqlalchemy.ForeignKey('stack.id'), nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('resource_action', sqlalchemy.String(255)), - sqlalchemy.Column('resource_status', sqlalchemy.String(255)), - sqlalchemy.Column('resource_name', sqlalchemy.String(255)), - sqlalchemy.Column('physical_resource_id', sqlalchemy.String(255)), - sqlalchemy.Column('resource_status_reason', sqlalchemy.String(255)), - sqlalchemy.Column('resource_type', sqlalchemy.String(255)), - sqlalchemy.Column('resource_properties', sqlalchemy.PickleType), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - watch_rule = sqlalchemy.Table( - 'watch_rule', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('name', sqlalchemy.String(255)), - sqlalchemy.Column('state', sqlalchemy.String(255)), - sqlalchemy.Column('rule', types.LongText), - sqlalchemy.Column('last_evaluated', sqlalchemy.DateTime), - sqlalchemy.Column('stack_id', sqlalchemy.String(36), - sqlalchemy.ForeignKey('stack.id'), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - watch_data = sqlalchemy.Table( - 'watch_data', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('data', types.LongText), - sqlalchemy.Column('watch_rule_id', sqlalchemy.Integer, - sqlalchemy.ForeignKey('watch_rule.id'), - nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - stack_lock = sqlalchemy.Table( - 'stack_lock', meta, - sqlalchemy.Column('stack_id', sqlalchemy.String(length=36), - sqlalchemy.ForeignKey('stack.id'), - primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('engine_id', sqlalchemy.String(length=36)), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - software_config = sqlalchemy.Table( - 'software_config', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), - primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('name', sqlalchemy.String(255)), - sqlalchemy.Column('group', sqlalchemy.String(255)), - sqlalchemy.Column('config', types.LongText), - sqlalchemy.Column('tenant', sqlalchemy.String(64), - nullable=False, - index=True), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - software_deployment = sqlalchemy.Table( - 'software_deployment', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), - primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime, - index=True), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('server_id', sqlalchemy.String(36), - nullable=False, - index=True), - sqlalchemy.Column('config_id', - sqlalchemy.String(36), - sqlalchemy.ForeignKey('software_config.id'), - nullable=False), - sqlalchemy.Column('input_values', types.Json), - sqlalchemy.Column('output_values', types.Json), - sqlalchemy.Column('action', sqlalchemy.String(255)), - sqlalchemy.Column('status', sqlalchemy.String(255)), - sqlalchemy.Column('status_reason', sqlalchemy.Text), - sqlalchemy.Column('tenant', sqlalchemy.String(64), - nullable=False, - index=True), - sqlalchemy.Column('stack_user_project_id', - sqlalchemy.String(length=64)), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - snapshot = sqlalchemy.Table( - 'snapshot', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), - primary_key=True, - nullable=False), - sqlalchemy.Column('stack_id', - sqlalchemy.String(36), - sqlalchemy.ForeignKey('stack.id'), - nullable=False), - sqlalchemy.Column('name', sqlalchemy.String(255)), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('status', sqlalchemy.String(255)), - sqlalchemy.Column('status_reason', sqlalchemy.String(255)), - sqlalchemy.Column('data', types.Json), - sqlalchemy.Column('tenant', sqlalchemy.String(64), - nullable=False, - index=True), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - service = sqlalchemy.Table( - 'service', meta, - sqlalchemy.Column('id', sqlalchemy.String(36), primary_key=True, - default=lambda: str(uuid.uuid4())), - sqlalchemy.Column('engine_id', sqlalchemy.String(36), nullable=False), - sqlalchemy.Column('host', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('hostname', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('binary', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('topic', sqlalchemy.String(255), nullable=False), - sqlalchemy.Column('report_interval', sqlalchemy.Integer, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('deleted_at', sqlalchemy.DateTime), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - stack_tag = sqlalchemy.Table( - 'stack_tag', meta, - sqlalchemy.Column('id', - sqlalchemy.Integer, - primary_key=True, - nullable=False), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - sqlalchemy.Column('tag', sqlalchemy.Unicode(80)), - sqlalchemy.Column('stack_id', - sqlalchemy.String(36), - sqlalchemy.ForeignKey('stack.id'), - nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - sync_point = sqlalchemy.Table( - 'sync_point', meta, - sqlalchemy.Column('entity_id', sqlalchemy.String(36)), - sqlalchemy.Column('traversal_id', sqlalchemy.String(36)), - sqlalchemy.Column('is_update', sqlalchemy.Boolean), - sqlalchemy.Column('atomic_key', sqlalchemy.Integer, - nullable=False), - sqlalchemy.Column('stack_id', sqlalchemy.String(36), - nullable=False), - sqlalchemy.Column('input_data', types.Json), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - - sqlalchemy.PrimaryKeyConstraint('entity_id', - 'traversal_id', - 'is_update'), - sqlalchemy.ForeignKeyConstraint(['stack_id'], ['stack.id'], - name='fk_stack_id'), - - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - - tables = ( - raw_template_files, - raw_template, - user_creds, - stack, - resource, - resource_data, - event, - watch_rule, - watch_data, - stack_lock, - software_config, - software_deployment, - snapshot, - service, - stack_tag, - sync_point, - ) - - for index, table in enumerate(tables): - try: - table.create() - except Exception: - # If an error occurs, drop all tables created so far to return - # to the previously existing state. - meta.drop_all(tables=tables[:index]) - raise diff --git a/heat/db/sqlalchemy/migrate_repo/versions/074_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/074_placeholder.py deleted file mode 100644 index e46d47580..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/074_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Newton backports. -# Do not use this number for new Ocata work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/075_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/075_placeholder.py deleted file mode 100644 index e46d47580..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/075_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Newton backports. -# Do not use this number for new Ocata work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/076_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/076_placeholder.py deleted file mode 100644 index e46d47580..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/076_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Newton backports. -# Do not use this number for new Ocata work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/077_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/077_placeholder.py deleted file mode 100644 index e46d47580..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/077_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Newton backports. -# Do not use this number for new Ocata work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/078_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/078_placeholder.py deleted file mode 100644 index e46d47580..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/078_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Newton backports. -# Do not use this number for new Ocata work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/079_resource_properties_data.py b/heat/db/sqlalchemy/migrate_repo/versions/079_resource_properties_data.py deleted file mode 100644 index 574450654..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/079_resource_properties_data.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset import constraint -import sqlalchemy - -from heat.db.sqlalchemy import types - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData(bind=migrate_engine) - - resource_properties_data = sqlalchemy.Table( - 'resource_properties_data', meta, - sqlalchemy.Column('id', sqlalchemy.Integer, - primary_key=True, - nullable=False), - sqlalchemy.Column('data', types.Json), - sqlalchemy.Column('encrypted', sqlalchemy.Boolean), - sqlalchemy.Column('created_at', sqlalchemy.DateTime), - sqlalchemy.Column('updated_at', sqlalchemy.DateTime), - mysql_engine='InnoDB', - mysql_charset='utf8' - ) - resource_properties_data.create() - - resource = sqlalchemy.Table('resource', meta, autoload=True) - rsrc_prop_data_id = sqlalchemy.Column('rsrc_prop_data_id', - sqlalchemy.Integer) - rsrc_prop_data_id.create(resource) - res_fkey = constraint.ForeignKeyConstraint( - columns=[resource.c.rsrc_prop_data_id], - refcolumns=[resource_properties_data.c.id], - name='rsrc_rsrc_prop_data_ref') - res_fkey.create() - - event = sqlalchemy.Table('event', meta, autoload=True) - rsrc_prop_data_id = sqlalchemy.Column('rsrc_prop_data_id', - sqlalchemy.Integer) - rsrc_prop_data_id.create(event) - ev_fkey = constraint.ForeignKeyConstraint( - columns=[event.c.rsrc_prop_data_id], - refcolumns=[resource_properties_data.c.id], - name='ev_rsrc_prop_data_ref') - ev_fkey.create() diff --git a/heat/db/sqlalchemy/migrate_repo/versions/080_resource_attrs_data.py b/heat/db/sqlalchemy/migrate_repo/versions/080_resource_attrs_data.py deleted file mode 100644 index 4636d1e76..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/080_resource_attrs_data.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset import constraint -import sqlalchemy - - -def upgrade(migrate_engine): - meta = sqlalchemy.MetaData(bind=migrate_engine) - - resource = sqlalchemy.Table('resource', meta, autoload=True) - resource_properties_data = sqlalchemy.Table('resource_properties_data', - meta, autoload=True) - attr_data_id = sqlalchemy.Column('attr_data_id', - sqlalchemy.Integer) - attr_data_id.create(resource) - res_fkey = constraint.ForeignKeyConstraint( - columns=[resource.c.attr_data_id], - refcolumns=[resource_properties_data.c.id], - name='rsrc_attr_data_ref') - res_fkey.create() diff --git a/heat/db/sqlalchemy/migrate_repo/versions/082_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/082_placeholder.py deleted file mode 100644 index 4378ff335..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/082_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Pike backports. -# Do not use this number for new Queens work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/083_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/083_placeholder.py deleted file mode 100644 index 4378ff335..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/083_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Pike backports. -# Do not use this number for new Queens work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/084_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/084_placeholder.py deleted file mode 100644 index 4378ff335..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/084_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Pike backports. -# Do not use this number for new Queens work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/085_placeholder.py b/heat/db/sqlalchemy/migrate_repo/versions/085_placeholder.py deleted file mode 100644 index 4378ff335..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/085_placeholder.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Pike backports. -# Do not use this number for new Queens work, which starts after -# all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/heat/db/sqlalchemy/migrate_repo/versions/086_drop_watch_rule_watch_data_tables.py b/heat/db/sqlalchemy/migrate_repo/versions/086_drop_watch_rule_watch_data_tables.py deleted file mode 100644 index a99ac5c4e..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/086_drop_watch_rule_watch_data_tables.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate import ForeignKeyConstraint -from sqlalchemy.engine import reflection -from sqlalchemy import MetaData -from sqlalchemy import Table - - -def upgrade(engine): - meta = MetaData() - meta.bind = engine - - def _get_columns(source_table, params): - columns = set() - for column in params: - columns.add(source_table.c[column]) - return columns - - def _remove_foreign_key_constraints(engine, meta, table_name): - inspector = reflection.Inspector.from_engine(engine) - - for fk in inspector.get_foreign_keys(table_name): - source_table = Table(table_name, meta, autoload=True) - target_table = Table(fk['referred_table'], meta, autoload=True) - - fkey = ForeignKeyConstraint( - columns=_get_columns(source_table, fk['constrained_columns']), - refcolumns=_get_columns(target_table, fk['referred_columns']), - name=fk['name']) - fkey.drop() - - def _drop_table_and_indexes(meta, table_name): - table = Table(table_name, meta, autoload=True) - for index in table.indexes: - index.drop() - table.drop() - - table_names = ('watch_data', 'watch_rule') - - for table_name in table_names: - _remove_foreign_key_constraints(engine, meta, table_name) - _drop_table_and_indexes(meta, table_name) diff --git a/heat/db/sqlalchemy/migrate_repo/versions/__init__.py b/heat/db/sqlalchemy/migrate_repo/versions/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/heat/db/sqlalchemy/migrate_repo/versions/__init__.py +++ /dev/null diff --git a/heat/db/sqlalchemy/migration.py b/heat/db/sqlalchemy/migration.py deleted file mode 100644 index 7f030d90a..000000000 --- a/heat/db/sqlalchemy/migration.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -from oslo_db.sqlalchemy import migration as oslo_migration - - -INIT_VERSION = 72 - - -def db_sync(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_sync(engine, path, version, - init_version=INIT_VERSION) - - -def db_version(engine): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version(engine, path, INIT_VERSION) - - -def db_version_control(engine, version=None): - path = os.path.join(os.path.abspath(os.path.dirname(__file__)), - 'migrate_repo') - return oslo_migration.db_version_control(engine, path, version) diff --git a/heat/db/sqlalchemy/utils.py b/heat/db/sqlalchemy/utils.py deleted file mode 100644 index 5fff4f763..000000000 --- a/heat/db/sqlalchemy/utils.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# SQLAlchemy helper functions - -import sqlalchemy -from sqlalchemy.orm import exc -import tenacity - - -def clone_table(name, parent, meta, newcols=None, ignorecols=None, - swapcols=None, ignorecons=None): - """Helper function that clones parent table schema onto new table. - - :param name: new table name - :param parent: parent table to copy schema from - :param newcols: names of new columns to be added - :param ignorecols: names of columns to be ignored while cloning - :param swapcols: alternative column schema - :param ignorecons: names of constraints to be ignored - - :return: sqlalchemy.Table instance - """ - - newcols = newcols or [] - ignorecols = ignorecols or [] - swapcols = swapcols or {} - ignorecons = ignorecons or [] - - cols = [c.copy() for c in parent.columns - if c.name not in ignorecols - if c.name not in swapcols] - cols.extend(swapcols.values()) - cols.extend(newcols) - new_table = sqlalchemy.Table(name, meta, *(cols)) - - def _is_ignorable(cons): - # consider constraints on columns only - if hasattr(cons, 'columns'): - for col in ignorecols: - if col in cons.columns: - return True - - return False - - constraints = [c.copy(target_table=new_table) for c in parent.constraints - if c.name not in ignorecons - if not _is_ignorable(c)] - - for c in constraints: - new_table.append_constraint(c) - - new_table.create() - return new_table - - -def migrate_data(migrate_engine, - table, - new_table, - skip_columns=None): - - table_name = table.name - - list_of_rows = list(table.select().execute()) - - colnames = [c.name for c in table.columns] - - for row in list_of_rows: - values = dict(zip(colnames, - map(lambda colname: getattr(row, colname), - colnames))) - if skip_columns is not None: - for column in skip_columns: - del values[column] - - migrate_engine.execute(new_table.insert(values)) - - table.drop() - - new_table.rename(table_name) - - -def retry_on_stale_data_error(func): - wrapper = tenacity.retry( - stop=tenacity.stop_after_attempt(3), - retry=tenacity.retry_if_exception_type(exc.StaleDataError), - reraise=True) - return wrapper(func) diff --git a/heat/db/sqlalchemy/types.py b/heat/db/types.py index d454024c6..d454024c6 100644 --- a/heat/db/sqlalchemy/types.py +++ b/heat/db/types.py diff --git a/heat/db/sqlalchemy/migrate_repo/versions/081_placeholder.py b/heat/db/utils.py index 4378ff335..67e2d541e 100644 --- a/heat/db/sqlalchemy/migrate_repo/versions/081_placeholder.py +++ b/heat/db/utils.py @@ -11,10 +11,15 @@ # License for the specific language governing permissions and limitations # under the License. -# This is a placeholder for Pike backports. -# Do not use this number for new Queens work, which starts after -# all the placeholders. +# SQLAlchemy helper functions +from sqlalchemy.orm import exc +import tenacity -def upgrade(migrate_engine): - pass + +def retry_on_stale_data_error(func): + wrapper = tenacity.retry( + stop=tenacity.stop_after_attempt(3), + retry=tenacity.retry_if_exception_type(exc.StaleDataError), + reraise=True) + return wrapper(func) diff --git a/heat/engine/resources/openstack/neutron/lbaas/health_monitor.py b/heat/engine/resources/openstack/neutron/lbaas/health_monitor.py index da945aab1..f8ed35931 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/health_monitor.py +++ b/heat/engine/resources/openstack/neutron/lbaas/health_monitor.py @@ -29,7 +29,12 @@ class HealthMonitor(neutron.NeutronResource): which watches status of the load balanced servers. """ - support_status = support.SupportStatus(version='6.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='6.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/l7policy.py b/heat/engine/resources/openstack/neutron/lbaas/l7policy.py index 8a1233cf0..50cefef92 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/l7policy.py +++ b/heat/engine/resources/openstack/neutron/lbaas/l7policy.py @@ -35,7 +35,12 @@ class L7Policy(neutron.NeutronResource): listener.default_pool_id. """ - support_status = support.SupportStatus(version='7.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='7.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/l7rule.py b/heat/engine/resources/openstack/neutron/lbaas/l7rule.py index 28d0052e3..8e8d6105d 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/l7rule.py +++ b/heat/engine/resources/openstack/neutron/lbaas/l7rule.py @@ -27,7 +27,12 @@ class L7Rule(neutron.NeutronResource): be matched and how it should be matched. """ - support_status = support.SupportStatus(version='7.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='7.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/listener.py b/heat/engine/resources/openstack/neutron/lbaas/listener.py index 1e60679b9..682e04190 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/listener.py +++ b/heat/engine/resources/openstack/neutron/lbaas/listener.py @@ -31,7 +31,12 @@ class Listener(neutron.NeutronResource): which represent a listening endpoint for the vip. """ - support_status = support.SupportStatus(version='6.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='6.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/loadbalancer.py b/heat/engine/resources/openstack/neutron/lbaas/loadbalancer.py index 3b1cda94e..3cc231d6e 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/loadbalancer.py +++ b/heat/engine/resources/openstack/neutron/lbaas/loadbalancer.py @@ -33,7 +33,12 @@ class LoadBalancer(neutron.NeutronResource): which allows traffic to be directed between servers. """ - support_status = support.SupportStatus(version='6.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='6.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/pool.py b/heat/engine/resources/openstack/neutron/lbaas/pool.py index 63d2868a6..ccebd2694 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/pool.py +++ b/heat/engine/resources/openstack/neutron/lbaas/pool.py @@ -32,7 +32,12 @@ class Pool(neutron.NeutronResource): and the nodes themselves. """ - support_status = support.SupportStatus(version='6.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='6.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/resources/openstack/neutron/lbaas/pool_member.py b/heat/engine/resources/openstack/neutron/lbaas/pool_member.py index d14ea4092..04c817815 100644 --- a/heat/engine/resources/openstack/neutron/lbaas/pool_member.py +++ b/heat/engine/resources/openstack/neutron/lbaas/pool_member.py @@ -29,7 +29,12 @@ class PoolMember(neutron.NeutronResource): A pool member represents a single backend node. """ - support_status = support.SupportStatus(version='6.0.0') + support_status = support.SupportStatus( + status=support.HIDDEN, + version='21.0.0', + message=_('Use octavia instead.'), + previous_status=support.SupportStatus(version='6.0.0') + ) required_service_extension = 'lbaasv2' diff --git a/heat/engine/service_software_config.py b/heat/engine/service_software_config.py index 637d7874e..195397101 100644 --- a/heat/engine/service_software_config.py +++ b/heat/engine/service_software_config.py @@ -23,7 +23,7 @@ from urllib import parse from heat.common import crypt from heat.common import exception from heat.common.i18n import _ -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine import api from heat.engine import resource from heat.engine import scheduler diff --git a/heat/engine/template_files.py b/heat/engine/template_files.py index 55a54e0d6..bcbe68b07 100644 --- a/heat/engine/template_files.py +++ b/heat/engine/template_files.py @@ -17,7 +17,7 @@ import weakref from heat.common import context from heat.common import exception from heat.common.i18n import _ -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import raw_template_files _d = weakref.WeakValueDictionary() diff --git a/heat/engine/worker.py b/heat/engine/worker.py index 28e2424e0..c2163ff59 100644 --- a/heat/engine/worker.py +++ b/heat/engine/worker.py @@ -24,7 +24,7 @@ from osprofiler import profiler from heat.common import context from heat.common import messaging as rpc_messaging -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine import check_resource from heat.engine import node_data from heat.engine import stack as parser diff --git a/heat/objects/event.py b/heat/objects/event.py index f51a6ac8b..c9a356e97 100644 --- a/heat/objects/event.py +++ b/heat/objects/event.py @@ -19,7 +19,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields from heat.common import identifier -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import resource_properties_data as rpd diff --git a/heat/objects/raw_template.py b/heat/objects/raw_template.py index 3a1ba35db..616411247 100644 --- a/heat/objects/raw_template.py +++ b/heat/objects/raw_template.py @@ -24,7 +24,7 @@ from oslo_versionedobjects import fields from heat.common import crypt from heat.common import environment_format as env_fmt -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields diff --git a/heat/objects/raw_template_files.py b/heat/objects/raw_template_files.py index 1afed6e7e..36d6cde91 100644 --- a/heat/objects/raw_template_files.py +++ b/heat/objects/raw_template_files.py @@ -16,7 +16,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields diff --git a/heat/objects/resource.py b/heat/objects/resource.py index c5aacd4fc..590ea5a59 100644 --- a/heat/objects/resource.py +++ b/heat/objects/resource.py @@ -26,7 +26,7 @@ import tenacity from heat.common import crypt from heat.common import exception from heat.common.i18n import _ -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields from heat.objects import resource_data diff --git a/heat/objects/resource_data.py b/heat/objects/resource_data.py index 4b6419bcc..7602bc040 100644 --- a/heat/objects/resource_data.py +++ b/heat/objects/resource_data.py @@ -19,7 +19,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields from heat.common import exception -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base diff --git a/heat/objects/resource_properties_data.py b/heat/objects/resource_properties_data.py index c1c313c25..4b6a18dad 100644 --- a/heat/objects/resource_properties_data.py +++ b/heat/objects/resource_properties_data.py @@ -19,7 +19,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields from heat.common import crypt -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import fields as heat_fields diff --git a/heat/objects/service.py b/heat/objects/service.py index 416e9081c..121af6145 100644 --- a/heat/objects/service.py +++ b/heat/objects/service.py @@ -18,7 +18,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields from heat.common import service_utils -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base diff --git a/heat/objects/snapshot.py b/heat/objects/snapshot.py index 95c758d05..23d569a40 100644 --- a/heat/objects/snapshot.py +++ b/heat/objects/snapshot.py @@ -18,7 +18,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields diff --git a/heat/objects/software_config.py b/heat/objects/software_config.py index 02ac8a908..e0d9d42c7 100644 --- a/heat/objects/software_config.py +++ b/heat/objects/software_config.py @@ -12,13 +12,12 @@ # License for the specific language governing permissions and limitations # under the License. - """SoftwareConfig object.""" from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields diff --git a/heat/objects/software_deployment.py b/heat/objects/software_deployment.py index 2ff47ad93..564a48465 100644 --- a/heat/objects/software_deployment.py +++ b/heat/objects/software_deployment.py @@ -18,7 +18,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields from heat.objects import software_config diff --git a/heat/objects/stack.py b/heat/objects/stack.py index 349aaada8..a5c781d16 100644 --- a/heat/objects/stack.py +++ b/heat/objects/stack.py @@ -22,7 +22,7 @@ from oslo_versionedobjects import fields from heat.common import exception from heat.common.i18n import _ from heat.common import identifier -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields from heat.objects import raw_template diff --git a/heat/objects/stack_lock.py b/heat/objects/stack_lock.py index 1a00c40e9..9d699ee00 100644 --- a/heat/objects/stack_lock.py +++ b/heat/objects/stack_lock.py @@ -11,13 +11,12 @@ # License for the specific language governing permissions and limitations # under the License. - """StackLock object.""" from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base diff --git a/heat/objects/stack_tag.py b/heat/objects/stack_tag.py index 3a75fffbf..50d86deda 100644 --- a/heat/objects/stack_tag.py +++ b/heat/objects/stack_tag.py @@ -17,7 +17,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base diff --git a/heat/objects/sync_point.py b/heat/objects/sync_point.py index 264d323b8..750a551b1 100644 --- a/heat/objects/sync_point.py +++ b/heat/objects/sync_point.py @@ -11,14 +11,12 @@ # License for the specific language governing permissions and limitations # under the License. - """SyncPoint object.""" - from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base from heat.objects import fields as heat_fields diff --git a/heat/objects/user_creds.py b/heat/objects/user_creds.py index abfa3610a..d482ee229 100644 --- a/heat/objects/user_creds.py +++ b/heat/objects/user_creds.py @@ -18,7 +18,7 @@ from oslo_versionedobjects import base from oslo_versionedobjects import fields -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.objects import base as heat_base diff --git a/heat/tests/common.py b/heat/tests/common.py index baafe94be..3013a1499 100644 --- a/heat/tests/common.py +++ b/heat/tests/common.py @@ -82,9 +82,11 @@ class HeatTestCase(testscenarios.WithScenarios, def setUp(self, mock_keystone=True, mock_resource_policy=True, quieten_logging=True, mock_find_file=True): - super(HeatTestCase, self).setUp() + super().setUp() self.setup_logging(quieten=quieten_logging) - self.warnings = self.useFixture(fixtures.WarningsCapture()) + + self.useFixture(utils.WarningsFixture()) + scheduler.ENABLE_SLEEP = False self.useFixture(fixtures.MonkeyPatch( 'heat.common.exception._FATAL_EXCEPTION_FORMAT_ERRORS', diff --git a/heat/tests/convergence/framework/engine_wrapper.py b/heat/tests/convergence/framework/engine_wrapper.py index b1dc5e6b4..c1fb47c01 100644 --- a/heat/tests/convergence/framework/engine_wrapper.py +++ b/heat/tests/convergence/framework/engine_wrapper.py @@ -11,7 +11,7 @@ # License for the specific language governing permissions and limitations # under the License. -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine import service from heat.engine import stack from heat.tests.convergence.framework import message_processor diff --git a/heat/tests/convergence/framework/reality.py b/heat/tests/convergence/framework/reality.py index 055782b38..0618c758d 100644 --- a/heat/tests/convergence/framework/reality.py +++ b/heat/tests/convergence/framework/reality.py @@ -14,7 +14,7 @@ from unittest import mock from heat.common import exception -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.tests import utils diff --git a/heat/tests/db/test_migrations.py b/heat/tests/db/test_migrations.py index 0dbf85bf3..ffad49f2e 100644 --- a/heat/tests/db/test_migrations.py +++ b/heat/tests/db/test_migrations.py @@ -1,4 +1,3 @@ -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -11,31 +10,20 @@ # License for the specific language governing permissions and limitations # under the License. -""" -Tests for database migrations. This test case reads the configuration -file test_migrations.conf for database connection settings -to use in the tests. For each connection found in the config file, -the test case runs a series of test cases to ensure that migrations work -properly both upgrading and downgrading, and that no data loss occurs -if possible. -""" +"""Tests for database migrations.""" +from alembic import command as alembic_api +from alembic import script as alembic_script import fixtures -import os - -from migrate.versioning import repository from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import test_fixtures from oslo_db.sqlalchemy import test_migrations -from oslo_db.sqlalchemy import utils from oslotest import base as test_base import sqlalchemy import testtools -from heat.db.sqlalchemy import migrate_repo -from heat.db.sqlalchemy import migration -from heat.db.sqlalchemy import models -from heat.tests import common +from heat.db import migration +from heat.db import models class DBNotAllowed(Exception): @@ -80,196 +68,147 @@ class TestBannedDBSchemaOperations(testtools.TestCase): self.assertRaises(DBNotAllowed, table.alter) -class HeatMigrationsCheckers(test_migrations.WalkVersionsMixin, - common.FakeLogMixin): - """Test sqlalchemy-migrate migrations.""" - - snake_walk = False - downgrade = False +class HeatModelsMigrationsSync(test_migrations.ModelsMigrationsSync): - @property - def INIT_VERSION(self): - return migration.INIT_VERSION + def setUp(self): + super().setUp() - @property - def REPOSITORY(self): - migrate_file = migrate_repo.__file__ - return repository.Repository( - os.path.abspath(os.path.dirname(migrate_file)) - ) + self.engine = enginefacade.writer.get_engine() + self.sessionmaker = enginefacade.writer.get_sessionmaker() - @property - def migration_api(self): - temp = __import__('oslo_db.sqlalchemy.migration', globals(), - locals(), ['versioning_api'], 0) - return temp.versioning_api + def get_metadata(self): + return models.BASE.metadata - @property - def migrate_engine(self): + def get_engine(self): return self.engine - def migrate_up(self, version, with_data=False): - """Check that migrations don't cause downtime. - - Schema migrations can be done online, allowing for rolling upgrades. - """ - # NOTE(xek): This is a list of migrations where we allow dropping - # things. The rules for adding exceptions are very very specific. - # Chances are you don't meet the critera. - # Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE - exceptions = [ - 64, # drop constraint - 86, # drop watch_rule/watch_data tables - ] - # Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE - - # NOTE(xek): We start requiring things be additive in - # liberty, so ignore all migrations before that point. - LIBERTY_START = 63 - - if version >= LIBERTY_START and version not in exceptions: - banned = ['Table', 'Column'] - else: - banned = None - with BannedDBSchemaOperations(banned): - super(HeatMigrationsCheckers, self).migrate_up(version, with_data) - - def test_walk_versions(self): - self.walk_versions(self.snake_walk, self.downgrade) - - def assertColumnExists(self, engine, table, column): - t = utils.get_table(engine, table) - self.assertIn(column, t.c) - - def assertColumnType(self, engine, table, column, sqltype): - t = utils.get_table(engine, table) - col = getattr(t.c, column) - self.assertIsInstance(col.type, sqltype) - - def assertColumnNotExists(self, engine, table, column): - t = utils.get_table(engine, table) - self.assertNotIn(column, t.c) - - def assertColumnIsNullable(self, engine, table, column): - t = utils.get_table(engine, table) - col = getattr(t.c, column) - self.assertTrue(col.nullable) - - def assertColumnIsNotNullable(self, engine, table, column_name): - table = utils.get_table(engine, table) - column = getattr(table.c, column_name) - self.assertFalse(column.nullable) - - def assertIndexExists(self, engine, table, index): - t = utils.get_table(engine, table) - index_names = [idx.name for idx in t.indexes] - self.assertIn(index, index_names) - - def assertIndexMembers(self, engine, table, index, members): - self.assertIndexExists(engine, table, index) - - t = utils.get_table(engine, table) - index_columns = [] - for idx in t.indexes: - if idx.name == index: - for ix in idx.columns: - index_columns.append(ix.name) - break - - self.assertEqual(sorted(members), sorted(index_columns)) - - def _check_073(self, engine, data): - # check if column still exists and is not nullable. - self.assertColumnIsNotNullable(engine, 'resource_data', 'resource_id') - # Ensure that only one foreign key exists and is created as expected. - inspector = sqlalchemy.engine.reflection.Inspector.from_engine(engine) - resource_data_fkeys = inspector.get_foreign_keys('resource_data') - self.assertEqual(1, len(resource_data_fkeys)) - fk = resource_data_fkeys[0] - self.assertEqual('fk_resource_id', fk['name']) - self.assertEqual(['resource_id'], fk['constrained_columns']) - self.assertEqual('resource', fk['referred_table']) - self.assertEqual(['id'], fk['referred_columns']) - - def _check_079(self, engine, data): - self.assertColumnExists(engine, 'resource', - 'rsrc_prop_data_id') - self.assertColumnExists(engine, 'event', - 'rsrc_prop_data_id') - column_list = [('id', False), - ('data', True), - ('encrypted', True), - ('updated_at', True), - ('created_at', True)] - - for column in column_list: - self.assertColumnExists(engine, - 'resource_properties_data', column[0]) - if not column[1]: - self.assertColumnIsNotNullable(engine, - 'resource_properties_data', - column[0]) - else: - self.assertColumnIsNullable(engine, - 'resource_properties_data', - column[0]) - - def _check_080(self, engine, data): - self.assertColumnExists(engine, 'resource', - 'attr_data_id') - - -class DbTestCase(test_fixtures.OpportunisticDBTestMixin, - test_base.BaseTestCase): - def setUp(self): - super(DbTestCase, self).setUp() + def db_sync(self, engine): + migration.db_sync(engine=engine) - self.engine = enginefacade.writer.get_engine() - self.sessionmaker = enginefacade.writer.get_sessionmaker() + def include_object(self, object_, name, type_, reflected, compare_to): + return True -class TestHeatMigrationsMySQL(DbTestCase, HeatMigrationsCheckers): +class ModelsMigrationsSyncMysql( + HeatModelsMigrationsSync, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): FIXTURE = test_fixtures.MySQLOpportunisticFixture -class TestHeatMigrationsPostgreSQL(DbTestCase, HeatMigrationsCheckers): +class ModelsMigrationsSyncPostgres( + HeatModelsMigrationsSync, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): FIXTURE = test_fixtures.PostgresqlOpportunisticFixture -class TestHeatMigrationsSQLite(DbTestCase, HeatMigrationsCheckers): +class ModelsMigrationsSyncSQLite( + HeatModelsMigrationsSync, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): pass -class ModelsMigrationSyncMixin(object): +class DatabaseSanityChecks( + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): + def setUp(self): + super().setUp() + self.engine = enginefacade.writer.get_engine() + # self.patch(api, 'get_engine', lambda: self.engine) + self.config = migration._find_alembic_conf() + self.init_version = migration.ALEMBIC_INIT_VERSION - def get_metadata(self): - return models.BASE.metadata + def test_single_base_revision(self): + """Ensure we only have a single base revision. - def get_engine(self): - return self.engine + There's no good reason for us to have diverging history, so validate + that only one base revision exists. This will prevent simple errors + where people forget to specify the base revision. If this fail for your + change, look for migrations that do not have a 'revises' line in them. + """ + script = alembic_script.ScriptDirectory.from_config(self.config) + self.assertEqual(1, len(script.get_bases())) - def db_sync(self, engine): - migration.db_sync(engine=engine) + def test_single_head_revision(self): + """Ensure we only have a single head revision. - def include_object(self, object_, name, type_, reflected, compare_to): - if name in ['migrate_version'] and type_ == 'table': - return False - return True + There's no good reason for us to have diverging history, so validate + that only one head revision exists. This will prevent merge conflicts + adding additional head revision points. If this fail for your change, + look for migrations with the same 'revises' line in them. + """ + script = alembic_script.ScriptDirectory.from_config(self.config) + self.assertEqual(1, len(script.get_heads())) -class ModelsMigrationsSyncMysql(DbTestCase, - ModelsMigrationSyncMixin, - test_migrations.ModelsMigrationsSync): - FIXTURE = test_fixtures.MySQLOpportunisticFixture +class MigrationsWalk( + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): + # Migrations can take a long time, particularly on underpowered CI nodes. + # Give them some breathing room. + TIMEOUT_SCALING_FACTOR = 4 + + def setUp(self): + super().setUp() + self.engine = enginefacade.writer.get_engine() + # self.patch(api, 'get_engine', lambda: self.engine) + self.config = migration._find_alembic_conf() + self.init_version = migration.ALEMBIC_INIT_VERSION + def _migrate_up(self, revision, connection): + check_method = getattr(self, f'_check_{revision}', None) + if revision != self.init_version: # no tests for the initial revision + self.assertIsNotNone( + check_method, + f"DB Migration {revision} doesn't have a test; add one" + ) -class ModelsMigrationsSyncPostgres(DbTestCase, - ModelsMigrationSyncMixin, - test_migrations.ModelsMigrationsSync): - FIXTURE = test_fixtures.PostgresqlOpportunisticFixture + pre_upgrade = getattr(self, f'_pre_upgrade_{revision}', None) + if pre_upgrade: + pre_upgrade(connection) + alembic_api.upgrade(self.config, revision) -class ModelsMigrationsSyncSQLite(DbTestCase, - ModelsMigrationSyncMixin, - test_migrations.ModelsMigrationsSync): + if check_method: + check_method(connection) + + def test_walk_versions(self): + with self.engine.begin() as connection: + self.config.attributes['connection'] = connection + script = alembic_script.ScriptDirectory.from_config(self.config) + revisions = list(script.walk_revisions()) + # Need revisions from older to newer so the walk works as intended + revisions.reverse() + for revision_script in revisions: + self._migrate_up(revision_script.revision, connection) + + +class TestMigrationsWalkSQLite( + MigrationsWalk, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): pass + + +class TestMigrationsWalkMySQL( + MigrationsWalk, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): + FIXTURE = test_fixtures.MySQLOpportunisticFixture + + +class TestMigrationsWalkPostgreSQL( + MigrationsWalk, + test_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): + FIXTURE = test_fixtures.PostgresqlOpportunisticFixture diff --git a/heat/tests/db/test_sqlalchemy_api.py b/heat/tests/db/test_sqlalchemy_api.py index 07a1c2670..a4f3309f0 100644 --- a/heat/tests/db/test_sqlalchemy_api.py +++ b/heat/tests/db/test_sqlalchemy_api.py @@ -30,8 +30,8 @@ from heat.common import context from heat.common import exception from heat.common import short_id from heat.common import template_format -from heat.db.sqlalchemy import api as db_api -from heat.db.sqlalchemy import models +from heat.db import api as db_api +from heat.db import models from heat.engine.clients.os import glance from heat.engine.clients.os import nova from heat.engine import environment @@ -2308,7 +2308,7 @@ class DBAPIStackTest(common.HeatTestCase): create_stack(self.ctx, self.template, self.user_creds, deleted_at=deleted) - with mock.patch('heat.db.sqlalchemy.api._purge_stacks') as mock_ps: + with mock.patch('heat.db.api._purge_stacks') as mock_ps: db_api.purge_deleted(age=0, batch_size=2) self.assertEqual(4, mock_ps.call_count) diff --git a/heat/tests/db/test_sqlalchemy_filters.py b/heat/tests/db/test_sqlalchemy_filters.py index e3424aee7..6e8f2d312 100644 --- a/heat/tests/db/test_sqlalchemy_filters.py +++ b/heat/tests/db/test_sqlalchemy_filters.py @@ -13,7 +13,7 @@ from unittest import mock -from heat.db.sqlalchemy import filters as db_filters +from heat.db import filters as db_filters from heat.tests import common diff --git a/heat/tests/db/test_sqlalchemy_types.py b/heat/tests/db/test_sqlalchemy_types.py index 47b460a55..051962d18 100644 --- a/heat/tests/db/test_sqlalchemy_types.py +++ b/heat/tests/db/test_sqlalchemy_types.py @@ -15,7 +15,7 @@ from sqlalchemy.dialects.mysql import base as mysql_base from sqlalchemy.dialects.sqlite import base as sqlite_base from sqlalchemy import types -from heat.db.sqlalchemy import types as db_types +from heat.db import types as db_types from heat.tests import common diff --git a/heat/tests/db/test_utils.py b/heat/tests/db/test_utils.py deleted file mode 100644 index c6bc30922..000000000 --- a/heat/tests/db/test_utils.py +++ /dev/null @@ -1,209 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from heat.db.sqlalchemy import utils as migrate_utils -from heat.tests import common -from heat.tests import utils - -from sqlalchemy.schema import (Column, MetaData, Table) -from sqlalchemy.types import (Boolean, String, Integer) -from sqlalchemy import (CheckConstraint, UniqueConstraint, - ForeignKey, ForeignKeyConstraint) - - -def _has_constraint(cset, ctype, cname): - for c in cset: - if (isinstance(c, ctype) - and c.name == cname): - return True - else: - return False - - -class DBMigrationUtilsTest(common.HeatTestCase): - - def setUp(self): - super(DBMigrationUtilsTest, self).setUp() - self.engine = utils.get_engine() - - def test_clone_table_adds_or_deletes_columns(self): - meta = MetaData() - meta.bind = self.engine - - table = Table('dummy', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Boolean, default=False) - ) - table.create() - - newcols = [ - Column('B', Boolean, default=False), - Column('C', String(255), default='foobar') - ] - ignorecols = [ - table.c.A.name - ] - new_table = migrate_utils.clone_table('new_dummy', table, meta, - newcols=newcols, - ignorecols=ignorecols) - - col_names = [c.name for c in new_table.columns] - - self.assertEqual(3, len(col_names)) - self.assertIsNotNone(new_table.c.B) - self.assertIsNotNone(new_table.c.C) - self.assertNotIn('A', col_names) - - def test_clone_table_swaps_columns(self): - meta = MetaData() - meta.bind = self.engine - - table = Table("dummy1", - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Boolean, default=False), - ) - table.create() - - swapcols = { - 'A': Column('A', Integer, default=1), - } - - new_table = migrate_utils.clone_table('swap_dummy', table, meta, - swapcols=swapcols) - - self.assertIsNotNone(new_table.c.A) - self.assertEqual(Integer, type(new_table.c.A.type)) - - def test_clone_table_retains_constraints(self): - meta = MetaData() - meta.bind = self.engine - parent = Table('parent', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Integer), - Column('B', Integer), - Column('C', Integer, - CheckConstraint('C>100', name="above 100")), - Column('D', Integer, unique=True), - - UniqueConstraint('A', 'B', name='uix_1') - ) - parent.create() - - child = Table('child', - meta, - Column('id', String(36), - ForeignKey('parent.id', name="parent_ref"), - primary_key=True, - nullable=False), - Column('A', Boolean, default=False) - ) - child.create() - - ignorecols = [ - parent.c.D.name, - ] - - new_parent = migrate_utils.clone_table('new_parent', parent, meta, - ignorecols=ignorecols) - new_child = migrate_utils.clone_table('new_child', child, meta) - - self.assertTrue(_has_constraint(new_parent.constraints, - UniqueConstraint, 'uix_1')) - self.assertTrue(_has_constraint(new_parent.c.C.constraints, - CheckConstraint, 'above 100')) - self.assertTrue(_has_constraint(new_child.constraints, - ForeignKeyConstraint, 'parent_ref')) - - def test_clone_table_ignores_constraints(self): - meta = MetaData() - meta.bind = self.engine - table = Table('constraints_check', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Integer), - Column('B', Integer), - Column('C', Integer, - CheckConstraint('C>100', name="above 100")), - - UniqueConstraint('A', 'B', name='uix_1') - ) - table.create() - - ignorecons = [ - 'uix_1', - ] - - new_table = migrate_utils.clone_table('constraints_check_tmp', table, - meta, ignorecons=ignorecons) - self.assertFalse(_has_constraint(new_table.constraints, - UniqueConstraint, 'uix_1')) - - def test_migrate_data(self): - meta = MetaData(bind=self.engine) - - # create TableA - table_a = Table('TableA', - meta, - Column('id', Integer, primary_key=True), - Column('first', String(8), nullable=False), - Column('second', Integer)) - table_a.create() - - # update it with sample data - values = [ - {'id': 1, 'first': 'a'}, - {'id': 2, 'first': 'b'}, - {'id': 3, 'first': 'c'} - ] - - for value in values: - self.engine.execute(table_a.insert(values=value)) - - # create TableB similar to TableA, except column 'second' - table_b = Table('TableB', - meta, - Column('id', Integer, primary_key=True), - Column('first', String(8), nullable=False)) - table_b.create() - - # migrate data - migrate_utils.migrate_data(self.engine, - table_a, - table_b, - ['second']) - - # validate table_a is dropped - self.assertTrue(self.engine.dialect.has_table( - self.engine.connect(), - 'TableA'), - 'Data migration failed to drop source table') - - # validate table_b is updated with data from table_a - table_b_rows = list(table_b.select().execute()) - self.assertEqual(3, - len(table_b_rows), - "Data migration is failed") - table_b_values = [] - for row in table_b_rows: - table_b_values.append({'id': row.id, - 'first': row.first}) - - self.assertEqual(values, - table_b_values, - "Data migration failed with invalid data copy") diff --git a/heat/tests/engine/service/test_software_config.py b/heat/tests/engine/service/test_software_config.py index ecd7bef62..9aba366a5 100644 --- a/heat/tests/engine/service/test_software_config.py +++ b/heat/tests/engine/service/test_software_config.py @@ -22,7 +22,7 @@ from oslo_utils import timeutils from heat.common import crypt from heat.common import exception from heat.common import template_format -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine.clients.os import swift from heat.engine.clients.os import zaqar from heat.engine import service diff --git a/heat/tests/engine/service/test_stack_update.py b/heat/tests/engine/service/test_stack_update.py index f1df832af..0712e888f 100644 --- a/heat/tests/engine/service/test_stack_update.py +++ b/heat/tests/engine/service/test_stack_update.py @@ -24,7 +24,7 @@ from heat.common import exception from heat.common import messaging from heat.common import service_utils from heat.common import template_format -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine.clients.os import glance from heat.engine.clients.os import nova from heat.engine.clients.os import swift diff --git a/heat/tests/engine/test_engine_worker.py b/heat/tests/engine/test_engine_worker.py index 0eed84fb0..de873b2aa 100644 --- a/heat/tests/engine/test_engine_worker.py +++ b/heat/tests/engine/test_engine_worker.py @@ -15,7 +15,7 @@ from unittest import mock -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine import check_resource from heat.engine import stack as parser from heat.engine import template as templatem diff --git a/heat/tests/test_common_service_utils.py b/heat/tests/test_common_service_utils.py index 0745e8f85..2b363607c 100644 --- a/heat/tests/test_common_service_utils.py +++ b/heat/tests/test_common_service_utils.py @@ -18,7 +18,7 @@ from oslo_utils import timeutils import uuid from heat.common import service_utils -from heat.db.sqlalchemy import models +from heat.db import models from heat.tests import common diff --git a/heat/tests/test_engine_api_utils.py b/heat/tests/test_engine_api_utils.py index 8703fcdda..6cee13b03 100644 --- a/heat/tests/test_engine_api_utils.py +++ b/heat/tests/test_engine_api_utils.py @@ -21,7 +21,7 @@ from oslo_utils import timeutils from heat.common import exception from heat.common import template_format from heat.common import timeutils as heat_timeutils -from heat.db.sqlalchemy import models +from heat.db import models from heat.engine import api from heat.engine.cfn import parameters as cfn_param from heat.engine import event diff --git a/heat/tests/test_event.py b/heat/tests/test_event.py index 8d9a2521b..0efdf9caf 100644 --- a/heat/tests/test_event.py +++ b/heat/tests/test_event.py @@ -16,7 +16,7 @@ from unittest import mock from oslo_config import cfg import uuid -from heat.db.sqlalchemy import models +from heat.db import models from heat.engine import event from heat.engine import stack from heat.engine import template diff --git a/heat/tests/test_resource.py b/heat/tests/test_resource.py index b4f8f05c0..e3d0c97ef 100644 --- a/heat/tests/test_resource.py +++ b/heat/tests/test_resource.py @@ -27,8 +27,8 @@ from heat.common import exception from heat.common.i18n import _ from heat.common import short_id from heat.common import timeutils -from heat.db.sqlalchemy import api as db_api -from heat.db.sqlalchemy import models +from heat.db import api as db_api +from heat.db import models from heat.engine import attributes from heat.engine.cfn import functions as cfn_funcs from heat.engine import clients diff --git a/heat/tests/test_resource_properties_data.py b/heat/tests/test_resource_properties_data.py index 2ef5374cf..bcf677564 100644 --- a/heat/tests/test_resource_properties_data.py +++ b/heat/tests/test_resource_properties_data.py @@ -13,7 +13,7 @@ from oslo_config import cfg -from heat.db.sqlalchemy import models +from heat.db import models from heat.objects import resource_properties_data as rpd_object from heat.tests import common from heat.tests import utils diff --git a/heat/tests/test_signal.py b/heat/tests/test_signal.py index bb0b8313b..b2d754abb 100644 --- a/heat/tests/test_signal.py +++ b/heat/tests/test_signal.py @@ -20,7 +20,7 @@ from oslo_utils import timeutils from heat.common import exception from heat.common import template_format -from heat.db.sqlalchemy import models +from heat.db import models from heat.engine.clients.os import heat_plugin from heat.engine.clients.os.keystone import fake_keystoneclient as fake_ks from heat.engine.clients.os import swift diff --git a/heat/tests/test_stack.py b/heat/tests/test_stack.py index 35374190c..278df8e23 100644 --- a/heat/tests/test_stack.py +++ b/heat/tests/test_stack.py @@ -27,7 +27,7 @@ from heat.common import context from heat.common import exception from heat.common import template_format from heat.common import timeutils -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine.clients.os import keystone from heat.engine.clients.os.keystone import fake_keystoneclient as fake_ks from heat.engine.clients.os import nova diff --git a/heat/tests/test_stack_update.py b/heat/tests/test_stack_update.py index a466cef4f..b441f3ebf 100644 --- a/heat/tests/test_stack_update.py +++ b/heat/tests/test_stack_update.py @@ -17,7 +17,7 @@ from unittest import mock from heat.common import exception from heat.common import template_format -from heat.db.sqlalchemy import api as db_api +from heat.db import api as db_api from heat.engine.clients.os.keystone import fake_keystoneclient from heat.engine import environment from heat.engine import resource diff --git a/heat/tests/utils.py b/heat/tests/utils.py index 19fc1a06f..44c81c294 100644 --- a/heat/tests/utils.py +++ b/heat/tests/utils.py @@ -14,16 +14,18 @@ import random import string import uuid +import warnings import fixtures from oslo_config import cfg from oslo_db import options from oslo_serialization import jsonutils import sqlalchemy +from sqlalchemy import exc as sqla_exc from heat.common import context -from heat.db.sqlalchemy import api as db_api -from heat.db.sqlalchemy import models +from heat.db import api as db_api +from heat.db import models from heat.engine import environment from heat.engine import node_data from heat.engine import resource @@ -235,6 +237,112 @@ class ForeignKeyConstraintFixture(fixtures.Fixture): self.addCleanup(disable_fks) +class WarningsFixture(fixtures.Fixture): + """Filters out warnings during test runs.""" + + def setUp(self): + super().setUp() + + self._original_warning_filters = warnings.filters[:] + + warnings.simplefilter("once", DeprecationWarning) + + # Enable deprecation warnings for heat itself to capture upcoming + # SQLAlchemy changes + + warnings.filterwarnings( + 'ignore', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'error', + module='heat', + category=sqla_exc.SADeprecationWarning, + ) + + # ...but filter everything out until we get around to fixing them + # TODO(stephenfin): Fix all of these + + warnings.filterwarnings( + 'ignore', + module='heat', + message=r'The Engine.execute\(\) method is considered legacy ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='The current statement is being autocommitted using ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='Using strings to indicate column or relationship paths ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message=r'The Query.get\(\) method is considered legacy ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='The Session.transaction attribute is considered legacy ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='The Session.begin.subtransactions flag is deprecated ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='The autoload parameter is deprecated ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message='The ``bind`` argument for schema methods that invoke ', + category=sqla_exc.SADeprecationWarning, + ) + + warnings.filterwarnings( + 'ignore', + module='heat', + message=r'The legacy calling style of select\(\) is deprecated ', + category=sqla_exc.SADeprecationWarning, + ) + + # Enable general SQLAlchemy warnings also to ensure we're not doing + # silly stuff. It's possible that we'll need to filter things out here + # with future SQLAlchemy versions, but that's a good thing + + warnings.filterwarnings( + 'error', + module='heat', + category=sqla_exc.SAWarning, + ) + + self.addCleanup(self._reset_warning_filters) + + def _reset_warning_filters(self): + warnings.filters[:] = self._original_warning_filters + + class AnyInstance(object): """Comparator for validating allowed instance type.""" diff --git a/heat_upgradetests/post_test_hook.sh b/heat_upgradetests/post_test_hook.sh index e69de29bb..a9bf588e2 100755 --- a/heat_upgradetests/post_test_hook.sh +++ b/heat_upgradetests/post_test_hook.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/heat_upgradetests/pre_test_hook.sh b/heat_upgradetests/pre_test_hook.sh index e69de29bb..a9bf588e2 100755 --- a/heat_upgradetests/pre_test_hook.sh +++ b/heat_upgradetests/pre_test_hook.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/releasenotes/notes/lbaasv2-hidden-a8f82ddfdba911eb.yaml b/releasenotes/notes/lbaasv2-hidden-a8f82ddfdba911eb.yaml new file mode 100644 index 000000000..1d064a876 --- /dev/null +++ b/releasenotes/notes/lbaasv2-hidden-a8f82ddfdba911eb.yaml @@ -0,0 +1,13 @@ +--- +upgrade: + - | + The following resources types are now hidden. Neutron LBaaS v2 was already + retired thus these resource types can no longer be used. + + - ``OS::Neutron::LBaaS::LoadBalancer`` + - ``OS::Neutron::LBaaS::Listener`` + - ``OS::Neutron::LBaaS::Pool`` + - ``OS::Neutron::LBaaS::PoolMember`` + - ``OS::Neutron::LBaaS::HealthMonitor`` + - ``OS::Neutron::LBaaS::L7Policy`` + - ``OS::Neutron::LBaaS::L7Rule`` diff --git a/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml b/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml index faaa3283c..9b6809680 100644 --- a/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml +++ b/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml @@ -5,7 +5,7 @@ features: for default roles and system scope. This is part of a broader community effort to support read-only roles and implement secure, consistent default policies. - + Refer to `the Keystone documentation`__ for more information on the reason for these changes. diff --git a/releasenotes/notes/switch-to-alembic-7af6f8e71e4bf56b.yaml b/releasenotes/notes/switch-to-alembic-7af6f8e71e4bf56b.yaml new file mode 100644 index 000000000..883f74db3 --- /dev/null +++ b/releasenotes/notes/switch-to-alembic-7af6f8e71e4bf56b.yaml @@ -0,0 +1,22 @@ +--- +upgrade: + - | + The database migration engine has changed from `sqlalchemy-migrate`__ to + `alembic`__. For most deployments, this should have minimal to no impact + and the switch should be mostly transparent. The main user-facing impact is + the change in schema versioning. While sqlalchemy-migrate used a linear, + integer-based versioning scheme, which required placeholder migrations to + allow for potential migration backports, alembic uses a distributed version + control-like schema where a migration's ancestor is encoded in the file and + branches are possible. The alembic migration files therefore use a + arbitrary UUID-like naming scheme and the ``heat-manage db_sync`` command + now expects such an version when manually specifying the version that should + be applied. For example:: + + $ heat-manage db_sync c6214ca60943 + + Attempting to specify an sqlalchemy-migrate-based version will result in an + error. + + .. __: https://sqlalchemy-migrate.readthedocs.io/en/latest/ + .. __: https://alembic.sqlalchemy.org/en/latest/ diff --git a/requirements.txt b/requirements.txt index 18150cfc4..dddc4239e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,8 @@ # date but we do not test them so no guarantee of having them all correct. If # you find any incorrect lower bounds, let us know or propose a fix. -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - pbr>=3.1.1 # Apache-2.0 +alembic>=1.8.0 # MIT Babel!=2.4.0,>=2.3.4 # BSD ddt>=1.4.1 # MIT croniter>=0.3.4 # MIT License @@ -66,7 +63,6 @@ requests>=2.23.0 # Apache-2.0 tenacity>=6.1.0 # Apache-2.0 Routes>=2.3.1 # MIT SQLAlchemy>=1.4.0 # MIT -sqlalchemy-migrate>=0.13.0 # Apache-2.0 stevedore>=3.1.0 # Apache-2.0 WebOb>=1.7.1 # MIT yaql>=1.1.3 # Apache 2.0 License @@ -5,9 +5,11 @@ minversion = 3.18.0 [testenv] basepython = python3 -setenv = VIRTUAL_ENV={envdir} - PYTHONWARNINGS=default::DeprecationWarning - OS_TEST_PATH=heat/tests +setenv = + OS_TEST_PATH=heat/tests + PYTHONDONTWRITEBYTECODE=1 +# TODO(stephenfin): Remove once we bump our upper-constraint to SQLAlchemy 2.0 + SQLALCHEMY_WARN_20=1 usedevelop = True deps = -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master} -r{toxinidir}/requirements.txt |