diff options
21 files changed, 648 insertions, 322 deletions
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..0ae225c54 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,29 @@ +--- +default_language_version: + # force all unspecified python hooks to run python3 + python: python3 +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: mixed-line-ending + args: ['--fix', 'lf'] + exclude: '.*\.(svg)$' + - id: check-byte-order-marker + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: debug-statements + - id: check-yaml + files: .*\.(yaml|yml)$ + exclude: 'rally-scenarios/heat-fakevirt.yaml' + - repo: local + hooks: + - id: flake8 + name: flake8 + additional_dependencies: + - hacking>=3.1.0,<3.2.0 + language: python + entry: flake8 + files: '^.*\.py$' + exclude: '^(doc|releasenotes|tools)/.*$' diff --git a/api-ref/source/v1/events.inc b/api-ref/source/v1/events.inc index a0f72ece2..ef3c08da2 100644 --- a/api-ref/source/v1/events.inc +++ b/api-ref/source/v1/events.inc @@ -169,7 +169,7 @@ Shows details for an event. .. rest_status_code:: success status.yaml - - 200 + - 200 .. rest_status_code:: error status.yaml diff --git a/api-ref/source/v1/services.inc b/api-ref/source/v1/services.inc index 67b344cd5..3ce6b94ab 100644 --- a/api-ref/source/v1/services.inc +++ b/api-ref/source/v1/services.inc @@ -11,7 +11,7 @@ Show orchestration engine status Enables administrative users to view details for all orchestration engines. -Orchestration engine details include engine id, binary, topic name, host, +Orchestration engine details include engine id, binary, topic name, host, report interval, last updated time, health status, and host name. Response Codes diff --git a/devstack/upgrade/resources.sh b/devstack/upgrade/resources.sh index 277e37dcf..b16b5deae 100755 --- a/devstack/upgrade/resources.sh +++ b/devstack/upgrade/resources.sh @@ -41,8 +41,6 @@ function _write_heat_integrationtests { cat > $upgrade_tests <<EOF heat_tempest_plugin.tests.api heat_integrationtests.functional.test_autoscaling -heat_integrationtests.functional.test_cancel_update -heat_integrationtests.functional.test_create_update heat_integrationtests.functional.test_instance_group heat_integrationtests.functional.test_resource_group.ResourceGroupTest heat_integrationtests.functional.test_resource_group.ResourceGroupUpdatePolicyTest diff --git a/doc/source/conf.py b/doc/source/conf.py index 913262c95..0cf74e0e1 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -178,6 +178,7 @@ apidoc_separate_modules = True apidoc_excluded_paths = [ 'cmd', 'cloudinit', + 'db/sqlalchemy/migrations/versions', 'db/sqlalchemy/migrate_repo/versions', 'engine/resources/aws', 'engine/resources/openstack', diff --git a/heat/db/sqlalchemy/alembic.ini b/heat/db/sqlalchemy/alembic.ini new file mode 100644 index 000000000..b4a2f235a --- /dev/null +++ b/heat/db/sqlalchemy/alembic.ini @@ -0,0 +1,72 @@ +[alembic] +# path to migration scripts +script_location = %(here)s/migrations + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# version location specification; This defaults +# to heat/db/sqlalchemy/migrations/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:heat/db/sqlalchemy/migrations/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +sqlalchemy.url = sqlite:///heat.db + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/heat/db/sqlalchemy/migrate_repo/migrate.cfg b/heat/db/sqlalchemy/migrate_repo/migrate.cfg index 134fc065d..3a6ff0abd 100644 --- a/heat/db/sqlalchemy/migrate_repo/migrate.cfg +++ b/heat/db/sqlalchemy/migrate_repo/migrate.cfg @@ -5,16 +5,16 @@ repository_id=heat # The name of the database table used to track the schema version. # This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. version_table=migrate_version -# When committing a change script, Migrate will attempt to generate the +# When committing a change script, Migrate will attempt to generate the # sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually # be using to ensure your updates to that database work properly. # This must be a list; example: ['postgres','sqlite'] required_dbs=[] diff --git a/heat/db/sqlalchemy/migrations/README.rst b/heat/db/sqlalchemy/migrations/README.rst new file mode 100644 index 000000000..b2283fc82 --- /dev/null +++ b/heat/db/sqlalchemy/migrations/README.rst @@ -0,0 +1,15 @@ +Database migrations +=================== + +This directory contains migrations for the database. These are implemented +using `alembic`__, a lightweight database migration tool designed for usage +with `SQLAlchemy`__. + +The best place to start understanding Alembic is with its own `tutorial`__. You +can also play around with the :command:`alembic` command:: + + $ alembic --help + +.. __: https://alembic.sqlalchemy.org/en/latest/ +.. __: https://www.sqlalchemy.org/ +.. __: https://alembic.sqlalchemy.org/en/latest/tutorial.html diff --git a/heat/db/sqlalchemy/migrations/env.py b/heat/db/sqlalchemy/migrations/env.py new file mode 100644 index 000000000..138d28374 --- /dev/null +++ b/heat/db/sqlalchemy/migrations/env.py @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from heat.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# this is the MetaData object for the various models in the database +target_metadata = models.BASE.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL and not an Engine, though an + Engine is acceptable here as well. By skipping the Engine creation we + don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine and associate a connection + with the context. + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/heat/db/sqlalchemy/migrations/script.py.mako b/heat/db/sqlalchemy/migrations/script.py.mako new file mode 100644 index 000000000..120937fbc --- /dev/null +++ b/heat/db/sqlalchemy/migrations/script.py.mako @@ -0,0 +1,20 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} diff --git a/heat/db/sqlalchemy/migrations/versions/c6214ca60943_initial_revision.py b/heat/db/sqlalchemy/migrations/versions/c6214ca60943_initial_revision.py new file mode 100644 index 000000000..b6a4e7bbc --- /dev/null +++ b/heat/db/sqlalchemy/migrations/versions/c6214ca60943_initial_revision.py @@ -0,0 +1,392 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Initial revision + +Revision ID: c6214ca60943 +Revises: +Create Date: 2023-03-22 18:04:02.387269 +""" + +from alembic import op +import sqlalchemy as sa + +import heat.db.sqlalchemy.types + +# revision identifiers, used by Alembic. +revision = 'c6214ca60943' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + 'raw_template_files', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('files', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'resource_properties_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('data', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column('encrypted', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'service', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('engine_id', sa.String(length=36), nullable=False), + sa.Column('host', sa.String(length=255), nullable=False), + sa.Column('hostname', sa.String(length=255), nullable=False), + sa.Column('binary', sa.String(length=255), nullable=False), + sa.Column('topic', sa.String(length=255), nullable=False), + sa.Column('report_interval', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'software_config', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('group', sa.String(length=255), nullable=True), + sa.Column('config', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_software_config_tenant'), + 'software_config', + ['tenant'], + unique=False, + ) + op.create_table( + 'user_creds', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('username', sa.String(length=255), nullable=True), + sa.Column('password', sa.String(length=255), nullable=True), + sa.Column('region_name', sa.String(length=255), nullable=True), + sa.Column('decrypt_method', sa.String(length=64), nullable=True), + sa.Column('tenant', sa.String(length=1024), nullable=True), + sa.Column('auth_url', sa.Text(), nullable=True), + sa.Column('tenant_id', sa.String(length=256), nullable=True), + sa.Column('trust_id', sa.String(length=255), nullable=True), + sa.Column('trustor_user_id', sa.String(length=64), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'raw_template', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('template', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column('files', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column( + 'environment', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column('files_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['files_id'], + ['raw_template_files.id'], + name='raw_tmpl_files_fkey_ref', + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'software_deployment', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('server_id', sa.String(length=36), nullable=False), + sa.Column('config_id', sa.String(length=36), nullable=False), + sa.Column( + 'input_values', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column( + 'output_values', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.Column( + 'stack_user_project_id', sa.String(length=64), nullable=True + ), + sa.ForeignKeyConstraint( + ['config_id'], + ['software_config.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'ix_software_deployment_created_at', + 'software_deployment', + ['created_at'], + unique=False, + ) + op.create_index( + op.f('ix_software_deployment_server_id'), + 'software_deployment', + ['server_id'], + unique=False, + ) + op.create_index( + op.f('ix_software_deployment_tenant'), + 'software_deployment', + ['tenant'], + unique=False, + ) + op.create_table( + 'stack', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('raw_template_id', sa.Integer(), nullable=False), + sa.Column('prev_raw_template_id', sa.Integer(), nullable=True), + sa.Column('user_creds_id', sa.Integer(), nullable=True), + sa.Column('username', sa.String(length=256), nullable=True), + sa.Column('owner_id', sa.String(length=36), nullable=True), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('timeout', sa.Integer(), nullable=True), + sa.Column('tenant', sa.String(length=256), nullable=True), + sa.Column('disable_rollback', sa.Boolean(), nullable=False), + sa.Column( + 'stack_user_project_id', sa.String(length=64), nullable=True + ), + sa.Column('backup', sa.Boolean(), nullable=True), + sa.Column('nested_depth', sa.Integer(), nullable=True), + sa.Column('convergence', sa.Boolean(), nullable=True), + sa.Column('current_traversal', sa.String(length=36), nullable=True), + sa.Column( + 'current_deps', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column( + 'parent_resource_name', sa.String(length=255), nullable=True + ), + sa.ForeignKeyConstraint( + ['prev_raw_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['raw_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['user_creds_id'], + ['user_creds.id'], + ), + sa.PrimaryKeyConstraint('id'), + ) + op.create_index( + 'ix_stack_name', 'stack', ['name'], unique=False, mysql_length=255 + ) + op.create_index( + 'ix_stack_tenant', 'stack', ['tenant'], unique=False, mysql_length=255 + ) + op.create_index( + op.f('ix_stack_owner_id'), 'stack', ['owner_id'], unique=False + ) + op.create_table( + 'event', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('resource_action', sa.String(length=255), nullable=True), + sa.Column('resource_status', sa.String(length=255), nullable=True), + sa.Column('resource_name', sa.String(length=255), nullable=True), + sa.Column( + 'physical_resource_id', sa.String(length=255), nullable=True + ), + sa.Column( + 'resource_status_reason', sa.String(length=255), nullable=True + ), + sa.Column('resource_type', sa.String(length=255), nullable=True), + sa.Column('resource_properties', sa.PickleType(), nullable=True), + sa.Column('rsrc_prop_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['rsrc_prop_data_id'], + ['resource_properties_data.id'], + name='ev_rsrc_prop_data_ref', + ), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid'), + mysql_engine='InnoDB', + ) + op.create_table( + 'resource', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=True), + sa.Column('nova_instance', sa.String(length=255), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('action', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text(), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column( + 'rsrc_metadata', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column( + 'properties_data', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column('engine_id', sa.String(length=36), nullable=True), + sa.Column('atomic_key', sa.Integer(), nullable=True), + sa.Column('needed_by', heat.db.sqlalchemy.types.List(), nullable=True), + sa.Column('requires', heat.db.sqlalchemy.types.List(), nullable=True), + sa.Column('replaces', sa.Integer(), nullable=True), + sa.Column('replaced_by', sa.Integer(), nullable=True), + sa.Column('current_template_id', sa.Integer(), nullable=True), + sa.Column('properties_data_encrypted', sa.Boolean(), nullable=True), + sa.Column('root_stack_id', sa.String(length=36), nullable=True), + sa.Column('rsrc_prop_data_id', sa.Integer(), nullable=True), + sa.Column('attr_data_id', sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ['attr_data_id'], + ['resource_properties_data.id'], + name='rsrc_attr_data_ref', + ), + sa.ForeignKeyConstraint( + ['current_template_id'], + ['raw_template.id'], + ), + sa.ForeignKeyConstraint( + ['rsrc_prop_data_id'], + ['resource_properties_data.id'], + name='rsrc_rsrc_prop_data_ref', + ), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('uuid'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_resource_root_stack_id'), + 'resource', + ['root_stack_id'], + unique=False, + ) + op.create_table( + 'snapshot', + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.String(length=255), nullable=True), + sa.Column('data', heat.db.sqlalchemy.types.Json(), nullable=True), + sa.Column('tenant', sa.String(length=64), nullable=False), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_index( + op.f('ix_snapshot_tenant'), 'snapshot', ['tenant'], unique=False + ) + op.create_table( + 'stack_lock', + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('engine_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('stack_id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'stack_tag', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('tag', sa.Unicode(length=80), nullable=True), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) + op.create_table( + 'sync_point', + sa.Column('entity_id', sa.String(length=36), nullable=False), + sa.Column('traversal_id', sa.String(length=36), nullable=False), + sa.Column('is_update', sa.Boolean(), nullable=False), + sa.Column('atomic_key', sa.Integer(), nullable=False), + sa.Column('stack_id', sa.String(length=36), nullable=False), + sa.Column( + 'input_data', heat.db.sqlalchemy.types.Json(), nullable=True + ), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.ForeignKeyConstraint( + ['stack_id'], + ['stack.id'], + ), + sa.PrimaryKeyConstraint('entity_id', 'traversal_id', 'is_update'), + ) + op.create_table( + 'resource_data', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('key', sa.String(length=255), nullable=True), + sa.Column('value', sa.Text(), nullable=True), + sa.Column('redact', sa.Boolean(), nullable=True), + sa.Column('decrypt_method', sa.String(length=64), nullable=True), + sa.Column('resource_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ['resource_id'], + ['resource.id'], + name='fk_resource_id', + ondelete='CASCADE', + ), + sa.PrimaryKeyConstraint('id'), + mysql_engine='InnoDB', + ) diff --git a/heat/db/sqlalchemy/utils.py b/heat/db/sqlalchemy/utils.py index 5fff4f763..67e2d541e 100644 --- a/heat/db/sqlalchemy/utils.py +++ b/heat/db/sqlalchemy/utils.py @@ -13,83 +13,10 @@ # SQLAlchemy helper functions -import sqlalchemy from sqlalchemy.orm import exc import tenacity -def clone_table(name, parent, meta, newcols=None, ignorecols=None, - swapcols=None, ignorecons=None): - """Helper function that clones parent table schema onto new table. - - :param name: new table name - :param parent: parent table to copy schema from - :param newcols: names of new columns to be added - :param ignorecols: names of columns to be ignored while cloning - :param swapcols: alternative column schema - :param ignorecons: names of constraints to be ignored - - :return: sqlalchemy.Table instance - """ - - newcols = newcols or [] - ignorecols = ignorecols or [] - swapcols = swapcols or {} - ignorecons = ignorecons or [] - - cols = [c.copy() for c in parent.columns - if c.name not in ignorecols - if c.name not in swapcols] - cols.extend(swapcols.values()) - cols.extend(newcols) - new_table = sqlalchemy.Table(name, meta, *(cols)) - - def _is_ignorable(cons): - # consider constraints on columns only - if hasattr(cons, 'columns'): - for col in ignorecols: - if col in cons.columns: - return True - - return False - - constraints = [c.copy(target_table=new_table) for c in parent.constraints - if c.name not in ignorecons - if not _is_ignorable(c)] - - for c in constraints: - new_table.append_constraint(c) - - new_table.create() - return new_table - - -def migrate_data(migrate_engine, - table, - new_table, - skip_columns=None): - - table_name = table.name - - list_of_rows = list(table.select().execute()) - - colnames = [c.name for c in table.columns] - - for row in list_of_rows: - values = dict(zip(colnames, - map(lambda colname: getattr(row, colname), - colnames))) - if skip_columns is not None: - for column in skip_columns: - del values[column] - - migrate_engine.execute(new_table.insert(values)) - - table.drop() - - new_table.rename(table_name) - - def retry_on_stale_data_error(func): wrapper = tenacity.retry( stop=tenacity.stop_after_attempt(3), diff --git a/heat/engine/resources/openstack/senlin/cluster.py b/heat/engine/resources/openstack/senlin/cluster.py index 6d68bb919..57ccf7cee 100644 --- a/heat/engine/resources/openstack/senlin/cluster.py +++ b/heat/engine/resources/openstack/senlin/cluster.py @@ -232,7 +232,7 @@ class Cluster(res_base.BaseSenlinResource): 'enabled': p[self.P_ENABLED], } action = { - 'func': 'cluster_attach_policy', + 'func': 'attach_policy_to_cluster', 'params': params, 'action_id': None, 'done': False, @@ -283,7 +283,7 @@ class Cluster(res_base.BaseSenlinResource): 'enabled': p[self.P_ENABLED] } action = { - 'func': 'cluster_update_policy', + 'func': 'update_cluster_policy', 'params': params, 'action_id': None, 'done': False, @@ -296,7 +296,7 @@ class Cluster(res_base.BaseSenlinResource): 'enabled': p[self.P_ENABLED] } action = { - 'func': 'cluster_detach_policy', + 'func': 'detach_policy_from_cluster', 'params': params, 'action_id': None, 'done': False, @@ -309,7 +309,7 @@ class Cluster(res_base.BaseSenlinResource): 'enabled': p[self.P_ENABLED] } action = { - 'func': 'cluster_attach_policy', + 'func': 'attach_policy_to_cluster', 'params': params, 'action_id': None, 'done': False, @@ -338,7 +338,7 @@ class Cluster(res_base.BaseSenlinResource): params['number'] = params.pop(self.DESIRED_CAPACITY) params['cluster'] = self.resource_id action = { - 'func': 'cluster_resize', + 'func': 'resize_cluster', 'params': params, 'action_id': None, 'done': False, diff --git a/heat/engine/resources/openstack/senlin/node.py b/heat/engine/resources/openstack/senlin/node.py index 1809eb777..7d254cbe4 100644 --- a/heat/engine/resources/openstack/senlin/node.py +++ b/heat/engine/resources/openstack/senlin/node.py @@ -156,7 +156,7 @@ class Node(res_base.BaseSenlinResource): 'nodes': [self.resource_id], } action = { - 'func': 'cluster_del_nodes', + 'func': 'remove_nodes_from_cluster', 'action_id': None, 'params': params, 'done': False, @@ -179,7 +179,7 @@ class Node(res_base.BaseSenlinResource): 'nodes': [self.resource_id], } action = { - 'func': 'cluster_add_nodes', + 'func': 'add_nodes_to_cluster', 'action_id': None, 'params': params, 'done': False, diff --git a/heat/tests/db/test_utils.py b/heat/tests/db/test_utils.py deleted file mode 100644 index c6bc30922..000000000 --- a/heat/tests/db/test_utils.py +++ /dev/null @@ -1,209 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from heat.db.sqlalchemy import utils as migrate_utils -from heat.tests import common -from heat.tests import utils - -from sqlalchemy.schema import (Column, MetaData, Table) -from sqlalchemy.types import (Boolean, String, Integer) -from sqlalchemy import (CheckConstraint, UniqueConstraint, - ForeignKey, ForeignKeyConstraint) - - -def _has_constraint(cset, ctype, cname): - for c in cset: - if (isinstance(c, ctype) - and c.name == cname): - return True - else: - return False - - -class DBMigrationUtilsTest(common.HeatTestCase): - - def setUp(self): - super(DBMigrationUtilsTest, self).setUp() - self.engine = utils.get_engine() - - def test_clone_table_adds_or_deletes_columns(self): - meta = MetaData() - meta.bind = self.engine - - table = Table('dummy', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Boolean, default=False) - ) - table.create() - - newcols = [ - Column('B', Boolean, default=False), - Column('C', String(255), default='foobar') - ] - ignorecols = [ - table.c.A.name - ] - new_table = migrate_utils.clone_table('new_dummy', table, meta, - newcols=newcols, - ignorecols=ignorecols) - - col_names = [c.name for c in new_table.columns] - - self.assertEqual(3, len(col_names)) - self.assertIsNotNone(new_table.c.B) - self.assertIsNotNone(new_table.c.C) - self.assertNotIn('A', col_names) - - def test_clone_table_swaps_columns(self): - meta = MetaData() - meta.bind = self.engine - - table = Table("dummy1", - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Boolean, default=False), - ) - table.create() - - swapcols = { - 'A': Column('A', Integer, default=1), - } - - new_table = migrate_utils.clone_table('swap_dummy', table, meta, - swapcols=swapcols) - - self.assertIsNotNone(new_table.c.A) - self.assertEqual(Integer, type(new_table.c.A.type)) - - def test_clone_table_retains_constraints(self): - meta = MetaData() - meta.bind = self.engine - parent = Table('parent', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Integer), - Column('B', Integer), - Column('C', Integer, - CheckConstraint('C>100', name="above 100")), - Column('D', Integer, unique=True), - - UniqueConstraint('A', 'B', name='uix_1') - ) - parent.create() - - child = Table('child', - meta, - Column('id', String(36), - ForeignKey('parent.id', name="parent_ref"), - primary_key=True, - nullable=False), - Column('A', Boolean, default=False) - ) - child.create() - - ignorecols = [ - parent.c.D.name, - ] - - new_parent = migrate_utils.clone_table('new_parent', parent, meta, - ignorecols=ignorecols) - new_child = migrate_utils.clone_table('new_child', child, meta) - - self.assertTrue(_has_constraint(new_parent.constraints, - UniqueConstraint, 'uix_1')) - self.assertTrue(_has_constraint(new_parent.c.C.constraints, - CheckConstraint, 'above 100')) - self.assertTrue(_has_constraint(new_child.constraints, - ForeignKeyConstraint, 'parent_ref')) - - def test_clone_table_ignores_constraints(self): - meta = MetaData() - meta.bind = self.engine - table = Table('constraints_check', - meta, - Column('id', String(36), primary_key=True, - nullable=False), - Column('A', Integer), - Column('B', Integer), - Column('C', Integer, - CheckConstraint('C>100', name="above 100")), - - UniqueConstraint('A', 'B', name='uix_1') - ) - table.create() - - ignorecons = [ - 'uix_1', - ] - - new_table = migrate_utils.clone_table('constraints_check_tmp', table, - meta, ignorecons=ignorecons) - self.assertFalse(_has_constraint(new_table.constraints, - UniqueConstraint, 'uix_1')) - - def test_migrate_data(self): - meta = MetaData(bind=self.engine) - - # create TableA - table_a = Table('TableA', - meta, - Column('id', Integer, primary_key=True), - Column('first', String(8), nullable=False), - Column('second', Integer)) - table_a.create() - - # update it with sample data - values = [ - {'id': 1, 'first': 'a'}, - {'id': 2, 'first': 'b'}, - {'id': 3, 'first': 'c'} - ] - - for value in values: - self.engine.execute(table_a.insert(values=value)) - - # create TableB similar to TableA, except column 'second' - table_b = Table('TableB', - meta, - Column('id', Integer, primary_key=True), - Column('first', String(8), nullable=False)) - table_b.create() - - # migrate data - migrate_utils.migrate_data(self.engine, - table_a, - table_b, - ['second']) - - # validate table_a is dropped - self.assertTrue(self.engine.dialect.has_table( - self.engine.connect(), - 'TableA'), - 'Data migration failed to drop source table') - - # validate table_b is updated with data from table_a - table_b_rows = list(table_b.select().execute()) - self.assertEqual(3, - len(table_b_rows), - "Data migration is failed") - table_b_values = [] - for row in table_b_rows: - table_b_values.append({'id': row.id, - 'first': row.first}) - - self.assertEqual(values, - table_b_values, - "Data migration failed with invalid data copy") diff --git a/heat/tests/openstack/senlin/test_cluster.py b/heat/tests/openstack/senlin/test_cluster.py index 5a451801c..e30dd2583 100644 --- a/heat/tests/openstack/senlin/test_cluster.py +++ b/heat/tests/openstack/senlin/test_cluster.py @@ -144,7 +144,7 @@ class SenlinClusterTest(common.HeatTestCase): } self.senlin_mock.create_cluster.assert_called_once_with( **create_cluster_kwargs) - self.senlin_mock.cluster_attach_policy.assert_called_once_with( + self.senlin_mock.attach_policy_to_cluster.assert_called_once_with( **attach_policy_kwargs) def test_cluster_create_error(self): @@ -216,7 +216,7 @@ class SenlinClusterTest(common.HeatTestCase): props['desired_capacity'] = 10 rsrc_defns = template.Template(new_t).resource_definitions(self.stack) new_cluster = rsrc_defns['senlin-cluster'] - self.senlin_mock.cluster_resize.return_value = { + self.senlin_mock.resize_cluster.return_value = { 'action': 'fake-action'} self.senlin_mock.get_action.return_value = mock.Mock( status='SUCCEEDED') @@ -226,7 +226,7 @@ class SenlinClusterTest(common.HeatTestCase): 'adjustment_type': 'EXACT_CAPACITY', 'number': 10 } - self.senlin_mock.cluster_resize.assert_called_once_with( + self.senlin_mock.resize_cluster.assert_called_once_with( cluster=cluster.resource_id, **cluster_resize_kwargs) self.assertEqual(2, self.senlin_mock.get_action.call_count) @@ -243,9 +243,9 @@ class SenlinClusterTest(common.HeatTestCase): props['policies'] = [{'policy': 'new_policy'}] rsrc_defns = template.Template(new_t).resource_definitions(self.stack) new_cluster = rsrc_defns['senlin-cluster'] - self.senlin_mock.cluster_detach_policy.return_value = { + self.senlin_mock.detach_policy_from_cluster.return_value = { 'action': 'fake-action'} - self.senlin_mock.cluster_attach_policy.return_value = { + self.senlin_mock.attach_policy_to_cluster.return_value = { 'action': 'fake-action'} self.senlin_mock.get_action.return_value = mock.Mock( status='SUCCEEDED') @@ -257,10 +257,10 @@ class SenlinClusterTest(common.HeatTestCase): 'enabled': True, } self.assertEqual(2, - self.senlin_mock.cluster_attach_policy.call_count) - self.senlin_mock.cluster_detach_policy.assert_called_once_with( + self.senlin_mock.attach_policy_to_cluster.call_count) + self.senlin_mock.detach_policy_from_cluster.assert_called_once_with( **detach_policy_kwargs) - self.assertEqual(0, self.senlin_mock.cluster_update_policy.call_count) + self.assertEqual(0, self.senlin_mock.update_cluster_policy.call_count) self.assertEqual(3, self.senlin_mock.get_action.call_count) def test_cluster_update_policy_exists(self): @@ -270,7 +270,7 @@ class SenlinClusterTest(common.HeatTestCase): props['policies'] = [{'policy': 'fake_policy', 'enabled': False}] rsrc_defns = template.Template(new_t).resource_definitions(self.stack) new_cluster = rsrc_defns['senlin-cluster'] - self.senlin_mock.cluster_update_policy.return_value = { + self.senlin_mock.update_cluster_policy.return_value = { 'action': 'fake-action'} self.senlin_mock.get_action.return_value = mock.Mock( status='SUCCEEDED') @@ -281,10 +281,12 @@ class SenlinClusterTest(common.HeatTestCase): 'cluster': cluster.resource_id, 'enabled': False, } - self.senlin_mock.cluster_update_policy.assert_called_once_with( + self.senlin_mock.update_cluster_policy.assert_called_once_with( **update_policy_kwargs) - self.assertEqual(1, self.senlin_mock.cluster_attach_policy.call_count) - self.assertEqual(0, self.senlin_mock.cluster_detach_policy.call_count) + self.assertEqual(1, self.senlin_mock. + attach_policy_to_cluster.call_count) + self.assertEqual(0, self.senlin_mock. + detach_policy_from_cluster.call_count) def test_cluster_update_failed(self): cluster = self._create_cluster(self.t) @@ -293,7 +295,7 @@ class SenlinClusterTest(common.HeatTestCase): props['desired_capacity'] = 3 rsrc_defns = template.Template(new_t).resource_definitions(self.stack) update_snippet = rsrc_defns['senlin-cluster'] - self.senlin_mock.cluster_resize.return_value = { + self.senlin_mock.resize_cluster.return_value = { 'action': 'fake-action'} self.senlin_mock.get_action.return_value = mock.Mock( status='FAILED', status_reason='Unknown') diff --git a/heat/tests/openstack/senlin/test_node.py b/heat/tests/openstack/senlin/test_node.py index 108d737ea..dc801b330 100644 --- a/heat/tests/openstack/senlin/test_node.py +++ b/heat/tests/openstack/senlin/test_node.py @@ -180,17 +180,17 @@ class SenlinNodeTest(common.HeatTestCase): props['cluster'] = 'new_cluster' rsrc_defns = template.Template(new_t).resource_definitions(self.stack) new_node = rsrc_defns['senlin-node'] - self.senlin_mock.cluster_del_nodes.return_value = { + self.senlin_mock.remove_nodes_from_cluster.return_value = { 'action': 'remove_node_from_cluster' } - self.senlin_mock.cluster_add_nodes.return_value = { + self.senlin_mock.add_nodes_to_cluster.return_value = { 'action': 'add_node_to_cluster' } scheduler.TaskRunner(node.update, new_node)() self.assertEqual((node.UPDATE, node.COMPLETE), node.state) - self.senlin_mock.cluster_del_nodes.assert_called_once_with( + self.senlin_mock.remove_nodes_from_cluster.assert_called_once_with( cluster='fake_cluster_id', nodes=[node.resource_id]) - self.senlin_mock.cluster_add_nodes.assert_called_once_with( + self.senlin_mock.add_nodes_to_cluster.assert_called_once_with( cluster='new_cluster_id', nodes=[node.resource_id]) def test_node_update_failed(self): diff --git a/heat_upgradetests/post_test_hook.sh b/heat_upgradetests/post_test_hook.sh index e69de29bb..a9bf588e2 100755 --- a/heat_upgradetests/post_test_hook.sh +++ b/heat_upgradetests/post_test_hook.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/heat_upgradetests/pre_test_hook.sh b/heat_upgradetests/pre_test_hook.sh index e69de29bb..a9bf588e2 100755 --- a/heat_upgradetests/pre_test_hook.sh +++ b/heat_upgradetests/pre_test_hook.sh @@ -0,0 +1 @@ +#!/bin/bash diff --git a/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml b/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml index faaa3283c..9b6809680 100644 --- a/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml +++ b/releasenotes/notes/support-rbac-824a2d02c8746d3d.yaml @@ -5,7 +5,7 @@ features: for default roles and system scope. This is part of a broader community effort to support read-only roles and implement secure, consistent default policies. - + Refer to `the Keystone documentation`__ for more information on the reason for these changes. diff --git a/requirements.txt b/requirements.txt index 18150cfc4..7794a608b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,11 +2,8 @@ # date but we do not test them so no guarantee of having them all correct. If # you find any incorrect lower bounds, let us know or propose a fix. -# The order of packages is significant, because pip processes them in the order -# of appearance. Changing the order has an impact on the overall integration -# process, which may cause wedges in the gate later. - pbr>=3.1.1 # Apache-2.0 +alembic>=1.8.0 # MIT Babel!=2.4.0,>=2.3.4 # BSD ddt>=1.4.1 # MIT croniter>=0.3.4 # MIT License |