diff options
| author | Stephen Finucane <stephenfin@redhat.com> | 2021-12-23 12:04:25 +0000 |
|---|---|---|
| committer | Stephen Finucane <stephenfin@redhat.com> | 2022-01-05 18:29:43 +0000 |
| commit | 7a146850e0dec437a15b74afebeaec1bab88e8cc (patch) | |
| tree | f81a3a01e767b50a9fd5c4f02ea19651721a5f3e | |
| parent | c80b183aa5eb78b68e41edae4fccf8a8d831ded1 (diff) | |
| download | keystone-7a146850e0dec437a15b74afebeaec1bab88e8cc.tar.gz | |
sql: Squash mitaka migrations
Make the following changes to the new "initial" migration.
- Add 'policy_assocation' table (081)
- Add 'identity_provider', 'federation_protocol', 'mapping',
'service_provider', and 'idp_remote_ids' tables (082)
- Add 'consumer', 'request_token', and 'access_token' tables (083)
- Add 'revocation_event' table (084)
- Add 'project_endpoint', 'endpoint_group' and 'project_endpoint_group'
tables (085)
- Add unique constraint to 'trust' table (086)
- Add 'implied_role' table (087)
- Add 'domain_id' column to 'role' table, and replace unique constraint
on 'name' column of same with unique constraint on 'name' and
'domain_id' column (088)
- Insert root domain entries to 'project' and 'domain' tables (089)
- Add 'local_user' and 'password' tables (090)
- Drop 'ixu_user_name_domain_id' unique constraint, and drop
'domain_id', 'name', and 'password' columns from 'user' table (091)
- Modify foreign key constraints on 'prior_role_id' and
'implied_role_id' columns of 'role' table to CASCADE on delete (092)
- Drop 'fk_project_domain_id' foreign key constraint, and add
foreign key constraint on 'domain_id' column of 'project' table
to 'id' column of same (093)
- Add 'federated_user' table (094)
- Modify 'id' column of 'revocation_event' from String to Integer (095)
Note that 096 and 097 are present since these were fixes for 088 and
091 respectively that have no effect with newer versions of the latter
migrations.
Schemas can be compared using the techniques discussed at [1]. A sample
command for SQLite:
# with change before this
python manage.py version_control \
--database 'sqlite:///before.db' --repository . --version 074
python manage.py upgrade \
--database 'sqlite:///before.db' --repository . --version 097
# with this change
python manage.py version_control \
--database 'sqlite:///after.db' --repository . --version 096
python manage.py upgrade \
--database 'sqlite:///after.db' --repository . --version 097
[1] https://that.guru/blog/comparing-nova-db-migrations/
Change-Id: I41584c652ab34a267009136ee2a2e159ee2f2a6e
Signed-off-by: Stephen Finucane <stephenfin@redhat.com>
24 files changed, 423 insertions, 1961 deletions
diff --git a/keystone/common/sql/migrate_repo/versions/076_placeholder.py b/keystone/common/sql/migrate_repo/versions/076_placeholder.py deleted file mode 100644 index 9f6e84156..000000000 --- a/keystone/common/sql/migrate_repo/versions/076_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Liberty backports. Do not use this number for new -# Mitaka work. New Mitaka work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/077_placeholder.py b/keystone/common/sql/migrate_repo/versions/077_placeholder.py deleted file mode 100644 index 9f6e84156..000000000 --- a/keystone/common/sql/migrate_repo/versions/077_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Liberty backports. Do not use this number for new -# Mitaka work. New Mitaka work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/078_placeholder.py b/keystone/common/sql/migrate_repo/versions/078_placeholder.py deleted file mode 100644 index 9f6e84156..000000000 --- a/keystone/common/sql/migrate_repo/versions/078_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Liberty backports. Do not use this number for new -# Mitaka work. New Mitaka work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/079_placeholder.py b/keystone/common/sql/migrate_repo/versions/079_placeholder.py deleted file mode 100644 index 9f6e84156..000000000 --- a/keystone/common/sql/migrate_repo/versions/079_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Liberty backports. Do not use this number for new -# Mitaka work. New Mitaka work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/080_placeholder.py b/keystone/common/sql/migrate_repo/versions/080_placeholder.py deleted file mode 100644 index 9f6e84156..000000000 --- a/keystone/common/sql/migrate_repo/versions/080_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Liberty backports. Do not use this number for new -# Mitaka work. New Mitaka work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py b/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py deleted file mode 100644 index f8bffa1cd..000000000 --- a/keystone/common/sql/migrate_repo/versions/081_add_endpoint_policy_table.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import upgrades - - -def upgrade(migrate_engine): - try: - extension_version = upgrades.get_db_version( - extension='endpoint_policy', - engine=migrate_engine) - except Exception: - extension_version = 0 - - # This migration corresponds to endpoint_policy extension migration 1. Only - # update if it has not been run. - if extension_version >= 1: - return - - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - endpoint_policy_table = sql.Table( - 'policy_association', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('policy_id', sql.String(64), - nullable=False), - sql.Column('endpoint_id', sql.String(64), - nullable=True), - sql.Column('service_id', sql.String(64), - nullable=True), - sql.Column('region_id', sql.String(64), - nullable=True), - sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), - mysql_engine='InnoDB', - mysql_charset='utf8') - - endpoint_policy_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py b/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py deleted file mode 100644 index bbda554a4..000000000 --- a/keystone/common/sql/migrate_repo/versions/082_add_federation_tables.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import upgrades -import keystone.conf - -CONF = keystone.conf.CONF -_RELAY_STATE_PREFIX = 'relay_state_prefix' - - -def upgrade(migrate_engine): - try: - extension_version = upgrades.get_db_version( - extension='federation', - engine=migrate_engine) - except Exception: - extension_version = 0 - - # This migration corresponds to federation extension migration 8. Only - # update if it has not been run. - if extension_version >= 8: - return - - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - idp_table = sql.Table( - 'identity_provider', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8') - idp_table.create(migrate_engine, checkfirst=True) - - federation_protocol_table = sql.Table( - 'federation_protocol', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('idp_id', sql.String(64), - sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), - primary_key=True), - sql.Column('mapping_id', sql.String(64), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8') - federation_protocol_table.create(migrate_engine, checkfirst=True) - - mapping_table = sql.Table( - 'mapping', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('rules', sql.Text(), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8') - mapping_table.create(migrate_engine, checkfirst=True) - - relay_state_prefix_default = CONF.saml.relay_state_prefix - sp_table = sql.Table( - 'service_provider', - meta, - sql.Column('auth_url', sql.String(256), nullable=False), - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('sp_url', sql.String(256), nullable=False), - sql.Column(_RELAY_STATE_PREFIX, sql.String(256), nullable=False, - server_default=relay_state_prefix_default), - mysql_engine='InnoDB', - mysql_charset='utf8') - sp_table.create(migrate_engine, checkfirst=True) - - idp_table = sql.Table('identity_provider', meta, autoload=True) - remote_id_table = sql.Table( - 'idp_remote_ids', - meta, - sql.Column('idp_id', sql.String(64), - sql.ForeignKey('identity_provider.id', ondelete='CASCADE')), - sql.Column('remote_id', sql.String(255), primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8') - remote_id_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py b/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py deleted file mode 100644 index 8bbccb962..000000000 --- a/keystone/common/sql/migrate_repo/versions/083_add_oauth1_tables.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import upgrades - - -def upgrade(migrate_engine): - try: - extension_version = upgrades.get_db_version( - extension='oauth1', - engine=migrate_engine) - except Exception: - extension_version = 0 - - # This migration corresponds to oauth extension migration 5. Only - # update if it has not been run. - if extension_version >= 5: - return - - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - consumer_table = sql.Table( - 'consumer', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('description', sql.String(64), nullable=True), - sql.Column('secret', sql.String(64), nullable=False), - sql.Column('extra', sql.Text(), nullable=False)) - consumer_table.create(migrate_engine, checkfirst=True) - - request_token_table = sql.Table( - 'request_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('request_secret', sql.String(64), nullable=False), - sql.Column('verifier', sql.String(64), nullable=True), - sql.Column('authorizing_user_id', sql.String(64), nullable=True), - sql.Column('requested_project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=True), - sql.Column('consumer_id', sql.String(64), - sql.ForeignKey('consumer.id'), - nullable=False, index=True), - sql.Column('expires_at', sql.String(64), nullable=True)) - request_token_table.create(migrate_engine, checkfirst=True) - - access_token_table = sql.Table( - 'access_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('access_secret', sql.String(64), nullable=False), - sql.Column('authorizing_user_id', sql.String(64), - nullable=False, index=True), - sql.Column('project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=False), - sql.Column('consumer_id', sql.String(64), - sql.ForeignKey('consumer.id'), - nullable=False, index=True), - sql.Column('expires_at', sql.String(64), nullable=True)) - access_token_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py b/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py deleted file mode 100644 index 0cf37e7d7..000000000 --- a/keystone/common/sql/migrate_repo/versions/084_add_revoke_tables.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright 2014 IBM Corp. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import upgrades - - -def upgrade(migrate_engine): - try: - extension_version = upgrades.get_db_version( - extension='revoke', - engine=migrate_engine) - except Exception: - extension_version = 0 - - # This migration corresponds to revoke extension migration 2. Only - # update if it has not been run. - if extension_version >= 2: - return - - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - service_table = sql.Table( - 'revocation_event', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64)), - sql.Column('project_id', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('role_id', sql.String(64)), - sql.Column('trust_id', sql.String(64)), - sql.Column('consumer_id', sql.String(64)), - sql.Column('access_token_id', sql.String(64)), - sql.Column('issued_before', sql.DateTime(), nullable=False), - sql.Column('expires_at', sql.DateTime()), - sql.Column('revoked_at', sql.DateTime(), index=True, nullable=False), - sql.Column('audit_id', sql.String(32), nullable=True), - sql.Column('audit_chain_id', sql.String(32), nullable=True)) - - service_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py b/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py deleted file mode 100644 index 0283bc335..000000000 --- a/keystone/common/sql/migrate_repo/versions/085_add_endpoint_filtering_table.py +++ /dev/null @@ -1,70 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - -from keystone.common.sql import upgrades - - -def upgrade(migrate_engine): - try: - extension_version = upgrades.get_db_version( - extension='endpoint_filter', - engine=migrate_engine) - except Exception: - extension_version = 0 - - # This migration corresponds to endpoint_filter extension migration 2. Only - # update if it has not been run. - if extension_version >= 2: - return - - # Upgrade operations go here. Don't create your own engine; bind - # migrate_engine to your metadata - meta = sql.MetaData() - meta.bind = migrate_engine - - EP_GROUP_ID = 'endpoint_group_id' - PROJECT_ID = 'project_id' - - endpoint_filtering_table = sql.Table( - 'project_endpoint', - meta, - sql.Column( - 'endpoint_id', - sql.String(64), - primary_key=True, - nullable=False), - sql.Column( - 'project_id', - sql.String(64), - primary_key=True, - nullable=False)) - endpoint_filtering_table.create(migrate_engine, checkfirst=True) - - endpoint_group_table = sql.Table( - 'endpoint_group', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), nullable=False), - sql.Column('description', sql.Text, nullable=True), - sql.Column('filters', sql.Text(), nullable=False)) - endpoint_group_table.create(migrate_engine, checkfirst=True) - - project_endpoint_group_table = sql.Table( - 'project_endpoint_group', - meta, - sql.Column(EP_GROUP_ID, sql.String(64), - sql.ForeignKey('endpoint_group.id'), nullable=False), - sql.Column(PROJECT_ID, sql.String(64), nullable=False), - sql.PrimaryKeyConstraint(EP_GROUP_ID, PROJECT_ID)) - project_endpoint_group_table.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/086_add_duplicate_constraint_trusts.py b/keystone/common/sql/migrate_repo/versions/086_add_duplicate_constraint_trusts.py deleted file mode 100644 index 2b115ea42..000000000 --- a/keystone/common/sql/migrate_repo/versions/086_add_duplicate_constraint_trusts.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2015 Intel Corporation -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate import UniqueConstraint -from sqlalchemy import MetaData, Table - - -def upgrade(migrate_engine): - meta = MetaData(bind=migrate_engine) - trusts = Table('trust', meta, autoload=True) - - UniqueConstraint('trustor_user_id', 'trustee_user_id', 'project_id', - 'impersonation', 'expires_at', table=trusts, - name='duplicate_trust_constraint').create() diff --git a/keystone/common/sql/migrate_repo/versions/087_implied_roles.py b/keystone/common/sql/migrate_repo/versions/087_implied_roles.py deleted file mode 100644 index 7713ce8fa..000000000 --- a/keystone/common/sql/migrate_repo/versions/087_implied_roles.py +++ /dev/null @@ -1,43 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import migrate -import sqlalchemy as sql - - -ROLE_TABLE = 'role' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - implied_role = sql.Table( - 'implied_role', meta, - sql.Column('prior_role_id', sql.String(length=64), primary_key=True), - sql.Column( - 'implied_role_id', sql.String(length=64), primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8') - implied_role.create() - role = sql.Table(ROLE_TABLE, meta, autoload=True) - fkeys = [ - {'columns': [implied_role.c.prior_role_id], - 'references': [role.c.id]}, - {'columns': [implied_role.c.implied_role_id], - 'references': [role.c.id]}, - ] - for fkey in fkeys: - migrate.ForeignKeyConstraint(columns=fkey['columns'], - refcolumns=fkey['references'], - name=fkey.get('name')).create() diff --git a/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py b/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py deleted file mode 100644 index 8b792dfa7..000000000 --- a/keystone/common/sql/migrate_repo/versions/088_domain_specific_roles.py +++ /dev/null @@ -1,60 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import migrate -import sqlalchemy as sql - - -_ROLE_NAME_NEW_CONSTRAINT = 'ixu_role_name_domain_id' -_ROLE_TABLE_NAME = 'role' -_ROLE_NAME_COLUMN_NAME = 'name' -_DOMAIN_ID_COLUMN_NAME = 'domain_id' -_NULL_DOMAIN_ID = '<<null>>' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - role_table = sql.Table(_ROLE_TABLE_NAME, meta, autoload=True) - domain_id = sql.Column(_DOMAIN_ID_COLUMN_NAME, sql.String(64), - nullable=False, server_default=_NULL_DOMAIN_ID) - - # NOTE(morganfainberg): the `role_name` unique constraint is not - # guaranteed to be a fixed name, such as 'ixu_role_name`, so we need to - # search for the correct constraint that only affects role_table.c.name - # and drop that constraint. - to_drop = None - if migrate_engine.name == 'mysql': - for c in role_table.indexes: - if (c.unique and len(c.columns) == 1 and - _ROLE_NAME_COLUMN_NAME in c.columns): - to_drop = c - break - else: - for c in role_table.constraints: - if len(c.columns) == 1 and _ROLE_NAME_COLUMN_NAME in c.columns: - to_drop = c - break - - if to_drop is not None: - migrate.UniqueConstraint(role_table.c.name, - name=to_drop.name).drop() - - # perform changes after constraint is dropped. - if 'domain_id' not in role_table.columns: - # Only create the column if it doesn't already exist. - role_table.create_column(domain_id) - - migrate.UniqueConstraint(role_table.c.name, - role_table.c.domain_id, - name=_ROLE_NAME_NEW_CONSTRAINT).create() diff --git a/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py b/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py deleted file mode 100644 index 477c719a6..000000000 --- a/keystone/common/sql/migrate_repo/versions/089_add_root_of_all_domains.py +++ /dev/null @@ -1,76 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -_PROJECT_TABLE_NAME = 'project' -_DOMAIN_TABLE_NAME = 'domain' -NULL_DOMAIN_ID = '<<keystone.domain.root>>' - - -def upgrade(migrate_engine): - - def _generate_root_domain_project(): - # Generate a project that will act as a root for all domains, in order - # for use to be able to use a FK constraint on domain_id. Projects - # acting as a domain will not reference this as their parent_id, just - # as domain_id. - # - # This special project is filtered out by the driver, so is never - # visible to the manager or API. - - project_ref = { - 'id': NULL_DOMAIN_ID, - 'name': NULL_DOMAIN_ID, - 'enabled': False, - 'description': '', - 'domain_id': NULL_DOMAIN_ID, - 'is_domain': True, - 'parent_id': None, - 'extra': '{}' - } - return project_ref - - def _generate_root_domain(): - # Generate a similar root for the domain table, this is an interim - # step so as to allow continuation of current project domain_id FK. - # - # This special domain is filtered out by the driver, so is never - # visible to the manager or API. - - domain_ref = { - 'id': NULL_DOMAIN_ID, - 'name': NULL_DOMAIN_ID, - 'enabled': False, - 'extra': '{}' - } - return domain_ref - - meta = sql.MetaData() - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) - domain_table = sql.Table(_DOMAIN_TABLE_NAME, meta, autoload=True) - - root_domain = _generate_root_domain() - new_entry = domain_table.insert().values(**root_domain) - session.execute(new_entry) - session.commit() - - root_domain_project = _generate_root_domain_project() - new_entry = project_table.insert().values(**root_domain_project) - session.execute(new_entry) - session.commit() - - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py b/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py deleted file mode 100644 index 800ba47e8..000000000 --- a/keystone/common/sql/migrate_repo/versions/090_add_local_user_and_password_tables.py +++ /dev/null @@ -1,42 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - user = sql.Table('user', meta, autoload=True) - - local_user = sql.Table( - 'local_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column('user_id', sql.String(64), - sql.ForeignKey(user.c.id, ondelete='CASCADE'), - nullable=False, unique=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('name', sql.String(255), nullable=False), - sql.UniqueConstraint('domain_id', 'name')) - local_user.create(migrate_engine, checkfirst=True) - - password = sql.Table( - 'password', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column('local_user_id', sql.Integer, - sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), - nullable=False), - sql.Column('password', sql.String(128), nullable=False)) - password.create(migrate_engine, checkfirst=True) diff --git a/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py b/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py deleted file mode 100644 index 7e98b99b7..000000000 --- a/keystone/common/sql/migrate_repo/versions/091_migrate_data_to_local_user_and_password_tables.py +++ /dev/null @@ -1,84 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import migrate -import sqlalchemy as sql -from sqlalchemy import func - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - user_table = sql.Table('user', meta, autoload=True) - local_user_table = sql.Table('local_user', meta, autoload=True) - password_table = sql.Table('password', meta, autoload=True) - - # migrate data to local_user table - local_user_values = [] - for row in user_table.select().execute(): - # skip the row that already exists in `local_user`, this could - # happen if run into a partially-migrated table due to the - # bug #1549705. - filter_by = local_user_table.c.user_id == row['id'] - user_count = sql.select([func.count()]).select_from( - local_user_table).where(filter_by).execute().fetchone()[0] - if user_count == 0: - local_user_values.append({'user_id': row['id'], - 'domain_id': row['domain_id'], - 'name': row['name']}) - if local_user_values: - local_user_table.insert().values(local_user_values).execute() - - # migrate data to password table - sel = ( - sql.select([user_table, local_user_table], use_labels=True) - .select_from(user_table.join(local_user_table, user_table.c.id == - local_user_table.c.user_id)) - ) - user_rows = sel.execute() - password_values = [] - for row in user_rows: - if row['user_password']: - password_values.append({'local_user_id': row['local_user_id'], - 'password': row['user_password']}) - if password_values: - password_table.insert().values(password_values).execute() - - # NOTE(gnuoy): the `domain_id` unique constraint is not guaranteed to - # be a fixed name, such as 'ixu_user_name_domain_id`, so we need to - # search for the correct constraint that only affects - # user_table.c.domain_id and drop that constraint. (Fix based on - # morganfainbergs fix in 088_domain_specific_roles.py) - to_drop = None - if migrate_engine.name == 'mysql': - for index in user_table.indexes: - if (index.unique and len(index.columns) == 2 and - 'domain_id' in index.columns and 'name' in index.columns): - to_drop = index - break - else: - for index in user_table.constraints: - if (len(index.columns) == 2 and 'domain_id' in index.columns and - 'name' in index.columns): - to_drop = index - break - # remove domain_id and name unique constraint - if migrate_engine.name != 'sqlite' and to_drop is not None: - migrate.UniqueConstraint(user_table.c.domain_id, - user_table.c.name, - name=to_drop.name).drop() - - # drop user columns - user_table.c.domain_id.drop() - user_table.c.name.drop() - user_table.c.password.drop() diff --git a/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py b/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py deleted file mode 100644 index 5e8418999..000000000 --- a/keystone/common/sql/migrate_repo/versions/092_make_implied_roles_fks_cascaded.py +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import migrate -import sqlalchemy as sql - - -ROLE_TABLE = 'role' -IMPLIED_ROLE_TABLE = 'implied_role' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - role = sql.Table(ROLE_TABLE, meta, autoload=True) - implied_role = sql.Table(IMPLIED_ROLE_TABLE, meta, autoload=True) - - fkeys = [ - {'columns': [implied_role.c.prior_role_id], - 'references': [role.c.id]}, - {'columns': [implied_role.c.implied_role_id], - 'references': [role.c.id]}, - ] - - # NOTE(stevemar): We need to divide these into two separate loops otherwise - # they may clobber each other and only end up with one foreign key. - for fkey in fkeys: - migrate.ForeignKeyConstraint(columns=fkey['columns'], - refcolumns=fkey['references'], - name=fkey.get('name')).drop() - for fkey in fkeys: - migrate.ForeignKeyConstraint(columns=fkey['columns'], - refcolumns=fkey['references'], - name=fkey.get('name'), - ondelete="CASCADE").create() diff --git a/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py b/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py deleted file mode 100644 index e0d6a4ee5..000000000 --- a/keystone/common/sql/migrate_repo/versions/093_migrate_domains_to_projects.py +++ /dev/null @@ -1,125 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json - -import sqlalchemy as sql - -from keystone.common.sql import upgrades - - -_PROJECT_TABLE_NAME = 'project' -_DOMAIN_TABLE_NAME = 'domain' -_PARENT_ID_COLUMN_NAME = 'parent_id' -_DOMAIN_ID_COLUMN_NAME = 'domain_id' - -# Above the driver level, the domain_id of a project acting as a domain is -# None. However, in order to enable sql integrity constraints to still operate -# on this column, we create a special "root of all domains" row, with an ID of -# NULL_DOMAIN_ID, which all projects acting as a domain reference in their -# domain_id attribute. This special row, as well as NULL_DOMAIN_ID, are never -# exposed outside of sql driver layer. -NULL_DOMAIN_ID = '<<keystone.domain.root>>' - - -def list_existing_project_constraints(project_table, domain_table): - constraints = [{'table': project_table, - 'fk_column': _PARENT_ID_COLUMN_NAME, - 'ref_column': project_table.c.id}, - {'table': project_table, - 'fk_column': _DOMAIN_ID_COLUMN_NAME, - 'ref_column': domain_table.c.id}] - - return constraints - - -def list_new_project_constraints(project_table): - constraints = [{'table': project_table, - 'fk_column': _PARENT_ID_COLUMN_NAME, - 'ref_column': project_table.c.id}, - {'table': project_table, - 'fk_column': _DOMAIN_ID_COLUMN_NAME, - 'ref_column': project_table.c.id}] - - return constraints - - -def upgrade(migrate_engine): - - def _project_from_domain(domain): - # Creates a project dict with is_domain=True from the provided - # domain. - - description = None - extra = {} - if domain.extra is not None: - # 'description' property is an extra attribute in domains but a - # first class attribute in projects - extra = json.loads(domain.extra) - description = extra.pop('description', None) - - return { - 'id': domain.id, - 'name': domain.name, - 'enabled': domain.enabled, - 'description': description, - 'domain_id': NULL_DOMAIN_ID, - 'is_domain': True, - 'parent_id': None, - 'extra': json.dumps(extra) - } - - meta = sql.MetaData() - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - project_table = sql.Table(_PROJECT_TABLE_NAME, meta, autoload=True) - domain_table = sql.Table(_DOMAIN_TABLE_NAME, meta, autoload=True) - - # NOTE(htruta): Remove the parent_id constraint during the migration - # because for every root project inside this domain, we will set - # the project domain_id to be its parent_id. We re-enable the constraint - # in the end of this method. We also remove the domain_id constraint, - # while be recreated a FK to the project_id at the end. - upgrades.remove_constraints( - list_existing_project_constraints(project_table, domain_table)) - - # For each domain, create a project acting as a domain. We ignore the - # "root of all domains" row, since we already have one of these in the - # project table. - domains = list(domain_table.select().execute()) - for domain in domains: - if domain.id == NULL_DOMAIN_ID: - continue - is_domain_project = _project_from_domain(domain) - new_entry = project_table.insert().values(**is_domain_project) - session.execute(new_entry) - session.commit() - - # For each project, that has no parent (i.e. a top level project), update - # it's parent_id to point at the project acting as its domain. We ignore - # the "root of all domains" row, since its parent_id must always be None. - projects = list(project_table.select().execute()) - for project in projects: - if (project.parent_id is not None or project.is_domain or - project.id == NULL_DOMAIN_ID): - continue - values = {'parent_id': project.domain_id} - update = project_table.update().where( - project_table.c.id == project.id).values(values) - session.execute(update) - session.commit() - - upgrades.add_constraints( - list_new_project_constraints(project_table)) - - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py b/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py deleted file mode 100644 index 8bc64ec1f..000000000 --- a/keystone/common/sql/migrate_repo/versions/094_add_federated_user_table.py +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import migrate -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - user_table = sql.Table('user', meta, autoload=True) - idp_table = sql.Table('identity_provider', meta, autoload=True) - protocol_table = sql.Table('federation_protocol', meta, autoload=True) - - federated_table = sql.Table( - 'federated_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column('user_id', sql.String(64), - sql.ForeignKey(user_table.c.id, ondelete='CASCADE'), - nullable=False), - sql.Column('idp_id', sql.String(64), - sql.ForeignKey(idp_table.c.id, ondelete='CASCADE'), - nullable=False), - sql.Column('protocol_id', sql.String(64), nullable=False), - sql.Column('unique_id', sql.String(255), nullable=False), - sql.Column('display_name', sql.String(255), nullable=True), - sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'), - mysql_engine='InnoDB', - mysql_charset='utf8') - federated_table.create(migrate_engine, checkfirst=True) - - migrate.ForeignKeyConstraint( - columns=[federated_table.c.protocol_id, federated_table.c.idp_id], - refcolumns=[protocol_table.c.id, protocol_table.c.idp_id]).create() diff --git a/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py b/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py deleted file mode 100644 index 7a75f7b19..000000000 --- a/keystone/common/sql/migrate_repo/versions/095_add_integer_pkey_to_revocation_event_table.py +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - # You can specify primary keys when creating tables, however adding - # auto-increment integer primary keys for existing tables is not - # cross-engine compatibility supported. Thus, the approach is to: - # (1) create a new revocation_event table with an int pkey, - # (2) migrate data from the old table to the new table, - # (3) delete the old revocation_event table - # (4) rename the new revocation_event table - revocation_table = sql.Table('revocation_event', meta, autoload=True) - - revocation_table_new = sql.Table( - 'revocation_event_new', - meta, - sql.Column('id', sql.Integer, primary_key=True), - sql.Column('domain_id', sql.String(64)), - sql.Column('project_id', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('role_id', sql.String(64)), - sql.Column('trust_id', sql.String(64)), - sql.Column('consumer_id', sql.String(64)), - sql.Column('access_token_id', sql.String(64)), - sql.Column('issued_before', sql.DateTime(), nullable=False), - sql.Column('expires_at', sql.DateTime()), - sql.Column('revoked_at', sql.DateTime(), index=True, nullable=False), - sql.Column('audit_id', sql.String(32), nullable=True), - sql.Column('audit_chain_id', sql.String(32), nullable=True)) - revocation_table_new.create(migrate_engine, checkfirst=True) - - revocation_table_new.insert().from_select(['domain_id', - 'project_id', - 'user_id', - 'role_id', - 'trust_id', - 'consumer_id', - 'access_token_id', - 'issued_before', - 'expires_at', - 'revoked_at', - 'audit_id', - 'audit_chain_id'], - revocation_table.select()) - - revocation_table.drop() - revocation_table_new.rename('revocation_event') diff --git a/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py b/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py deleted file mode 100644 index 0156de217..000000000 --- a/keystone/common/sql/migrate_repo/versions/096_drop_role_name_constraint.py +++ /dev/null @@ -1,50 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import migrate -import sqlalchemy as sql - -_ROLE_TABLE_NAME = 'role' -_ROLE_NAME_COLUMN_NAME = 'name' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - role_table = sql.Table(_ROLE_TABLE_NAME, meta, autoload=True) - - # NOTE(morganfainberg): the `role_name` unique constraint is not - # guaranteed to be named 'ixu_role_name', so we need to search for the - # correct constraint that only affects role_table.c.name and drop - # that constraint. - # - # This is an idempotent change that reflects the fix to migration - # 88 if the role_name unique constraint was not named consistently and - # someone manually fixed the migrations / db without dropping the - # old constraint. - to_drop = None - if migrate_engine.name == 'mysql': - for c in role_table.indexes: - if (c.unique and len(c.columns) == 1 and - _ROLE_NAME_COLUMN_NAME in c.columns): - to_drop = c - break - else: - for c in role_table.constraints: - if len(c.columns) == 1 and _ROLE_NAME_COLUMN_NAME in c.columns: - to_drop = c - break - - if to_drop is not None: - migrate.UniqueConstraint(role_table.c.name, - name=to_drop.name).drop() diff --git a/keystone/common/sql/migrate_repo/versions/097_drop_user_name_domainid_constraint.py b/keystone/common/sql/migrate_repo/versions/097_drop_user_name_domainid_constraint.py deleted file mode 100644 index d99d6aa6a..000000000 --- a/keystone/common/sql/migrate_repo/versions/097_drop_user_name_domainid_constraint.py +++ /dev/null @@ -1,67 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import migrate -import sqlalchemy as sql - -_USER_TABLE_NAME = 'user' -_USER_NAME_COLUMN_NAME = 'name' -_USER_DOMAINID_COLUMN_NAME = 'domain_id' -_USER_PASSWORD_COLUMN_NAME = 'password' # nosec - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - user_table = sql.Table(_USER_TABLE_NAME, meta, autoload=True) - - # NOTE(gnuoy): the `domain_id` unique constraint is not guaranteed to - # be a fixed name, such as 'ixu_user_name_domain_id`, so we need to - # search for the correct constraint that only affects - # user_table.c.domain_id and drop that constraint. (Fix based on - # morganfainbergs fix in 088_domain_specific_roles.py) - # - # This is an idempotent change that reflects the fix to migration - # 91 if the user name & domain_id unique constraint was not named - # consistently and someone manually fixed the migrations / db - # without dropping the old constraint. - to_drop = None - if migrate_engine.name == 'mysql': - for index in user_table.indexes: - if (index.unique and len(index.columns) == 2 and - _USER_DOMAINID_COLUMN_NAME in index.columns and - _USER_NAME_COLUMN_NAME in index.columns): - to_drop = index - break - else: - for index in user_table.constraints: - if (len(index.columns) == 2 and - _USER_DOMAINID_COLUMN_NAME in index.columns and - _USER_NAME_COLUMN_NAME in index.columns): - to_drop = index - break - - # remove domain_id and name unique constraint - if to_drop is not None: - migrate.UniqueConstraint(user_table.c.domain_id, - user_table.c.name, - name=to_drop.name).drop() - - # If migration 91 was aborted due to Bug #1572341 then columns may not - # have been dropped. - if _USER_DOMAINID_COLUMN_NAME in user_table.c: - user_table.c.domain_id.drop() - if _USER_NAME_COLUMN_NAME in user_table.c: - user_table.c.name.drop() - if _USER_PASSWORD_COLUMN_NAME in user_table.c: - user_table.c.password.drop() diff --git a/keystone/common/sql/migrate_repo/versions/075_liberty.py b/keystone/common/sql/migrate_repo/versions/097_mitaka.py index 8673b7f73..72c561d1f 100644 --- a/keystone/common/sql/migrate_repo/versions/075_liberty.py +++ b/keystone/common/sql/migrate_repo/versions/097_mitaka.py @@ -17,11 +17,14 @@ import sqlalchemy as sql from keystone.assignment.backends import sql as assignment_sql from keystone.common import sql as ks_sql +import keystone.conf from keystone.identity.mapping_backends import mapping as mapping_backend - +CONF = keystone.conf.CONF LOG = log.getLogger(__name__) +NULL_DOMAIN_ID = '<<keystone.domain.root>>' + def upgrade(migrate_engine): meta = sql.MetaData() @@ -38,6 +41,35 @@ def upgrade(migrate_engine): % migrate_engine.url.database ) + access_token = sql.Table( + 'access_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('access_secret', sql.String(64), nullable=False), + sql.Column( + 'authorizing_user_id', sql.String(64), nullable=False, index=True + ), + sql.Column('project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=False), + sql.Column( + 'consumer_id', + sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, + index=True, + ), + sql.Column('expires_at', sql.String(64), nullable=True), + ) + + consumer = sql.Table( + 'consumer', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('description', sql.String(64), nullable=True), + sql.Column('secret', sql.String(64), nullable=False), + sql.Column('extra', sql.Text(), nullable=False), + ) + credential = sql.Table( 'credential', meta, @@ -83,6 +115,54 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + endpoint_group = sql.Table( + 'endpoint_group', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('name', sql.String(255), nullable=False), + sql.Column('description', sql.Text, nullable=True), + sql.Column('filters', sql.Text(), nullable=False), + ) + + federated_user = sql.Table( + 'federated_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'user_id', + sql.String(64), + sql.ForeignKey('user.id', ondelete='CASCADE'), + nullable=False, + ), + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + nullable=False, + ), + sql.Column('protocol_id', sql.String(64), nullable=False), + sql.Column('unique_id', sql.String(255), nullable=False), + sql.Column('display_name', sql.String(255), nullable=True), + sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + federation_protocol = sql.Table( + 'federation_protocol', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + primary_key=True, + ), + sql.Column('mapping_id', sql.String(64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + group = sql.Table( 'group', meta, @@ -95,6 +175,76 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + identity_provider = sql.Table( + 'identity_provider', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + idp_remote_ids = sql.Table( + 'idp_remote_ids', + meta, + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + ), + sql.Column('remote_id', sql.String(255), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + implied_role = sql.Table( + 'implied_role', + meta, + sql.Column('prior_role_id', sql.String(length=64), primary_key=True), + sql.Column('implied_role_id', sql.String(length=64), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + local_user = sql.Table( + 'local_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'user_id', + sql.String(64), + sql.ForeignKey('user.id', ondelete='CASCADE'), + nullable=False, + unique=True, + ), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('name', sql.String(255), nullable=False), + sql.UniqueConstraint('domain_id', 'name'), + ) + + mapping = sql.Table( + 'mapping', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('rules', sql.Text(), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + password = sql.Table( + 'password', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'local_user_id', + sql.Integer, + sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), + nullable=False, + ), + sql.Column('password', sql.String(128), nullable=False), + ) + policy = sql.Table( 'policy', meta, @@ -106,6 +256,19 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + policy_association = sql.Table( + 'policy_association', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('policy_id', sql.String(64), nullable=False), + sql.Column('endpoint_id', sql.String(64), nullable=True), + sql.Column('service_id', sql.String(64), nullable=True), + sql.Column('region_id', sql.String(64), nullable=True), + sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + project = sql.Table( 'project', meta, @@ -127,6 +290,30 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + project_endpoint = sql.Table( + 'project_endpoint', + meta, + sql.Column( + 'endpoint_id', sql.String(64), primary_key=True, nullable=False + ), + sql.Column( + 'project_id', sql.String(64), primary_key=True, nullable=False + ), + ) + + project_endpoint_group = sql.Table( + 'project_endpoint_group', + meta, + sql.Column( + 'endpoint_group_id', + sql.String(64), + sql.ForeignKey('endpoint_group.id'), + nullable=False, + ), + sql.Column('project_id', sql.String(64), nullable=False), + sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'), + ) + config_register = sql.Table( 'config_register', meta, @@ -136,12 +323,66 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + request_token = sql.Table( + 'request_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('request_secret', sql.String(64), nullable=False), + sql.Column('verifier', sql.String(64), nullable=True), + sql.Column('authorizing_user_id', sql.String(64), nullable=True), + sql.Column('requested_project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=True), + sql.Column( + 'consumer_id', + sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, + index=True, + ), + sql.Column('expires_at', sql.String(64), nullable=True), + ) + + revocation_event = sql.Table( + 'revocation_event', + meta, + sql.Column('id', sql.Integer, primary_key=True), + sql.Column('domain_id', sql.String(64)), + sql.Column('project_id', sql.String(64)), + sql.Column('user_id', sql.String(64)), + sql.Column('role_id', sql.String(64)), + sql.Column('trust_id', sql.String(64)), + sql.Column('consumer_id', sql.String(64)), + sql.Column('access_token_id', sql.String(64)), + sql.Column('issued_before', sql.DateTime(), nullable=False), + sql.Column('expires_at', sql.DateTime()), + sql.Column('revoked_at', sql.DateTime(), nullable=False), + sql.Column('audit_id', sql.String(32), nullable=True), + sql.Column('audit_chain_id', sql.String(32), nullable=True), + # NOTE(stephenfin): The '_new' suffix here is due to migration 095, + # which changed the 'id' column from String(64) to Integer. It did this + # by creating a 'revocation_event_new' table and populating it with + # data from the 'revocation_event' table before deleting the + # 'revocation_event' table and renaming the 'revocation_event_new' + # table to 'revocation_event'. Because the 'revoked_at' column had + # 'index=True', sqlalchemy automatically generated the index name as + # 'ix_{table}_{column}'. However, when intitially created, '{table}' + # was 'revocation_event_new' so the index got that name. We may wish to + # rename this eventually. + sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'), + ) + role = sql.Table( 'role', meta, sql.Column('id', sql.String(length=64), primary_key=True), sql.Column('name', sql.String(length=255), nullable=False), sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column( + 'domain_id', + sql.String(64), + nullable=False, + server_default='<<null>>', + ), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -163,6 +404,24 @@ def upgrade(migrate_engine): mysql_charset='utf8', ) + service_provider = sql.Table( + 'service_provider', + meta, + sql.Column('auth_url', sql.String(256), nullable=False), + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + sql.Column('sp_url', sql.String(256), nullable=False), + sql.Column( + 'relay_state_prefix', + sql.String(256), + nullable=False, + server_default=CONF.saml.relay_state_prefix, + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + token = sql.Table( 'token', meta, @@ -188,6 +447,14 @@ def upgrade(migrate_engine): sql.Column('expires_at', sql.DateTime), sql.Column('remaining_uses', sql.Integer, nullable=True), sql.Column('extra', ks_sql.JsonBlob.impl), + sql.UniqueConstraint( + 'trustor_user_id', + 'trustee_user_id', + 'project_id', + 'impersonation', + 'expires_at', + name='duplicate_trust_constraint', + ), mysql_engine='InnoDB', mysql_charset='utf8', ) @@ -209,11 +476,8 @@ def upgrade(migrate_engine): 'user', meta, sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('name', sql.String(length=255), nullable=False), sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('password', sql.String(length=128)), sql.Column('enabled', sql.Boolean), - sql.Column('domain_id', sql.String(length=64), nullable=False), sql.Column('default_project_id', sql.String(length=64)), mysql_engine='InnoDB', mysql_charset='utf8', @@ -330,6 +594,23 @@ def upgrade(migrate_engine): whitelisted_config, sensitive_config, config_register, + policy_association, + identity_provider, + federation_protocol, + mapping, + service_provider, + idp_remote_ids, + consumer, + request_token, + access_token, + revocation_event, + project_endpoint, + endpoint_group, + project_endpoint_group, + implied_role, + local_user, + password, + federated_user, ] for table in tables: @@ -341,18 +622,14 @@ def upgrade(migrate_engine): # Unique Constraints migrate.UniqueConstraint( - user.c.domain_id, - user.c.name, - name='ixu_user_name_domain_id', - ).create() - migrate.UniqueConstraint( group.c.domain_id, group.c.name, name='ixu_group_name_domain_id', ).create() migrate.UniqueConstraint( role.c.name, - name='ixu_role_name', + role.c.domain_id, + name='ixu_role_name_domain_id', ).create() migrate.UniqueConstraint( project.c.domain_id, @@ -402,8 +679,7 @@ def upgrade(migrate_engine): }, { 'columns': [project.c.domain_id], - 'references': [domain.c.id], - 'name': 'fk_project_domain_id', + 'references': [project.c.id], }, { 'columns': [endpoint.c.region_id], @@ -415,6 +691,26 @@ def upgrade(migrate_engine): 'references': [project.c.id], 'name': 'project_parent_id_fkey', }, + { + 'columns': [implied_role.c.prior_role_id], + 'references': [role.c.id], + 'ondelete': 'CASCADE', + }, + { + 'columns': [implied_role.c.implied_role_id], + 'references': [role.c.id], + 'ondelete': 'CASCADE', + }, + { + 'columns': [ + federated_user.c.protocol_id, + federated_user.c.idp_id, + ], + 'references': [ + federation_protocol.c.id, + federation_protocol.c.idp_id, + ], + }, ] if migrate_engine.name == 'sqlite': @@ -433,4 +729,59 @@ def upgrade(migrate_engine): columns=fkey['columns'], refcolumns=fkey['references'], name=fkey.get('name'), + ondelete=fkey.get('ondelete'), ).create() + + # data generation + + def _generate_root_domain_project(): + # Generate a project that will act as a root for all domains, in order + # for use to be able to use a FK constraint on domain_id. Projects + # acting as a domain will not reference this as their parent_id, just + # as domain_id. + # + # This special project is filtered out by the driver, so is never + # visible to the manager or API. + + project_ref = { + 'id': NULL_DOMAIN_ID, + 'name': NULL_DOMAIN_ID, + 'enabled': False, + 'description': '', + 'domain_id': NULL_DOMAIN_ID, + 'is_domain': True, + 'parent_id': None, + 'extra': '{}', + } + return project_ref + + def _generate_root_domain(): + # Generate a similar root for the domain table, this is an interim + # step so as to allow continuation of current project domain_id FK. + # + # This special domain is filtered out by the driver, so is never + # visible to the manager or API. + + domain_ref = { + 'id': NULL_DOMAIN_ID, + 'name': NULL_DOMAIN_ID, + 'enabled': False, + 'extra': '{}', + } + return domain_ref + + meta = sql.MetaData() + meta.bind = migrate_engine + session = sql.orm.sessionmaker(bind=migrate_engine)() + + root_domain = _generate_root_domain() + new_entry = domain.insert().values(**root_domain) + session.execute(new_entry) + session.commit() + + root_domain_project = _generate_root_domain_project() + new_entry = project.insert().values(**root_domain_project) + session.execute(new_entry) + session.commit() + + session.close() diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index 296639b2a..6a63d16b7 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -66,7 +66,6 @@ from keystone.common.sql import upgrades from keystone.credential.providers import fernet as credential_fernet from keystone.resource.backends import base as resource_base from keystone.tests import unit -from keystone.tests.unit import default_fixtures from keystone.tests.unit import ksfixtures from keystone.tests.unit.ksfixtures import database @@ -99,7 +98,7 @@ INITIAL_TABLE_STRUCTURE = { 'parent_id', 'is_domain', ], 'role': [ - 'id', 'name', 'extra', + 'id', 'name', 'extra', 'domain_id', ], 'service': [ 'id', 'type', 'extra', 'enabled', @@ -115,8 +114,7 @@ INITIAL_TABLE_STRUCTURE = { 'trust_id', 'role_id', ], 'user': [ - 'id', 'name', 'extra', 'password', 'enabled', 'domain_id', - 'default_project_id', + 'id', 'extra', 'enabled', 'default_project_id', ], 'user_group_membership': [ 'user_id', 'group_id', @@ -136,6 +134,62 @@ INITIAL_TABLE_STRUCTURE = { 'sensitive_config': [ 'domain_id', 'group', 'option', 'value', ], + 'policy_association': [ + 'id', 'policy_id', 'endpoint_id', 'service_id', 'region_id', + ], + 'identity_provider': [ + 'id', 'enabled', 'description', + ], + 'federation_protocol': [ + 'id', 'idp_id', 'mapping_id', + ], + 'mapping': [ + 'id', 'rules', + ], + 'service_provider': [ + 'auth_url', 'id', 'enabled', 'description', 'sp_url', + 'relay_state_prefix', + ], + 'idp_remote_ids': [ + 'idp_id', 'remote_id', + ], + 'consumer': [ + 'id', 'description', 'secret', 'extra', + ], + 'request_token': [ + 'id', 'request_secret', 'verifier', 'authorizing_user_id', + 'requested_project_id', 'role_ids', 'consumer_id', 'expires_at', + ], + 'access_token': [ + 'id', 'access_secret', 'authorizing_user_id', 'project_id', 'role_ids', + 'consumer_id', 'expires_at', + ], + 'revocation_event': [ + 'id', 'domain_id', 'project_id', 'user_id', 'role_id', 'trust_id', + 'consumer_id', 'access_token_id', 'issued_before', 'expires_at', + 'revoked_at', 'audit_id', 'audit_chain_id', + ], + 'project_endpoint': [ + 'endpoint_id', 'project_id' + ], + 'endpoint_group': [ + 'id', 'name', 'description', 'filters', + ], + 'project_endpoint_group': [ + 'endpoint_group_id', 'project_id', + ], + 'implied_role': [ + 'prior_role_id', 'implied_role_id', + ], + 'local_user': [ + 'id', 'user_id', 'domain_id', 'name', + ], + 'password': [ + 'id', 'local_user_id', 'password', + ], + 'federated_user': [ + 'id', 'user_id', 'idp_id', 'protocol_id', 'unique_id', 'display_name', + ], } LEGACY_REPO = 'migrate_repo' @@ -348,6 +402,7 @@ class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin, class SqlLegacyRepoUpgradeTests(SqlMigrateBase): + def test_blank_db_to_start(self): self.assertTableDoesNotExist('user') @@ -367,591 +422,12 @@ class SqlLegacyRepoUpgradeTests(SqlMigrateBase): for table in INITIAL_TABLE_STRUCTURE: self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table]) - def test_endpoint_policy_upgrade(self): - self.assertTableDoesNotExist('policy_association') - self.upgrade(81) - self.assertTableColumns('policy_association', - ['id', 'policy_id', 'endpoint_id', - 'service_id', 'region_id']) - - @mock.patch.object(upgrades, 'get_db_version', return_value=1) - def test_endpoint_policy_already_migrated(self, mock_ep): - - # By setting the return value to 1, the migration has already been - # run, and there's no need to create the table again - - self.upgrade(81) - - mock_ep.assert_called_once_with(extension='endpoint_policy', - engine=mock.ANY) - - # It won't exist because we are mocking it, but we can verify - # that 081 did not create the table - self.assertTableDoesNotExist('policy_association') - - def test_create_federation_tables(self): - self.identity_provider = 'identity_provider' - self.federation_protocol = 'federation_protocol' - self.service_provider = 'service_provider' - self.mapping = 'mapping' - self.remote_ids = 'idp_remote_ids' - - self.assertTableDoesNotExist(self.identity_provider) - self.assertTableDoesNotExist(self.federation_protocol) - self.assertTableDoesNotExist(self.service_provider) - self.assertTableDoesNotExist(self.mapping) - self.assertTableDoesNotExist(self.remote_ids) - - self.upgrade(82) - self.assertTableColumns(self.identity_provider, - ['id', 'description', 'enabled']) - - self.assertTableColumns(self.federation_protocol, - ['id', 'idp_id', 'mapping_id']) - - self.assertTableColumns(self.mapping, - ['id', 'rules']) - - self.assertTableColumns(self.service_provider, - ['id', 'description', 'enabled', 'auth_url', - 'relay_state_prefix', 'sp_url']) - - self.assertTableColumns(self.remote_ids, ['idp_id', 'remote_id']) - - federation_protocol = sqlalchemy.Table(self.federation_protocol, - self.metadata, - autoload=True) - self.assertFalse(federation_protocol.c.mapping_id.nullable) - - sp_table = sqlalchemy.Table(self.service_provider, - self.metadata, - autoload=True) - self.assertFalse(sp_table.c.auth_url.nullable) - self.assertFalse(sp_table.c.sp_url.nullable) - - @mock.patch.object(upgrades, 'get_db_version', return_value=8) - def test_federation_already_migrated(self, mock_federation): - - # By setting the return value to 8, the migration has already been - # run, and there's no need to create the table again. - self.upgrade(82) - - mock_federation.assert_any_call(extension='federation', - engine=mock.ANY) - - # It won't exist because we are mocking it, but we can verify - # that 082 did not create the table. - self.assertTableDoesNotExist('identity_provider') - self.assertTableDoesNotExist('federation_protocol') - self.assertTableDoesNotExist('mapping') - self.assertTableDoesNotExist('service_provider') - self.assertTableDoesNotExist('idp_remote_ids') - - def test_create_oauth_tables(self): - consumer = 'consumer' - request_token = 'request_token' - access_token = 'access_token' - self.assertTableDoesNotExist(consumer) - self.assertTableDoesNotExist(request_token) - self.assertTableDoesNotExist(access_token) - self.upgrade(83) - self.assertTableColumns(consumer, - ['id', - 'description', - 'secret', - 'extra']) - self.assertTableColumns(request_token, - ['id', - 'request_secret', - 'verifier', - 'authorizing_user_id', - 'requested_project_id', - 'role_ids', - 'consumer_id', - 'expires_at']) - self.assertTableColumns(access_token, - ['id', - 'access_secret', - 'authorizing_user_id', - 'project_id', - 'role_ids', - 'consumer_id', - 'expires_at']) - - @mock.patch.object(upgrades, 'get_db_version', return_value=5) - def test_oauth1_already_migrated(self, mock_oauth1): - - # By setting the return value to 5, the migration has already been - # run, and there's no need to create the table again. - self.upgrade(83) - - mock_oauth1.assert_any_call(extension='oauth1', engine=mock.ANY) - - # It won't exist because we are mocking it, but we can verify - # that 083 did not create the table. - self.assertTableDoesNotExist('consumer') - self.assertTableDoesNotExist('request_token') - self.assertTableDoesNotExist('access_token') - - def test_create_revoke_table(self): - self.assertTableDoesNotExist('revocation_event') - self.upgrade(84) - self.assertTableColumns('revocation_event', - ['id', 'domain_id', 'project_id', 'user_id', - 'role_id', 'trust_id', 'consumer_id', - 'access_token_id', 'issued_before', - 'expires_at', 'revoked_at', - 'audit_chain_id', 'audit_id']) - - @mock.patch.object(upgrades, 'get_db_version', return_value=2) - def test_revoke_already_migrated(self, mock_revoke): - - # By setting the return value to 2, the migration has already been - # run, and there's no need to create the table again. - self.upgrade(84) - - mock_revoke.assert_any_call(extension='revoke', engine=mock.ANY) - - # It won't exist because we are mocking it, but we can verify - # that 084 did not create the table. - self.assertTableDoesNotExist('revocation_event') - - def test_implied_roles_upgrade(self): - self.upgrade(87) - self.assertTableColumns('implied_role', - ['prior_role_id', 'implied_role_id']) - self.assertTrue(self.does_fk_exist('implied_role', 'prior_role_id')) - self.assertTrue(self.does_fk_exist('implied_role', 'implied_role_id')) - - def test_endpoint_filter_upgrade(self): - def assert_tables_columns_exist(): - self.assertTableColumns('project_endpoint', - ['endpoint_id', 'project_id']) - self.assertTableColumns('endpoint_group', - ['id', 'name', 'description', 'filters']) - self.assertTableColumns('project_endpoint_group', - ['endpoint_group_id', 'project_id']) - - self.assertTableDoesNotExist('project_endpoint') - self.upgrade(85) - assert_tables_columns_exist() - - @mock.patch.object(upgrades, 'get_db_version', return_value=2) - def test_endpoint_filter_already_migrated(self, mock_endpoint_filter): - - # By setting the return value to 2, the migration has already been - # run, and there's no need to create the table again. - self.upgrade(85) - - mock_endpoint_filter.assert_any_call(extension='endpoint_filter', - engine=mock.ANY) - - # It won't exist because we are mocking it, but we can verify - # that 085 did not create the table. - self.assertTableDoesNotExist('project_endpoint') - self.assertTableDoesNotExist('endpoint_group') - self.assertTableDoesNotExist('project_endpoint_group') - - def test_add_trust_unique_constraint_upgrade(self): - self.upgrade(86) - inspector = inspect(self.engine) - constraints = inspector.get_unique_constraints('trust') - constraint_names = [constraint['name'] for constraint in constraints] - self.assertIn('duplicate_trust_constraint', constraint_names) - - def test_add_domain_specific_roles(self): - """Check database upgraded successfully for domain specific roles. - - The following items need to be checked: - - - The domain_id column has been added - - That it has been added to the uniqueness constraints - - Existing roles have their domain_id columns set to the specific - string of '<<null>>' - - """ - NULL_DOMAIN_ID = '<<null>>' - - self.upgrade(87) - session = self.sessionmaker() - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - # Add a role before we upgrade, so we can check that its new domain_id - # attribute is handled correctly - role_id = uuid.uuid4().hex - self.insert_dict(session, 'role', - {'id': role_id, 'name': uuid.uuid4().hex}) - session.close() - - self.upgrade(88) - - session = self.sessionmaker() - self.assertTableColumns('role', ['id', 'name', 'domain_id', 'extra']) - # Check the domain_id has been added to the uniqueness constraint - inspector = inspect(self.engine) - constraints = inspector.get_unique_constraints('role') - constraint_columns = [ - constraint['column_names'] for constraint in constraints - if constraint['name'] == 'ixu_role_name_domain_id'] - self.assertIn('domain_id', constraint_columns[0]) - - # Now check our role has its domain_id attribute set correctly - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - cols = [role_table.c.domain_id] - filter = role_table.c.id == role_id - statement = sqlalchemy.select(cols).where(filter) - role_entry = session.execute(statement).fetchone() - self.assertEqual(NULL_DOMAIN_ID, role_entry[0]) - - def test_add_root_of_all_domains(self): - NULL_DOMAIN_ID = '<<keystone.domain.root>>' - self.upgrade(89) - session = self.sessionmaker() - - domain_table = sqlalchemy.Table( - 'domain', self.metadata, autoload=True) - query = session.query(domain_table).filter_by(id=NULL_DOMAIN_ID) - domain_from_db = query.one() - self.assertIn(NULL_DOMAIN_ID, domain_from_db) - - project_table = sqlalchemy.Table( - 'project', self.metadata, autoload=True) - query = session.query(project_table).filter_by(id=NULL_DOMAIN_ID) - project_from_db = query.one() - self.assertIn(NULL_DOMAIN_ID, project_from_db) - - session.close() - - def test_add_local_user_and_password_tables(self): - local_user_table = 'local_user' - password_table = 'password' - self.upgrade(89) - self.assertTableDoesNotExist(local_user_table) - self.assertTableDoesNotExist(password_table) - self.upgrade(90) - self.assertTableColumns(local_user_table, - ['id', - 'user_id', - 'domain_id', - 'name']) - self.assertTableColumns(password_table, - ['id', - 'local_user_id', - 'password']) - - def test_migrate_data_to_local_user_and_password_tables(self): - def get_expected_users(): - expected_users = [] - for test_user in default_fixtures.USERS: - user = {} - user['id'] = uuid.uuid4().hex - user['name'] = test_user['name'] - user['domain_id'] = test_user['domain_id'] - user['password'] = test_user['password'] - user['enabled'] = True - user['extra'] = json.dumps(uuid.uuid4().hex) - user['default_project_id'] = uuid.uuid4().hex - expected_users.append(user) - return expected_users - - def add_users_to_db(expected_users, user_table): - for user in expected_users: - ins = user_table.insert().values( - {'id': user['id'], - 'name': user['name'], - 'domain_id': user['domain_id'], - 'password': user['password'], - 'enabled': user['enabled'], - 'extra': user['extra'], - 'default_project_id': user['default_project_id']}) - ins.execute() - - def get_users_from_db(user_table, local_user_table, password_table): - sel = ( - sqlalchemy.select([user_table.c.id, - user_table.c.enabled, - user_table.c.extra, - user_table.c.default_project_id, - local_user_table.c.name, - local_user_table.c.domain_id, - password_table.c.password]) - .select_from(user_table.join(local_user_table, - user_table.c.id == - local_user_table.c.user_id) - .join(password_table, - local_user_table.c.id == - password_table.c.local_user_id)) - ) - user_rows = sel.execute() - users = [] - for row in user_rows: - users.append( - {'id': row['id'], - 'name': row['name'], - 'domain_id': row['domain_id'], - 'password': row['password'], - 'enabled': row['enabled'], - 'extra': row['extra'], - 'default_project_id': row['default_project_id']}) - return users - - user_table_name = 'user' - local_user_table_name = 'local_user' - password_table_name = 'password' - - # populate current user table - self.upgrade(90) - user_table = sqlalchemy.Table( - user_table_name, self.metadata, autoload=True) - expected_users = get_expected_users() - add_users_to_db(expected_users, user_table) - - # upgrade to migration and test - self.upgrade(91) - self.assertTableCountsMatch(user_table_name, local_user_table_name) - self.assertTableCountsMatch(local_user_table_name, password_table_name) - user_table = sqlalchemy.Table( - user_table_name, self.metadata, autoload=True) - local_user_table = sqlalchemy.Table( - local_user_table_name, self.metadata, autoload=True) - password_table = sqlalchemy.Table( - password_table_name, self.metadata, autoload=True) - actual_users = get_users_from_db(user_table, local_user_table, - password_table) - self.assertCountEqual(expected_users, actual_users) - - def test_migrate_user_with_null_password_to_password_tables(self): - USER_TABLE_NAME = 'user' - LOCAL_USER_TABLE_NAME = 'local_user' - PASSWORD_TABLE_NAME = 'password' - self.upgrade(90) - user_ref = unit.new_user_ref(uuid.uuid4().hex) - user_ref.pop('password') - # pop extra attribute which doesn't recognized by SQL expression - # layer. - user_ref.pop('email') - session = self.sessionmaker() - self.insert_dict(session, USER_TABLE_NAME, user_ref) - self.upgrade(91) - # migration should be successful. - self.assertTableCountsMatch(USER_TABLE_NAME, LOCAL_USER_TABLE_NAME) - # no new entry was added to the password table because the - # user doesn't have a password. - rows = self.calc_table_row_count(PASSWORD_TABLE_NAME) - self.assertEqual(0, rows) - - def test_migrate_user_skip_user_already_exist_in_local_user(self): - USER_TABLE_NAME = 'user' - LOCAL_USER_TABLE_NAME = 'local_user' - self.upgrade(90) - user1_ref = unit.new_user_ref(uuid.uuid4().hex) - # pop extra attribute which doesn't recognized by SQL expression - # layer. - user1_ref.pop('email') - user2_ref = unit.new_user_ref(uuid.uuid4().hex) - user2_ref.pop('email') - session = self.sessionmaker() - self.insert_dict(session, USER_TABLE_NAME, user1_ref) - self.insert_dict(session, USER_TABLE_NAME, user2_ref) - user_id = user1_ref.pop('id') - user_name = user1_ref.pop('name') - domain_id = user1_ref.pop('domain_id') - local_user_ref = {'user_id': user_id, 'name': user_name, - 'domain_id': domain_id} - self.insert_dict(session, LOCAL_USER_TABLE_NAME, local_user_ref) - self.upgrade(91) - # migration should be successful and user2_ref has been migrated to - # `local_user` table. - self.assertTableCountsMatch(USER_TABLE_NAME, LOCAL_USER_TABLE_NAME) - - def test_implied_roles_fk_on_delete_cascade(self): - if self.engine.name == 'sqlite': - self.skipTest('sqlite backend does not support foreign keys') - - self.upgrade(92) - - session = self.sessionmaker() - - ROLE_TABLE_NAME = 'role' - role_table = sqlalchemy.Table(ROLE_TABLE_NAME, self.metadata, - autoload=True) - IMPLIED_ROLE_TABLE_NAME = 'implied_role' - implied_role_table = sqlalchemy.Table( - IMPLIED_ROLE_TABLE_NAME, self.metadata, autoload=True) - - def _create_three_roles(): - id_list = [] - for _ in range(3): - new_role_fields = { - 'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - } - self.insert_dict(session, ROLE_TABLE_NAME, new_role_fields, - table=role_table) - id_list.append(new_role_fields['id']) - return id_list - - role_id_list = _create_three_roles() - implied_role_fields = { - 'prior_role_id': role_id_list[0], - 'implied_role_id': role_id_list[1], - } - self.insert_dict(session, IMPLIED_ROLE_TABLE_NAME, implied_role_fields, - table=implied_role_table) - - implied_role_fields = { - 'prior_role_id': role_id_list[0], - 'implied_role_id': role_id_list[2], - } - self.insert_dict(session, IMPLIED_ROLE_TABLE_NAME, implied_role_fields, - table=implied_role_table) - - # assert that there are two roles implied by role 0. - implied_roles = session.query(implied_role_table).filter_by( - prior_role_id=role_id_list[0]).all() - self.assertThat(implied_roles, matchers.HasLength(2)) - - session.execute( - role_table.delete().where(role_table.c.id == role_id_list[0])) - # assert the cascade deletion is effective. - implied_roles = session.query(implied_role_table).filter_by( - prior_role_id=role_id_list[0]).all() - self.assertThat(implied_roles, matchers.HasLength(0)) - - def test_domain_as_project_upgrade(self): - self.skipTest('Domain as Project Upgrade Test is no longer needed and ' - 'unfortunately broken by the resource options code.') - - def _populate_domain_and_project_tables(session): - # Three domains, with various different attributes - self.domains = [{'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True, - 'extra': {'description': uuid.uuid4().hex, - 'another_attribute': True}}, - {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': True, - 'extra': {'description': uuid.uuid4().hex}}, - {'id': uuid.uuid4().hex, - 'name': uuid.uuid4().hex, - 'enabled': False}] - # Four projects, two top level, two children - self.projects = [] - self.projects.append(unit.new_project_ref( - domain_id=self.domains[0]['id'], - parent_id=None)) - self.projects.append(unit.new_project_ref( - domain_id=self.domains[0]['id'], - parent_id=self.projects[0]['id'])) - self.projects.append(unit.new_project_ref( - domain_id=self.domains[1]['id'], - parent_id=None)) - self.projects.append(unit.new_project_ref( - domain_id=self.domains[1]['id'], - parent_id=self.projects[2]['id'])) - - for domain in self.domains: - this_domain = domain.copy() - if 'extra' in this_domain: - this_domain['extra'] = json.dumps(this_domain['extra']) - self.insert_dict(session, 'domain', this_domain) - for project in self.projects: - # Tags are done via relationship, not column - project.pop('tags', None) - self.insert_dict(session, 'project', project) - - def _check_projects(projects): - - def _assert_domain_matches_project(project): - for domain in self.domains: - if project.id == domain['id']: - self.assertEqual(domain['name'], project.name) - self.assertEqual(domain['enabled'], project.enabled) - if domain['id'] == self.domains[0]['id']: - self.assertEqual(domain['extra']['description'], - project.description) - self.assertEqual({'another_attribute': True}, - json.loads(project.extra)) - elif domain['id'] == self.domains[1]['id']: - self.assertEqual(domain['extra']['description'], - project.description) - self.assertEqual({}, json.loads(project.extra)) - - # We had domains 3 we created, which should now be projects acting - # as domains, To this we add the 4 original projects, plus the root - # of all domains row. - self.assertEqual(8, projects.count()) - - project_ids = [] - for project in projects: - if project.is_domain: - self.assertEqual(NULL_DOMAIN_ID, project.domain_id) - self.assertIsNone(project.parent_id) - else: - self.assertIsNotNone(project.domain_id) - self.assertIsNotNone(project.parent_id) - project_ids.append(project.id) - - for domain in self.domains: - self.assertIn(domain['id'], project_ids) - for project in self.projects: - self.assertIn(project['id'], project_ids) - - # Now check the attributes of the domains came across OK - for project in projects: - _assert_domain_matches_project(project) - - NULL_DOMAIN_ID = '<<keystone.domain.root>>' - self.upgrade(92) - - session = self.sessionmaker() - - _populate_domain_and_project_tables(session) - - self.upgrade(93) - proj_table = sqlalchemy.Table('project', self.metadata, autoload=True) - - projects = session.query(proj_table) - _check_projects(projects) - - def test_add_federated_user_table(self): - federated_user_table = 'federated_user' - self.upgrade(93) - self.assertTableDoesNotExist(federated_user_table) - self.upgrade(94) - self.assertTableColumns(federated_user_table, - ['id', - 'user_id', - 'idp_id', - 'protocol_id', - 'unique_id', - 'display_name']) - - def test_add_int_pkey_to_revocation_event_table(self): - REVOCATION_EVENT_TABLE_NAME = 'revocation_event' - self.upgrade(94) - revocation_event_table = sqlalchemy.Table(REVOCATION_EVENT_TABLE_NAME, - self.metadata, autoload=True) - # assert id column is a string (before) - self.assertEqual('VARCHAR(64)', str(revocation_event_table.c.id.type)) - self.upgrade(95) - revocation_event_table = sqlalchemy.Table(REVOCATION_EVENT_TABLE_NAME, - self.metadata, autoload=True) - # assert id column is an integer (after) - self.assertIsInstance(revocation_event_table.c.id.type, sql.Integer) - def _add_unique_constraint_to_role_name(self, constraint_name='ixu_role_name'): role_table = sqlalchemy.Table('role', self.metadata, autoload=True) migrate.UniqueConstraint(role_table.c.name, name=constraint_name).create() - def _drop_unique_constraint_to_role_name(self, - constraint_name='ixu_role_name'): - role_table = sqlalchemy.Table('role', self.metadata, autoload=True) - migrate.UniqueConstraint(role_table.c.name, - name=constraint_name).drop() - def _add_unique_constraint_to_user_name_domainid( self, constraint_name='ixu_role_name'): @@ -966,204 +442,6 @@ class SqlLegacyRepoUpgradeTests(SqlMigrateBase): user_table.create_column(column_name) user_table.create_column(column_domain_id) - def _drop_unique_constraint_to_user_name_domainid( - self, - constraint_name='ixu_user_name_domain_id'): - user_table = sqlalchemy.Table('user', self.metadata, autoload=True) - migrate.UniqueConstraint(user_table.c.name, user_table.c.domain_id, - name=constraint_name).drop() - - def test_migration_88_drops_unique_constraint(self): - self.upgrade(87) - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertTrue(self.does_constraint_exist('role', - 'ixu_role_name')) - self.upgrade(88) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - def test_migration_88_inconsistent_constraint_name(self): - self.upgrade(87) - self._drop_unique_constraint_to_role_name() - - constraint_name = uuid.uuid4().hex - self._add_unique_constraint_to_role_name( - constraint_name=constraint_name) - - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('role', constraint_name)) - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertTrue(self.does_constraint_exist('role', - constraint_name)) - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - self.upgrade(88) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('role', constraint_name)) - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertFalse(self.does_constraint_exist('role', - constraint_name)) - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - def test_migration_91_drops_unique_constraint(self): - self.upgrade(90) - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('user', - 'ixu_user_name_domain_id')) - else: - self.assertTrue(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - self.upgrade(91) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - def test_migration_91_inconsistent_constraint_name(self): - self.upgrade(90) - self._drop_unique_constraint_to_user_name_domainid() - - constraint_name = uuid.uuid4().hex - self._add_unique_constraint_to_user_name_domainid( - constraint_name=constraint_name) - - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('user', constraint_name)) - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertTrue(self.does_constraint_exist('user', - constraint_name)) - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - self.upgrade(91) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('user', constraint_name)) - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertFalse(self.does_constraint_exist('user', - constraint_name)) - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - def test_migration_96(self): - self.upgrade(95) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - self.upgrade(96) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - def test_migration_96_constraint_exists(self): - self.upgrade(95) - self._add_unique_constraint_to_role_name() - - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertTrue(self.does_constraint_exist('role', - 'ixu_role_name')) - - self.upgrade(96) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('role', 'ixu_role_name')) - else: - self.assertFalse(self.does_constraint_exist('role', - 'ixu_role_name')) - - def test_migration_97(self): - self.upgrade(96) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - self.upgrade(97) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - def test_migration_97_constraint_exists(self): - self.upgrade(96) - self._add_name_domain_id_columns_to_user() - self._add_unique_constraint_to_user_name_domainid( - constraint_name='ixu_user_name_domain_id') - - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertTrue(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - self.upgrade(97) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist( - 'user', - 'ixu_user_name_domain_id')) - else: - self.assertFalse(self.does_constraint_exist( - 'user', - 'ixu_user_name_domain_id')) - - def test_migration_97_inconsistent_constraint_exists(self): - self.upgrade(96) - constraint_name = uuid.uuid4().hex - self._add_name_domain_id_columns_to_user() - self._add_unique_constraint_to_user_name_domainid( - constraint_name=constraint_name) - - if self.engine.name == 'mysql': - self.assertTrue(self.does_index_exist('user', constraint_name)) - else: - self.assertTrue(self.does_constraint_exist('user', - constraint_name)) - - self.upgrade(97) - if self.engine.name == 'mysql': - self.assertFalse(self.does_index_exist('user', constraint_name)) - else: - self.assertFalse(self.does_constraint_exist('user', - constraint_name)) - def test_migration_101(self): self.upgrade(100) if self.engine.name == 'mysql': @@ -1599,7 +877,7 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase): def test_running_db_sync_expand_without_up_to_date_legacy_fails(self): # Set Legacy version and then test that running expand fails if Legacy # isn't at the latest version. - self.upgrade(75) + self.upgrade(97) latest_version = self.repos[EXPAND_REPO].max_version self.assertRaises( db_exception.DBMigrationError, |