Add migration script to change backend to store

As glance is changing metadata value of location from 'backend' to
'store', adding migration script which will update old image locations
to use 'store' as a metadata value.

NOTE:
Bump master (train development) to 19

Change-Id: I1386c535bc8ff4519e6b0bb879026b05c930b791
Sem-Ver: api-break

6 files changed, 236 insertions, 3 deletions

diff --git a/glance/db/migration.py b/glance/db/migration.py
index 79d458427..b85866b8e 100644
--- a/glance/db/migration.py
+++ b/glance/db/migration.py
@@ -47,9 +47,9 @@ def get_backend():
 # Migration-related constants
 EXPAND_BRANCH = 'expand'
 CONTRACT_BRANCH = 'contract'
-CURRENT_RELEASE = 'stein'
+CURRENT_RELEASE = 'train'
 ALEMBIC_INIT_VERSION = 'liberty'
-LATEST_REVISION = 'queens_contract01'
+LATEST_REVISION = 'rocky_contract02'
 INIT_VERSION = 0
 
 MIGRATE_REPO_PATH = os.path.join(
diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py
new file mode 100644
index 000000000..cfb2b8208
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/train_migrate01_backend_to_store.py
@@ -0,0 +1,42 @@
+# Copyright 2019 RedHat Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def has_migrations(engine):
+    """Returns true if at least one data row can be migrated.
+
+    There are rows left to migrate if meta_data column has
+    {"backend": "...."}
+
+    Note: This method can return a false positive if data migrations
+    are running in the background as it's being called.
+    """
+    sql_query = ("select meta_data from image_locations where "
+                 "INSTR(meta_data, '\"backend\":') > 0")
+    with engine.connect() as con:
+        metadata_backend = con.execute(sql_query)
+        if metadata_backend.rowcount > 0:
+            return True
+
+    return False
+
+
+def migrate(engine):
+    """Replace 'backend' with 'store' in meta_data column of image_locations"""
+    sql_query = ("UPDATE image_locations SET meta_data = REPLACE(meta_data, "
+                 "'\"backend\":', '\"store\":') where INSTR(meta_data, "
+                 " '\"backend\":') > 0")
+    with engine.connect() as con:
+        migrated_rows = con.execute(sql_query)
+        return migrated_rows.rowcount
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py
new file mode 100644
index 000000000..eaa6b546d
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/train_contract01_empty.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2019 RedHat Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+# revision identifiers, used by Alembic.
+revision = 'train_contract01'
+down_revision = 'rocky_contract02'
+branch_labels = None
+depends_on = 'train_expand01'
+
+
+def upgrade():
+    pass
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py b/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py
new file mode 100644
index 000000000..d8fd277d8
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/train_expand01_empty.py
@@ -0,0 +1,30 @@
+# Copyright (C) 2019 RedHat Inc
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""empty expand for symmetry with train_contract01
+
+Revision ID: train_expand01
+Revises: rocky_expand02
+Create Date: 2019-06-17 11:55:16.657499
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'train_expand01'
+down_revision = 'rocky_expand02'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
diff --git a/glance/tests/functional/db/migrations/test_train_migrate01.py b/glance/tests/functional/db/migrations/test_train_migrate01.py
new file mode 100644
index 000000000..6d13eb753
--- /dev/null
+++ b/glance/tests/functional/db/migrations/test_train_migrate01.py
@@ -0,0 +1,131 @@
+# Copyright 2019 RedHat Inc
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import datetime
+
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import utils as db_utils
+
+from glance.db.sqlalchemy.alembic_migrations import data_migrations
+from glance.tests.functional.db import test_migrations
+
+
+class TestTrainMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='train_expand01')
+
+    def _pre_upgrade_train_expand01(self, engine):
+        images = db_utils.get_table(engine, 'images')
+        image_locations = db_utils.get_table(engine, 'image_locations')
+        now = datetime.datetime.now()
+
+        # inserting a public image record
+        image_1 = dict(deleted=False,
+                       created_at=now,
+                       status='active',
+                       min_disk=0,
+                       min_ram=0,
+                       visibility='public',
+                       id='image_1')
+        images.insert().values(image_1).execute()
+
+        image_2 = dict(deleted=False,
+                       created_at=now,
+                       status='active',
+                       min_disk=0,
+                       min_ram=0,
+                       visibility='public',
+                       id='image_2')
+        images.insert().values(image_2).execute()
+
+        # adding records to image_locations tables
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='image_1',
+                    value='image_location_1',
+                    meta_data='{"backend": "fast"}',
+                    id=1)
+        image_locations.insert().values(temp).execute()
+
+        temp = dict(deleted=False,
+                    created_at=now,
+                    image_id='image_2',
+                    value='image_location_2',
+                    meta_data='{"backend": "cheap"}',
+                    id=2)
+        image_locations.insert().values(temp).execute()
+
+    def _check_train_expand01(self, engine, data):
+        image_locations = db_utils.get_table(engine, 'image_locations')
+
+        # check that meta_data has 'backend' key for existing image_locations
+        rows = (image_locations.select()
+                .order_by(image_locations.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(2, len(rows))
+        for row in rows:
+            self.assertIn('"backend":', row['meta_data'])
+
+        # run data migrations
+        data_migrations.migrate(engine)
+
+        # check that meta_data has 'backend' key replaced with 'store'
+        rows = (image_locations.select()
+                .order_by(image_locations.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(2, len(rows))
+        for row in rows:
+            self.assertNotIn('"backend":', row['meta_data'])
+            self.assertIn('"store":', row['meta_data'])
+
+
+class TestTrainMigrate01MySQL(TestTrainMigrate01Mixin,
+                              test_base.MySQLOpportunisticTestCase):
+    pass
+
+
+class TestTrainMigrate01_EmptyDBMixin(test_migrations.AlembicMigrationsMixin):
+    """This mixin is used to create an initial glance database and upgrade it
+    up to the train_expand01 revision.
+    """
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='train_expand01')
+
+    def _pre_upgrade_train_expand01(self, engine):
+        # New/empty database
+        pass
+
+    def _check_train_expand01(self, engine, data):
+        images = db_utils.get_table(engine, 'images')
+
+        # check that there are no rows in the images table
+        rows = (images.select()
+                .order_by(images.c.id)
+                .execute()
+                .fetchall())
+        self.assertEqual(0, len(rows))
+
+        # run data migrations
+        data_migrations.migrate(engine)
+
+
+class TestTrainMigrate01_EmptyDBMySQL(TestTrainMigrate01_EmptyDBMixin,
+                                      test_base.MySQLOpportunisticTestCase):
+    """This test runs the Train data migrations on an empty databse."""
+    pass
diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py
index 4da916bda..f193d9171 100644
--- a/glance/tests/unit/test_manage.py
+++ b/glance/tests/unit/test_manage.py
@@ -379,6 +379,9 @@ class TestManage(TestManageBase):
                       exit.code)
 
     @mock.patch(
+        'glance.db.sqlalchemy.alembic_migrations.data_migrations.'
+        'has_pending_migrations')
+    @mock.patch(
         'glance.db.sqlalchemy.alembic_migrations.get_current_alembic_heads')
     @mock.patch(
         'glance.db.sqlalchemy.alembic_migrations.get_alembic_branch_head')
@@ -386,13 +389,15 @@ class TestManage(TestManageBase):
     @mock.patch.object(manage.DbCommands, '_sync')
     def test_contract(self, mock_sync, mock_validate_engine,
                       mock_get_alembic_branch_head,
-                      mock_get_current_alembic_heads):
+                      mock_get_current_alembic_heads,
+                      mock_has_pending_migrations):
         engine = mock_validate_engine.return_value
         engine.engine.name = 'mysql'
         mock_get_current_alembic_heads.side_effect = ['pike_expand01',
                                                       'pike_contract01']
         mock_get_alembic_branch_head.side_effect = ['pike_contract01',
                                                     'pike_expand01']
+        mock_has_pending_migrations.return_value = False
         self.db.contract()
         mock_sync.assert_called_once_with(version='pike_contract01')