Add 'version' to datastore version

This patch is part of story 2008358 implementation.

1. DB schema change
2. trove-manage datastore_version_update subcommand change
3. API change for creating and listing datastore version.

Story: 2008358
Task: 41264

Change-Id: I8069e6f4b972497f2b8be540ea35047d5fb2f9a5

11 files changed, 171 insertions, 42 deletions

diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 3d45bf84..afda34a6 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -481,10 +481,11 @@ function create_guest_image {
       --property hw_rng_model='virtio' \
       --file ${image_file} \
       -c id -f value)
+     echo "Glance image ${glance_image_id} uploaded"
 
     echo "Register the image in datastore"
     $TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE ""
-    $TROVE_MANAGE datastore_version_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION $TROVE_DATASTORE_TYPE $glance_image_id "trove" "" 1
+    $TROVE_MANAGE datastore_version_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION $TROVE_DATASTORE_TYPE "" "trove" "" 1
     $TROVE_MANAGE datastore_update $TROVE_DATASTORE_TYPE $TROVE_DATASTORE_VERSION
 
     echo "Add parameter validation rules if available"
diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack
index 4e6742a1..5672ca13 100755
--- a/integration/scripts/trovestack
+++ b/integration/scripts/trovestack
@@ -521,11 +521,8 @@ function set_bin_path() {
 }
 
 function cmd_set_datastore() {
-    local IMAGEID=$1
-
     rd_manage datastore_update "$datastore" ""
-    # trove-manage datastore_version_update <datastore_name> <version_name> <datastore_manager> <image_id> <image_tags> <packages> <active>
-    rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" ${IMAGEID} "trove" "" 1
+    rd_manage datastore_version_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}" "${DATASTORE_TYPE}" "" "trove" "" 1
     rd_manage datastore_update "${DATASTORE_TYPE}" "${DATASTORE_VERSION}"
 
     if [[ -f "$PATH_TROVE"/trove/templates/${DATASTORE_TYPE}/validation-rules.json ]]; then
@@ -781,7 +778,7 @@ function cmd_build_and_upload_image() {
     exclaim "Using Glance image ID: $glance_imageid"
 
     exclaim "Updating Datastores"
-    cmd_set_datastore "${glance_imageid}"
+    cmd_set_datastore
 }
 
 
diff --git a/trove/cmd/manage.py b/trove/cmd/manage.py
index 52a342ec..abe3c028 100644
--- a/trove/cmd/manage.py
+++ b/trove/cmd/manage.py
@@ -62,15 +62,18 @@ class Commands(object):
             print(e)
 
     def datastore_version_update(self, datastore, version_name, manager,
-                                 image_id, image_tags, packages, active):
+                                 image_id, image_tags, packages, active,
+                                 version=None):
         try:
             datastore_models.update_datastore_version(datastore,
                                                       version_name,
                                                       manager,
                                                       image_id,
                                                       image_tags,
-                                                      packages, active)
-            print("Datastore version '%s' updated." % version_name)
+                                                      packages, active,
+                                                      version=version)
+            print("Datastore version '%s(%s)' updated." %
+                  (version_name, version))
         except exception.DatastoreNotFound as e:
             print(e)
 
@@ -223,6 +226,10 @@ def main():
             'active', type=int,
             help='Whether the datastore version is active or not. '
             'Accepted values are 0 and 1.')
+        parser.add_argument(
+            '--version',
+            help='The version number of the datastore version, e.g. 5.7.30. '
+                 'If not specified, use <version_name> as default value.')
 
         parser = subparser.add_parser(
             'db_recreate', description='Drop the database and recreate it.')
diff --git a/trove/common/apischema.py b/trove/common/apischema.py
index 9af0e19c..4b4bf9a8 100644
--- a/trove/common/apischema.py
+++ b/trove/common/apischema.py
@@ -979,7 +979,8 @@ mgmt_datastore_version = {
                     "image": uuid,
                     "image_tags": image_tags,
                     "active": {"enum": [True, False]},
-                    "default": {"enum": [True, False]}
+                    "default": {"enum": [True, False]},
+                    "version": non_empty_string
                 }
             }
         }
diff --git a/trove/common/exception.py b/trove/common/exception.py
index 40d22463..f89bef1f 100644
--- a/trove/common/exception.py
+++ b/trove/common/exception.py
@@ -180,7 +180,8 @@ class DatastoreVersionInactive(TroveError):
 
 class DatastoreVersionAlreadyExists(BadRequest):
 
-    message = _("A datastore version with the name '%(name)s' already exists.")
+    message = _("The datastore version '%(name)s(%(version)s)' already "
+                "exists.")
 
 
 class DatastoreVersionsExist(BadRequest):
diff --git a/trove/datastore/models.py b/trove/datastore/models.py
index bf989133..5f79dea9 100644
--- a/trove/datastore/models.py
+++ b/trove/datastore/models.py
@@ -16,6 +16,7 @@
 #    under the License.
 
 from oslo_log import log as logging
+from oslo_utils import uuidutils
 
 from trove.common import cfg
 from trove.common.clients import create_nova_client
@@ -63,7 +64,7 @@ class DBCapabilityOverrides(dbmodels.DatabaseModelBase):
 
 class DBDatastoreVersion(dbmodels.DatabaseModelBase):
     _data_fields = ['datastore_id', 'name', 'image_id', 'image_tags',
-                    'packages', 'active', 'manager']
+                    'packages', 'active', 'manager', 'version']
     _table_name = 'datastore_versions'
 
 
@@ -399,18 +400,20 @@ class DatastoreVersion(object):
         return "%s(%s)" % (self.name, self.id)
 
     @classmethod
-    def load(cls, datastore, id_or_name):
-        try:
+    def load(cls, datastore, id_or_name, version=None):
+        if uuidutils.is_uuid_like(id_or_name):
             return cls(DBDatastoreVersion.find_by(datastore_id=datastore.id,
                                                   id=id_or_name))
-        except exception.ModelNotFoundError:
-            versions = DBDatastoreVersion.find_all(datastore_id=datastore.id,
-                                                   name=id_or_name)
-            if versions.count() == 0:
-                raise exception.DatastoreVersionNotFound(version=id_or_name)
-            if versions.count() > 1:
-                raise exception.NoUniqueMatch(name=id_or_name)
-            return cls(versions.first())
+
+        version = version or id_or_name
+        versions = DBDatastoreVersion.find_all(datastore_id=datastore.id,
+                                               name=id_or_name,
+                                               version=version)
+        if versions.count() == 0:
+            raise exception.DatastoreVersionNotFound(version=version)
+        if versions.count() > 1:
+            raise exception.NoUniqueMatch(name=id_or_name)
+        return cls(versions.first())
 
     @classmethod
     def load_by_uuid(cls, uuid):
@@ -474,6 +477,10 @@ class DatastoreVersion(object):
 
         return self._capabilities
 
+    @property
+    def version(self):
+        return self.db_info.version
+
 
 class DatastoreVersions(object):
 
@@ -581,26 +588,30 @@ def update_datastore(name, default_version):
 
 
 def update_datastore_version(datastore, name, manager, image_id, image_tags,
-                             packages, active):
+                             packages, active, version=None):
+    """Create or update datastore version."""
+    version = version or name
     db_api.configure_db(CONF)
     datastore = Datastore.load(datastore)
     try:
-        version = DBDatastoreVersion.find_by(datastore_id=datastore.id,
-                                             name=name)
+        ds_version = DBDatastoreVersion.find_by(datastore_id=datastore.id,
+                                                name=name,
+                                                version=version)
     except exception.ModelNotFoundError:
         # Create a new one
-        version = DBDatastoreVersion()
-        version.id = utils.generate_uuid()
-        version.name = name
-        version.datastore_id = datastore.id
-    version.manager = manager
-    version.image_id = image_id
-    version.image_tags = (",".join(image_tags)
-                          if type(image_tags) is list else image_tags)
-    version.packages = packages
-    version.active = active
-
-    db_api.save(version)
+        ds_version = DBDatastoreVersion()
+        ds_version.id = utils.generate_uuid()
+        ds_version.name = name
+        ds_version.version = version
+        ds_version.datastore_id = datastore.id
+    ds_version.manager = manager
+    ds_version.image_id = image_id
+    ds_version.image_tags = (",".join(image_tags)
+                             if type(image_tags) is list else image_tags)
+    ds_version.packages = packages
+    ds_version.active = active
+
+    db_api.save(ds_version)
 
 
 class DatastoreVersionMetadata(object):
diff --git a/trove/datastore/views.py b/trove/datastore/views.py
index 3dfb321a..be607257 100644
--- a/trove/datastore/views.py
+++ b/trove/datastore/views.py
@@ -80,6 +80,7 @@ class DatastoreVersionView(object):
         datastore_version_dict = {
             "id": self.datastore_version.id,
             "name": self.datastore_version.name,
+            "version": self.datastore_version.version,
             "links": self._build_links(),
         }
         if include_datastore_id:
diff --git a/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py b/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py
new file mode 100644
index 00000000..bf0dd268
--- /dev/null
+++ b/trove/db/sqlalchemy/migrate_repo/versions/048_add_version_to_datastore_version.py
@@ -0,0 +1,71 @@
+# Copyright 2020 Catalyst Cloud
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate.changeset.constraint import UniqueConstraint
+from sqlalchemy import text
+from sqlalchemy.schema import Column
+from sqlalchemy.schema import MetaData
+from sqlalchemy.sql.expression import select
+from sqlalchemy.sql.expression import update
+
+from trove.db.sqlalchemy import utils as db_utils
+from trove.db.sqlalchemy.migrate_repo.schema import String
+from trove.db.sqlalchemy.migrate_repo.schema import Table
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+
+    ds_table = Table('datastores', meta, autoload=True)
+    ds_version_table = Table('datastore_versions', meta, autoload=True)
+    ds_version_table.create_column(
+        Column('version', String(255), nullable=True))
+
+    ds_versions = select(
+        columns=[text("id"), text("name")],
+        from_obj=ds_version_table
+    ).execute()
+
+    # Use 'name' value as init 'version' value
+    for version in ds_versions:
+        update(
+            table=ds_version_table,
+            whereclause=text("id='%s'" % version.id),
+            values=dict(version=version.name)
+        ).execute()
+
+    # Change unique constraint, need to drop the foreign key first and add back
+    # later
+    constraint_names = db_utils.get_foreign_key_constraint_names(
+        engine=migrate_engine,
+        table='datastore_versions',
+        columns=['datastore_id'],
+        ref_table='datastores',
+        ref_columns=['id'])
+    db_utils.drop_foreign_key_constraints(
+        constraint_names=constraint_names,
+        columns=[ds_version_table.c.datastore_id],
+        ref_columns=[ds_table.c.id])
+
+    UniqueConstraint('datastore_id', 'name', name='ds_versions',
+                     table=ds_version_table).drop()
+    UniqueConstraint('datastore_id', 'name', 'version', name='ds_versions',
+                     table=ds_version_table).create()
+
+    db_utils.create_foreign_key_constraints(
+        constraint_names=constraint_names,
+        columns=[ds_version_table.c.datastore_id],
+        ref_columns=[ds_table.c.id])
diff --git a/trove/extensions/mgmt/datastores/service.py b/trove/extensions/mgmt/datastores/service.py
index fb730c9e..04c7e70a 100644
--- a/trove/extensions/mgmt/datastores/service.py
+++ b/trove/extensions/mgmt/datastores/service.py
@@ -49,6 +49,9 @@ class DatastoreVersionController(wsgi.Controller):
             packages = ','.join(packages)
         active = body['version']['active']
         default = body['version'].get('default', False)
+        # For backward compatibility, use name as default value for version if
+        # not specified
+        version_str = body['version'].get('version', version_name)
 
         LOG.info("Tenant: '%(tenant)s' is adding the datastore "
                  "version: '%(version)s' to datastore: '%(datastore)s'",
@@ -72,12 +75,15 @@ class DatastoreVersionController(wsgi.Controller):
             datastore.save()
 
         try:
-            models.DatastoreVersion.load(datastore, version_name)
-            raise exception.DatastoreVersionAlreadyExists(name=version_name)
+            models.DatastoreVersion.load(datastore, version_name,
+                                         version=version_str)
+            raise exception.DatastoreVersionAlreadyExists(
+                name=version_name, version=version_str)
         except exception.DatastoreVersionNotFound:
             models.update_datastore_version(datastore.name, version_name,
                                             manager, image_id, image_tags,
-                                            packages, active)
+                                            packages, active,
+                                            version=version_str)
 
         if default:
             models.update_datastore(datastore.name, version_name)
diff --git a/trove/extensions/mgmt/datastores/views.py b/trove/extensions/mgmt/datastores/views.py
index 971bbc0d..ca6328cb 100644
--- a/trove/extensions/mgmt/datastores/views.py
+++ b/trove/extensions/mgmt/datastores/views.py
@@ -22,6 +22,7 @@ class DatastoreVersionView(object):
         datastore_version_dict = {
             "id": self.datastore_version.id,
             "name": self.datastore_version.name,
+            "version": self.datastore_version.version,
             "datastore_id": self.datastore_version.datastore_id,
             "datastore_name": self.datastore_version.datastore_name,
             "datastore_manager": self.datastore_version.manager,
diff --git a/trove/tests/unittests/extensions/mgmt/datastores/test_service.py b/trove/tests/unittests/extensions/mgmt/datastores/test_service.py
index 185a740e..ebed3553 100644
--- a/trove/tests/unittests/extensions/mgmt/datastores/test_service.py
+++ b/trove/tests/unittests/extensions/mgmt/datastores/test_service.py
@@ -32,6 +32,7 @@ class TestDatastoreVersionController(trove_testtools.TestCase):
     def setUpClass(cls):
         util.init_db()
         cls.ds_name = cls.random_name('datastore')
+        cls.ds_version_number = '5.7.30'
         models.update_datastore(name=cls.ds_name, default_version=None)
 
         models.update_datastore_version(
@@ -39,11 +40,12 @@ class TestDatastoreVersionController(trove_testtools.TestCase):
             1)
         models.update_datastore_version(
             cls.ds_name, 'test_vr2', 'mysql', cls.random_uuid(), '', 'pkg-1',
-            1)
+            1, version=cls.ds_version_number)
 
         cls.ds = models.Datastore.load(cls.ds_name)
         cls.ds_version1 = models.DatastoreVersion.load(cls.ds, 'test_vr1')
-        cls.ds_version2 = models.DatastoreVersion.load(cls.ds, 'test_vr2')
+        cls.ds_version2 = models.DatastoreVersion.load(
+            cls.ds, 'test_vr2', version=cls.ds_version_number)
         cls.version_controller = DatastoreVersionController()
 
         super(TestDatastoreVersionController, cls).setUpClass()
@@ -136,6 +138,34 @@ class TestDatastoreVersionController(trove_testtools.TestCase):
 
         new_ver = models.DatastoreVersion.load(self.ds, ver_name)
         self.assertEqual(image_id, new_ver.image_id)
+        self.assertEqual(ver_name, new_ver.version)
+
+    @patch.object(clients, 'create_glance_client')
+    def test_create_same_version_number(self, mock_glance_client):
+        image_id = self.random_uuid()
+        ver_name = self.random_name('dsversion')
+        body = {
+            "version": {
+                "datastore_name": self.ds_name,
+                "name": ver_name,
+                "datastore_manager": "mysql",
+                "image": image_id,
+                "image_tags": [],
+                "packages": "",
+                "active": True,
+                "default": False,
+                "version": self.ds_version_number
+            }
+        }
+        output = self.version_controller.create(MagicMock(), body, mock.ANY)
+        self.assertEqual(202, output.status)
+
+        new_ver = models.DatastoreVersion.load(self.ds, ver_name,
+                                               version=self.ds_version_number)
+        self.assertEqual(image_id, new_ver.image_id)
+        self.assertEqual(ver_name, new_ver.name)
+        self.assertEqual(self.ds_version_number, new_ver.version)
+        self.assertNotEqual(self.ds_version2.id, new_ver.id)
 
     @patch.object(clients, 'create_glance_client')
     def test_create_by_image_tags(self, mock_create_client):
@@ -304,6 +334,8 @@ class TestDatastoreVersionController(trove_testtools.TestCase):
                          output._data['version']['packages'])
         self.assertEqual(self.ds_version2.active,
                          output._data['version']['active'])
+        self.assertEqual(self.ds_version2.version,
+                         output._data['version']['version'])
 
     def test_show_image_tags(self):
         ver_name = self.random_name('dsversion')