Support to restore backup from remote location

In multi-region deployment with geo-replicated Swift, the user can
restore a backup in one region by manually specifying the original
backup data location created in another region.

Change-Id: Iefef3bf969163af707935445bc23299400dc88c3

15 files changed, 329 insertions, 65 deletions

diff --git a/api-ref/source/backups.inc b/api-ref/source/backups.inc
index a9a086c2..2e3d56f0 100644
--- a/api-ref/source/backups.inc
+++ b/api-ref/source/backups.inc
@@ -76,6 +76,19 @@ In the Trove deployment with service tenant enabled, The backup data is
 stored as objects in OpenStack Swift service in the user's container. If not
 specified, the container name is defined by the cloud admin.
 
+The user can create a backup strategy within the project scope or specific to
+a particular instance.
+
+In multi-region deployment with geo-replicated Swift, the user can also restore
+a backup in a region by manually specifying the backup data location created in
+another region, then create instances from the backup. Instance ID is not
+required in this case.
+
+.. warning::
+
+   The restored backup is dependent on the original backup data, if the
+   original backup is deleted, the restored backup is invalid.
+
 Normal response codes: 202
 
 Request
@@ -90,6 +103,7 @@ Request
    - incremental: backup_incremental
    - description: backup_description
    - swift_container: swift_container
+   - restore_from: backup_restore_from
 
 Request Example
 ---------------
diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml
index 6918f2e4..a2c6c910 100755
--- a/api-ref/source/parameters.yaml
+++ b/api-ref/source/parameters.yaml
@@ -148,7 +148,7 @@ backup_instanceId:
   description: |
     The ID of the instance to create backup for.
   in: body
-  required: true
+  required: false
   type: string
 backup_list:
   description: |
@@ -180,6 +180,17 @@ backup_parentId1:
   in: body
   required: true
   type: string
+backup_restore_from:
+  description: |
+    The information needed to restore a backup, including:
+
+    - ``remote_location``: The original backup data location.
+    - ``local_datastore_version_id``: The local datastore version corresponding
+      to the original backup.
+    - ``size``: The original backup size.
+  in: body
+  required: false
+  type: object
 backup_size:
   description: |
     Size of the backup, the unit is GB.
diff --git a/doc/source/user/backup-db.rst b/doc/source/user/backup-db.rst
index 9a1a10e5..9aa5e13b 100644
--- a/doc/source/user/backup-db.rst
+++ b/doc/source/user/backup-db.rst
@@ -276,3 +276,21 @@ Create an incremental backup based on a parent backup:
     |    status   |                 NEW                  |
     |   updated   |         2014-03-19T14:09:13          |
     +-------------+--------------------------------------+
+
+Restore backup from other regions
+---------------------------------
+
+Restoring backup from other regions were introduced in Wallaby,
+
+In multi-region deployment with geo-replicated Swift, the user is able to
+create a backup in one region using the backup data created in the others,
+which is useful in Disaster Recovery scenario. Instance ID is not required in
+this case when restoring backup, but the original backup data location (a swift
+object URL), the local datastore version and the backup data size are required.
+
+.. warning::
+
+   The restored backup is dependent on the original backup data, if the
+   original backup is deleted, the restored backup is invalid.
+
+TODO: Add CLI example once supported in python-troveclient.
diff --git a/releasenotes/notes/wallaby-restore-backup.yaml b/releasenotes/notes/wallaby-restore-backup.yaml
new file mode 100644
index 00000000..f7e369e7
--- /dev/null
+++ b/releasenotes/notes/wallaby-restore-backup.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - In multi-region deployment with geo-replicated Swift, the user can restore
+    a backup in one region by manually specifying the original backup data
+    location created in another region.
diff --git a/trove/backup/models.py b/trove/backup/models.py
index 10c58ad2..5903b1b6 100644
--- a/trove/backup/models.py
+++ b/trove/backup/models.py
@@ -22,7 +22,9 @@ from swiftclient.client import ClientException
 from trove.backup.state import BackupState
 from trove.common import cfg
 from trove.common import clients
+from trove.common import constants
 from trove.common import exception
+from trove.common import swift
 from trove.common import utils
 from trove.common.i18n import _
 from trove.datastore import models as datastore_models
@@ -49,7 +51,8 @@ class Backup(object):
 
     @classmethod
     def create(cls, context, instance, name, description=None,
-               parent_id=None, incremental=False, swift_container=None):
+               parent_id=None, incremental=False, swift_container=None,
+               restore_from=None):
         """
         create db record for Backup
         :param cls:
@@ -59,31 +62,61 @@ class Backup(object):
         :param description:
         :param parent_id:
         :param incremental: flag to indicate incremental backup
-        based on previous backup
+                            based on previous backup
         :param swift_container: Swift container name.
+        :param restore_from: A dict that contains backup information of another
+                             region.
         :return:
         """
-
-        def _create_resources():
-            # parse the ID from the Ref
+        backup_state = BackupState.NEW
+        checksum = None
+        instance_id = None
+        parent = None
+        last_backup_id = None
+        location = None
+        backup_type = constants.BACKUP_TYPE_FULL
+        size = None
+
+        if restore_from:
+            # Check location and datastore version.
+            LOG.info(f"Restoring backup, restore_from: {restore_from}")
+            backup_state = BackupState.RESTORED
+
+            ds_version_id = restore_from.get('local_datastore_version_id')
+            ds_version = datastore_models.DatastoreVersion.load_by_uuid(
+                ds_version_id)
+
+            location = restore_from.get('remote_location')
+            swift_client = clients.create_swift_client(context)
+            try:
+                obj_meta = swift.get_metadata(swift_client, location,
+                                              extra_attrs=['etag'])
+            except Exception:
+                msg = f'Failed to restore backup from {location}'
+                LOG.exception(msg)
+                raise exception.BackupCreationError(msg)
+
+            checksum = obj_meta['etag']
+            if 'parent_location' in obj_meta:
+                backup_type = constants.BACKUP_TYPE_INC
+
+            size = restore_from['size']
+        else:
             instance_id = utils.get_id_from_href(instance)
-
-            # verify that the instance exists and can perform actions
-            from trove.instance.models import Instance
-            instance_model = Instance.load(context, instance_id)
+            # Import here to avoid circular imports.
+            from trove.instance import models as inst_model
+            instance_model = inst_model.Instance.load(context, instance_id)
             instance_model.validate_can_perform_action()
-            cls.validate_can_perform_action(
-                instance_model, 'backup_create')
-
-            cls.verify_swift_auth_token(context)
-
             if instance_model.cluster_id is not None:
                 raise exception.ClusterInstanceOperationNotSupported()
 
+            cls.validate_can_perform_action(instance_model, 'backup_create')
+
+            cls.verify_swift_auth_token(context)
+
             ds = instance_model.datastore
             ds_version = instance_model.datastore_version
-            parent = None
-            last_backup_id = None
+
             if parent_id:
                 # Look up the parent info or fail early if not found or if
                 # the user does not have access to the parent.
@@ -100,36 +133,53 @@ class Backup(object):
                         'checksum': _parent.checksum
                     }
                     last_backup_id = _parent.id
+
+            if parent:
+                backup_type = constants.BACKUP_TYPE_INC
+
+        def _create_resources():
             try:
-                db_info = DBBackup.create(name=name,
-                                          description=description,
-                                          tenant_id=context.project_id,
-                                          state=BackupState.NEW,
-                                          instance_id=instance_id,
-                                          parent_id=parent_id or
-                                          last_backup_id,
-                                          datastore_version_id=ds_version.id,
-                                          deleted=False)
+                db_info = DBBackup.create(
+                    name=name,
+                    description=description,
+                    tenant_id=context.project_id,
+                    state=backup_state,
+                    instance_id=instance_id,
+                    parent_id=parent_id or last_backup_id,
+                    datastore_version_id=ds_version.id,
+                    deleted=False,
+                    location=location,
+                    checksum=checksum,
+                    backup_type=backup_type,
+                    size=size
+                )
             except exception.InvalidModelError as ex:
                 LOG.exception("Unable to create backup record for "
                               "instance: %s", instance_id)
                 raise exception.BackupCreationError(str(ex))
 
-            backup_info = {'id': db_info.id,
-                           'name': name,
-                           'description': description,
-                           'instance_id': instance_id,
-                           'backup_type': db_info.backup_type,
-                           'checksum': db_info.checksum,
-                           'parent': parent,
-                           'datastore': ds.name,
-                           'datastore_version': ds_version.name,
-                           'swift_container': swift_container
-                           }
-            api.API(context).create_backup(backup_info, instance_id)
+            if not restore_from:
+                backup_info = {
+                    'id': db_info.id,
+                    'name': name,
+                    'description': description,
+                    'instance_id': instance_id,
+                    'backup_type': db_info.backup_type,
+                    'checksum': db_info.checksum,
+                    'parent': parent,
+                    'datastore': ds.name,
+                    'datastore_version': ds_version.name,
+                    'swift_container': swift_container
+                }
+                api.API(context).create_backup(backup_info, instance_id)
+            else:
+                context.notification.payload.update(
+                    {'backup_id': db_info.id}
+                )
+
             return db_info
-        return run_with_quotas(context.project_id,
-                               {'backups': 1},
+
+        return run_with_quotas(context.project_id, {'backups': 1},
                                _create_resources)
 
     @classmethod
@@ -372,7 +422,7 @@ class DBBackup(DatabaseModelBase):
 
     @property
     def is_done_successfuly(self):
-        return self.state == BackupState.COMPLETED
+        return self.state in [BackupState.COMPLETED, BackupState.RESTORED]
 
     @property
     def filename(self):
diff --git a/trove/backup/service.py b/trove/backup/service.py
index ed783199..8b3d9d0f 100644
--- a/trove/backup/service.py
+++ b/trove/backup/service.py
@@ -80,27 +80,32 @@ class BackupController(wsgi.Controller):
         context = req.environ[wsgi.CONTEXT_KEY]
         policy.authorize_on_tenant(context, 'backup:create')
         data = body['backup']
-        instance = data['instance']
+        instance = data.get('instance')
         name = data['name']
         desc = data.get('description')
         parent = data.get('parent_id')
         incremental = data.get('incremental')
         swift_container = data.get('swift_container')
+        restore_from = data.get('restore_from')
 
-        context.notification = notification.DBaaSBackupCreate(context,
-                                                              request=req)
+        context.notification = notification.DBaaSBackupCreate(
+            context, request=req)
 
-        if not swift_container:
-            instance_id = utils.get_id_from_href(instance)
-            backup_strategy = BackupStrategy.get(context, instance_id)
-            if backup_strategy:
-                swift_container = backup_strategy.swift_container
+        if not restore_from:
+            if not instance:
+                raise exception.BackupCreationError('instance is missing.')
+            if not swift_container:
+                instance_id = utils.get_id_from_href(instance)
+                backup_strategy = BackupStrategy.get(context, instance_id)
+                if backup_strategy:
+                    swift_container = backup_strategy.swift_container
 
         with StartNotification(context, name=name, instance_id=instance,
                                description=desc, parent_id=parent):
             backup = Backup.create(context, instance, name, desc,
                                    parent_id=parent, incremental=incremental,
-                                   swift_container=swift_container)
+                                   swift_container=swift_container,
+                                   restore_from=restore_from)
 
         return wsgi.Result(views.BackupView(backup).data(), 202)
 
diff --git a/trove/backup/state.py b/trove/backup/state.py
index b0c46766..ce6e1588 100644
--- a/trove/backup/state.py
+++ b/trove/backup/state.py
@@ -21,6 +21,7 @@ class BackupState(object):
     SAVING = "SAVING"
     COMPLETED = "COMPLETED"
     FAILED = "FAILED"
+    RESTORED = "RESTORED"
     DELETE_FAILED = "DELETE_FAILED"
     RUNNING_STATES = [NEW, BUILDING, SAVING]
-    END_STATES = [COMPLETED, FAILED, DELETE_FAILED]
+    END_STATES = [COMPLETED, FAILED, DELETE_FAILED, RESTORED]
diff --git a/trove/common/apischema.py b/trove/common/apischema.py
index 4c799992..bc815fad 100644
--- a/trove/common/apischema.py
+++ b/trove/common/apischema.py
@@ -652,14 +652,27 @@ backup = {
         "properties": {
             "backup": {
                 "type": "object",
-                "required": ["instance", "name"],
+                "required": ["name"],
                 "properties": {
                     "description": non_empty_string,
                     "instance": uuid,
                     "name": non_empty_string,
                     "parent_id": uuid,
                     "incremental": boolean_string,
-                    "swift_container": non_empty_string
+                    "swift_container": non_empty_string,
+                    "restore_from": {
+                        "type": "object",
+                        "required": [
+                            "remote_location",
+                            "local_datastore_version_id",
+                            "size"
+                        ],
+                        "properties": {
+                            "remote_location": non_empty_string,
+                            "local_datastore_version_id": uuid,
+                            "size": {"type": "number"}
+                        }
+                    }
                 }
             }
         }
diff --git a/trove/common/constants.py b/trove/common/constants.py
new file mode 100644
index 00000000..0f53477b
--- /dev/null
+++ b/trove/common/constants.py
@@ -0,0 +1,16 @@
+# Copyright 2021 Catalyst Cloud Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+BACKUP_TYPE_FULL = 'full'
+BACKUP_TYPE_INC = 'incremental'
diff --git a/trove/common/swift.py b/trove/common/swift.py
new file mode 100644
index 00000000..43910af1
--- /dev/null
+++ b/trove/common/swift.py
@@ -0,0 +1,42 @@
+# Copyright 2021 Catalyst Cloud Ltd.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+def parse_location(location):
+    storage_url = "/".join(location.split('/')[:-2])
+    container_name = location.split('/')[-2]
+    object_name = location.split('/')[-1]
+    return storage_url, container_name, object_name
+
+
+def _get_attr(original):
+    """Get a friendly name from an object header key."""
+    key = original.replace('-', '_')
+    key = key.replace('x_object_meta_', '')
+    return key
+
+
+def get_metadata(client, location, extra_attrs=[]):
+    _, container_name, object_name = parse_location(location)
+    headers = client.head_object(container_name, object_name)
+
+    meta = {}
+    for key, value in headers.items():
+        if key.startswith('x-object-meta'):
+            meta[_get_attr(key)] = value
+
+    for key in extra_attrs:
+        meta[key] = headers.get(key)
+
+    return meta
diff --git a/trove/instance/models.py b/trove/instance/models.py
index b04e3ae6..e00bff0c 100644
--- a/trove/instance/models.py
+++ b/trove/instance/models.py
@@ -351,10 +351,10 @@ class SimpleInstance(object):
         - Then server status
         - Otherwise, unknown
         """
-        LOG.info(f"Getting instance status for {self.id}, "
-                 f"task status: {self.db_info.task_status}, "
-                 f"datastore status: {self.datastore_status.status}, "
-                 f"server status: {self.db_info.server_status}")
+        LOG.debug(f"Getting instance status for {self.id}, "
+                  f"task status: {self.db_info.task_status}, "
+                  f"datastore status: {self.datastore_status.status}, "
+                  f"server status: {self.db_info.server_status}")
 
         task_status = self.db_info.task_status
         server_status = self.db_info.server_status
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index e5e0026f..4acb9203 100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -1470,8 +1470,7 @@ class BackupTasks(object):
         def _delete(backup):
             backup.deleted = True
             backup.deleted_at = timeutils.utcnow()
-            # Set datastore_version_id to None so that datastore_version could
-            # be deleted.
+            # Set datastore_version_id to None to remove dependency.
             backup.datastore_version_id = None
             backup.save()
 
@@ -1479,7 +1478,9 @@ class BackupTasks(object):
         backup = bkup_models.Backup.get_by_id(context, backup_id)
         try:
             filename = backup.filename
-            if filename:
+            # Do not remove the object if the backup was restored from remote
+            # location.
+            if filename and backup.state != bkup_models.BackupState.RESTORED:
                 BackupTasks.delete_files_from_swift(context,
                                                     backup.container_name,
                                                     filename)
diff --git a/trove/tests/unittests/backup/test_backup_models.py b/trove/tests/unittests/backup/test_backup_models.py
index 2f97b451..ffa205dd 100644
--- a/trove/tests/unittests/backup/test_backup_models.py
+++ b/trove/tests/unittests/backup/test_backup_models.py
@@ -176,7 +176,7 @@ class BackupCreateTest(trove_testtools.TestCase):
                               BACKUP_NAME, BACKUP_DESC)
 
     def test_create_backup_swift_token_invalid(self):
-        instance = MagicMock()
+        instance = MagicMock(cluster_id=None)
         with patch.object(instance_models.BuiltInstance, 'load',
                           return_value=instance):
             instance.validate_can_perform_action = MagicMock(
@@ -191,7 +191,7 @@ class BackupCreateTest(trove_testtools.TestCase):
                                       BACKUP_NAME, BACKUP_DESC)
 
     def test_create_backup_datastore_operation_not_supported(self):
-        instance = MagicMock()
+        instance = MagicMock(cluster_id=None)
         with patch.object(instance_models.BuiltInstance, 'load',
                           return_value=instance):
             with patch.object(
diff --git a/trove/tests/unittests/backup/test_service.py b/trove/tests/unittests/backup/test_service.py
new file mode 100644
index 00000000..9be60d5f
--- /dev/null
+++ b/trove/tests/unittests/backup/test_service.py
@@ -0,0 +1,84 @@
+# Copyright 2021 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+from unittest import mock
+
+from trove.backup import service
+from trove.backup.state import BackupState
+from trove.common import context
+from trove.common import wsgi
+from trove.datastore import models as ds_models
+from trove.tests.unittests import trove_testtools
+from trove.tests.unittests.util import util
+
+
+class TestBackupController(trove_testtools.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        util.init_db()
+
+        cls.ds_name = cls.random_name('datastore',
+                                      prefix='TestBackupController')
+        ds_models.update_datastore(name=cls.ds_name, default_version=None)
+        cls.ds = ds_models.Datastore.load(cls.ds_name)
+
+        ds_models.update_datastore_version(
+            cls.ds_name, 'fake-ds-version', 'mysql', '', ['trove', 'mysql'],
+            '', 1)
+        cls.ds_version = ds_models.DatastoreVersion.load(
+            cls.ds, 'fake-ds-version')
+
+        cls.controller = service.BackupController()
+
+        super(TestBackupController, cls).setUpClass()
+
+    @classmethod
+    def tearDownClass(cls):
+        util.cleanup_db()
+        super(TestBackupController, cls).tearDownClass()
+
+    def setUp(self):
+        trove_testtools.patch_notifier(self)
+        self.context = context.TroveContext(project_id=self.random_uuid())
+
+        super(TestBackupController, self).setUp()
+
+    @mock.patch('trove.common.clients.create_swift_client')
+    def test_create_restore_from(self, mock_swift_client):
+        swift_client = mock.MagicMock()
+        swift_client.head_object.return_value = {'etag': 'fake-etag'}
+        mock_swift_client.return_value = swift_client
+
+        req = mock.MagicMock(environ={wsgi.CONTEXT_KEY: self.context})
+
+        name = self.random_name(
+            name='backup', prefix='TestBackupController')
+        body = {
+            'backup': {
+                "name": name,
+                "restore_from": {
+                    "remote_location": "http://192.168.206.8:8080/v1/"
+                                       "AUTH_055b2fb9a2264ae5a5f6b3cc066c4a1d/"
+                                       "fake-container/fake-object",
+                    "local_datastore_version_id": self.ds_version.id,
+                    "size": 0.2
+                }
+            }
+        }
+        ret = self.controller.create(req, body, self.context.project_id)
+        self.assertEqual(202, ret.status)
+
+        ret_backup = ret.data(None)['backup']
+
+        self.assertEqual(BackupState.RESTORED, ret_backup.get('status'))
+        self.assertEqual(name, ret_backup.get('name'))
diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py
index 353f82f4..b81194a3 100644
--- a/trove/tests/unittests/taskmanager/test_models.py
+++ b/trove/tests/unittests/taskmanager/test_models.py
@@ -1037,10 +1037,8 @@ class BackupTasksTest(trove_testtools.TestCase):
 
         self.assertTrue(self.backup.deleted)
 
-    @patch('trove.taskmanager.models.LOG')
     @patch('trove.common.clients.create_swift_client')
-    def test_delete_backup_fail_delete_manifest(self, mock_swift_client,
-                                                mock_logging):
+    def test_delete_backup_fail_delete_manifest(self, mock_swift_client):
         client_mock = MagicMock()
         client_mock.head_object.return_value = {}
         client_mock.delete_object.side_effect = ClientException("foo")
@@ -1070,6 +1068,12 @@ class BackupTasksTest(trove_testtools.TestCase):
         client_mock.delete_object.assert_called_once_with('container',
                                                           '12e48.xbstream.gz')
 
+    def test_delete_backup_restored(self):
+        self.backup.state = state.BackupState.RESTORED
+        taskmanager_models.BackupTasks.delete_backup(mock.ANY, self.backup.id)
+
+        self.assertTrue(self.backup.deleted)
+
     def test_parse_manifest(self):
         manifest = 'container/prefix'
         cont, prefix = taskmanager_models.BackupTasks._parse_manifest(manifest)