diff options
34 files changed, 664 insertions, 602 deletions
@@ -11,7 +11,6 @@ jobs: - openstack-tox-cover: voting: false - - openstack-tox-pylint - trove-tox-bandit-baseline: voting: false - trove-tempest: @@ -21,7 +20,6 @@ gate: queue: trove jobs: - - openstack-tox-pylint - trove-tempest: voting: false experimental: diff --git a/api-ref/source/backup-strategy.inc b/api-ref/source/backup-strategy.inc new file mode 100644 index 00000000..1f36f475 --- /dev/null +++ b/api-ref/source/backup-strategy.inc @@ -0,0 +1,111 @@ +.. -*- rst -*- + +=============== +Backup Strategy +=============== + +Backup strategy allows the user to customize the way of creating backups. Users +can create strategy either in the project scope or for a particular database +instance. + + +List backup strategies +~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: GET /v1.0/{project_id}/backup_strategies + +List backup strategies for a project. You can filter the results by +using query string parameters. The following filters are supported: + +- ``instance_id={instance_id}`` - Return the list of backup strategies for a + particular database instance. +- ``project_id={project_id}`` - Return the list of backup strategies for a + particular project, admin only. + +Normal response codes: 200 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - project_id: project_id + +Response Parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - backup_strategies: backup_strategy_list + - project_id: project_id + - instance_id: instanceId1 + - backend: backup_backend + - swift_container: swift_container_required + +Response Example +---------------- + +.. literalinclude:: samples/backup-strategy-list-response.json + :language: javascript + + +Create backup strategy +~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: POST /v1.0/{project_id}/backup_strategies + +Creates or updates backup strategy for the project or a database instance. + +Normal response codes: 202 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - project_id: project_id + - instance_id: instance_id_optional + - swift_container: swift_container_required + +Request Example +--------------- + +.. literalinclude:: samples/backup-strategy-create-request.json + :language: javascript + +Response Parameters +------------------- + +.. rest_parameters:: parameters.yaml + + - project_id: project_id + - instance_id: instanceId1 + - backend: backup_backend + - swift_container: swift_container_required + +Response Example +---------------- + +.. literalinclude:: samples/backup-strategy-create-response.json + :language: javascript + + +Delete database strategy +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. rest_method:: DELETE /v1.0/{project_id}/backup_strategies + +Deletes a database strategy for a project. If ``instance_id`` is specified in +the URL query parameters, delete the database strategy for that particular +database instance. Additionally, admin user is allowed to delete backup +strategy of other projects by specifying ``project_id`` in the URL query +parameters. + +Normal response codes: 202 + +Request +------- + +.. rest_parameters:: parameters.yaml + + - project_id: project_id
\ No newline at end of file diff --git a/api-ref/source/backups.inc b/api-ref/source/backups.inc index 9fd8d331..b7156a64 100644 --- a/api-ref/source/backups.inc +++ b/api-ref/source/backups.inc @@ -70,8 +70,8 @@ Create database backup Creates a database backup for instance. In the Trove deployment with service tenant enabled, The backup data is -stored as objects in OpenStack Swift service in the user's -container(``database_backups`` by default) +stored as objects in OpenStack Swift service in the user's container. If not +specified, the container name is defined by the cloud admin. Normal response codes: 202 @@ -86,6 +86,7 @@ Request - parent_id: backup_parentId - incremental: backup_incremental - description: backup_description + - swift_container: swift_container Request Example --------------- diff --git a/api-ref/source/index.rst b/api-ref/source/index.rst index ed988b90..e2fa2732 100755 --- a/api-ref/source/index.rst +++ b/api-ref/source/index.rst @@ -13,6 +13,7 @@ .. include:: instance-actions.inc .. include:: instance-logs.inc .. include:: backups.inc +.. include:: backup-strategy.inc .. include:: configurations.inc .. include:: databases.inc .. include:: users.inc diff --git a/api-ref/source/parameters.yaml b/api-ref/source/parameters.yaml index a685ce88..1edc9f7a 100755 --- a/api-ref/source/parameters.yaml +++ b/api-ref/source/parameters.yaml @@ -105,6 +105,12 @@ availability_zone: in: body required: false type: string +backup_backend: + description: | + The storage backend of instance backups, currently only swift is supported. + in: body + required: true + type: string backup_description: description: | An optional description for the backup. @@ -173,6 +179,12 @@ backup_status: in: body required: true type: string +backup_strategy_list: + description: | + A list of ``backup_strategy`` objects. + in: body + required: true + type: array characterSet: description: | A set of symbols and encodings. Default is @@ -417,6 +429,12 @@ instance_hostname: in: body require: false type: string +instance_id_optional: + description: | + The ID of the database instance. + in: body + required: false + type: string instance_ip_address: description: | The IP address of an instance(deprecated). @@ -738,6 +756,16 @@ slave_of: in: body required: false type: string +swift_container: + description: User defined swift container name. + in: body + required: false + type: string +swift_container_required: + description: User defined swift container name. + in: body + required: true + type: string tenant_id: description: | The ID of a tenant. diff --git a/api-ref/source/samples/backup-strategy-create-request.json b/api-ref/source/samples/backup-strategy-create-request.json new file mode 100644 index 00000000..7c30d9c8 --- /dev/null +++ b/api-ref/source/samples/backup-strategy-create-request.json @@ -0,0 +1,6 @@ +{ + "backup_strategy": { + "instance_id": "0602db72-c63d-11ea-b87c-00224d6b7bc1", + "swift_container": "my_trove_backups" + } +}
\ No newline at end of file diff --git a/api-ref/source/samples/backup-strategy-create-response.json b/api-ref/source/samples/backup-strategy-create-response.json new file mode 100644 index 00000000..a6c3ac6c --- /dev/null +++ b/api-ref/source/samples/backup-strategy-create-response.json @@ -0,0 +1,8 @@ +{ + "backup_strategy": { + "project_id": "922b47766bcb448f83a760358337f2b4", + "instance_id": "0602db72-c63d-11ea-b87c-00224d6b7bc1", + "backend": "swift", + "swift_container": "my_trove_backups" + } +}
\ No newline at end of file diff --git a/api-ref/source/samples/backup-strategy-list-response.json b/api-ref/source/samples/backup-strategy-list-response.json new file mode 100644 index 00000000..98ee13ce --- /dev/null +++ b/api-ref/source/samples/backup-strategy-list-response.json @@ -0,0 +1,10 @@ +{ + "backup_strategies": [ + { + "backend": "swift", + "instance_id": "0602db72-c63d-11ea-b87c-00224d6b7bc1", + "project_id": "922b47766bcb448f83a760358337f2b4", + "swift_container": "my_trove_backups" + } + ] +}
\ No newline at end of file diff --git a/backup/main.py b/backup/main.py index 8e24478e..c52becbf 100644 --- a/backup/main.py +++ b/backup/main.py @@ -92,7 +92,8 @@ def stream_backup_to_storage(runner_cls, storage): with runner_cls(filename=CONF.backup_id, **parent_metadata) as bkup: checksum, location = storage.save( bkup, - metadata=CONF.swift_extra_metadata + metadata=CONF.swift_extra_metadata, + container=CONF.swift_container ) LOG.info('Backup successfully, checksum: %s, location: %s', checksum, location) diff --git a/doc/source/user/backup-db-incremental.rst b/doc/source/user/backup-db-incremental.rst deleted file mode 100644 index 5c5b8304..00000000 --- a/doc/source/user/backup-db-incremental.rst +++ /dev/null @@ -1,130 +0,0 @@ -======================= -Use incremental backups -======================= - -Incremental backups let you chain together a series of backups. You -start with a regular backup. Then, when you want to create a subsequent -incremental backup, you specify the parent backup. - -Restoring a database instance from an incremental backup is the same as -creating a database instance from a regular backup—the Database service -handles the complexities of applying the chain of incremental backups. - -The artifacts created by backup are stored in OpenStack Swift, by default in a -container named 'database_backups'. As the end user, you are able to access all -the objects but make sure not to delete those objects manually. When a backup -is deleted in Trove, the related objects are automatically removed from Swift. - -.. caution:: - - If the objects in 'database_backups' container are deleted manually, the - database can't be properly restored. - -This example shows you how to use incremental backups with a MySQL -database. - -**Assumptions.** Assume that you have created a regular -backup for the following database instance: - -- Instance name: ``guest1`` - -- ID of the instance (``INSTANCE_ID``): - ``792a6a56-278f-4a01-9997-d997fa126370`` - -- ID of the regular backup artifact (``BACKUP_ID``): - ``6dc3a9b7-1f3e-4954-8582-3f2e4942cddd`` - -Create and use incremental backups -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -#. **Create your first incremental backup** - - Use the :command:`openstack database backup create` command and specify: - - - The ``INSTANCE_ID`` of the database instance you are doing the - incremental backup for (in this example, - ``792a6a56-278f-4a01-9997-d997fa126370``) - - - The name of the incremental backup you are creating: ``backup1.1`` - - - The ``BACKUP_ID`` of the parent backup. In this case, the parent - is the regular backup, with an ID of - ``6dc3a9b7-1f3e-4954-8582-3f2e4942cddd`` - - .. code-block:: console - - $ openstack database backup create INSTANCE_ID backup1.1 --parent BACKUP_ID - +-------------+--------------------------------------+ - | Property | Value | - +-------------+--------------------------------------+ - | created | 2014-03-19T14:09:13 | - | description | None | - | id | 1d474981-a006-4f62-b25f-43d7b8a7097e | - | instance_id | 792a6a56-278f-4a01-9997-d997fa126370 | - | locationRef | None | - | name | backup1.1 | - | parent_id | 6dc3a9b7-1f3e-4954-8582-3f2e4942cddd | - | size | None | - | status | NEW | - | updated | 2014-03-19T14:09:13 | - +-------------+--------------------------------------+ - - Note that this command returns both the ID of the database instance - you are incrementally backing up (``instance_id``) and a new ID for - the new incremental backup artifact you just created (``id``). - -#. **Create your second incremental backup** - - The name of your second incremental backup is ``backup1.2``. This - time, when you specify the parent, pass in the ID of the incremental - backup you just created in the previous step (``backup1.1``). In this - example, it is ``1d474981-a006-4f62-b25f-43d7b8a7097e``. - - .. code-block:: console - - $ openstack database backup create INSTANCE_ID backup1.2 --parent BACKUP_ID - +-------------+--------------------------------------+ - | Property | Value | - +-------------+--------------------------------------+ - | created | 2014-03-19T14:09:13 | - | description | None | - | id | bb84a240-668e-49b5-861e-6a98b67e7a1f | - | instance_id | 792a6a56-278f-4a01-9997-d997fa126370 | - | locationRef | None | - | name | backup1.2 | - | parent_id | 1d474981-a006-4f62-b25f-43d7b8a7097e | - | size | None | - | status | NEW | - | updated | 2014-03-19T14:09:13 | - +-------------+--------------------------------------+ - -#. **Restore using incremental backups** - - Now assume that your ``guest1`` database instance is damaged and you - need to restore it from your incremental backups. In this example, - you use the :command:`openstack database instance create` command to create a new database - instance called ``guest2``. - - To incorporate your incremental backups, you simply use the - `--backup`` parameter to pass in the ``BACKUP_ID`` of your most - recent incremental backup. The Database service handles the - complexities of applying the chain of all previous incremental - backups. - - .. code-block:: console - - $ openstack database instance create guest2 10 --size 1 --nic net-id=$network_id --backup BACKUP_ID - +-------------------+-----------------------------------------------------------+ - | Property | Value | - +-------------------+-----------------------------------------------------------+ - | created | 2014-03-19T14:10:56 | - | datastore | {u'version': u'mysql-5.5', u'type': u'mysql'} | - | datastore_version | mysql-5.5 | - | flavor | {u'id': u'10'} | - | id | a3680953-eea9-4cf2-918b-5b8e49d7e1b3 | - | name | guest2 | - | status | BUILD | - | updated | 2014-03-19T14:10:56 | - | volume | {u'size': 1} | - +-------------------+-----------------------------------------------------------+ - diff --git a/doc/source/user/backup-db.rst b/doc/source/user/backup-db.rst index fa7e45ba..35bb8782 100644 --- a/doc/source/user/backup-db.rst +++ b/doc/source/user/backup-db.rst @@ -3,64 +3,67 @@ Backup and restore a database ============================= You can use Database services to backup a database and store the backup -artifact in the Object Storage service. Later on, if the original -database is damaged, you can use the backup artifact to restore the -database. The restore process creates a database instance. +artifact in the Object Storage service. Later on, if the original database is +damaged, you can use the backup artifact to restore the database. The restore +process creates a new database instance. -The artifacts created by backup are stored in OpenStack Swift, by default in a -container named 'database_backups'. As the end user, you are able to access all -the objects but make sure not to delete those objects manually. When a backup -is deleted in Trove, the related objects are automatically removed from Swift. +The backup data is stored in OpenStack Swift, the user is able to customize +which container to store the data. The following ways are described in the +order of precedence from greatest to least: -.. caution:: +1. The container name can be specified when creating backups, this could + override either the backup strategy setting or the default setting in Trove + configuration. - If the objects in 'database_backups' container are deleted manually, the - database can't be properly restored. +2. Users could create backup strategy either for the project scope or for a + particular instance. -This example shows you how to back up and restore a MySQL database. +3. If not configured by the end user, will use the default value in Trove + configuration. -#. **Backup the database instance** +.. caution:: - As background, assume that you have created a database - instance with the following - characteristics: + If the objects in the backup container are manually deleted, the + database can't be properly restored. - - Name of the database instance: ``guest1`` +This example shows you how to create backup strategy, create backup and restore +instance from the backup. - - Flavor ID: ``10`` +#. **Before creating backup** - - Root volume size: ``2`` + 1. Make sure you have created an instance, e.g. in this example, we use the following instance: - - Databases: ``db1`` and ``db2`` + .. code-block:: console - - Users: The ``user1`` user with the ``password`` password + $ openstack database instance list + +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ + | id | name | datastore | datastore_version | status | flavor_id | size | + +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ + | 97b4b853-80f6-414f-ba6f-c6f455a79ae6 | guest1 | mysql | mysql-5.5 | ACTIVE | 10 | 2 | + +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ - First, get the ID of the ``guest1`` database instance by using the - :command:`openstack database instance list` command: + 2. Optionally, create a backup strategy for the instance. You can also specify a different swift container name (``--swift-container``) when creating the backup. - .. code-block:: console + .. code-block:: console - $ openstack database instance list - +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ - | id | name | datastore | datastore_version | status | flavor_id | size | - +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ - | 97b4b853-80f6-414f-ba6f-c6f455a79ae6 | guest1 | mysql | mysql-5.5 | ACTIVE | 10 | 2 | - +--------------------------------------+--------+-----------+-------------------+--------+-----------+------+ - - Back up the database instance by using the :command:`openstack database backup create` - command. In this example, the backup is called ``backup1``. In this - example, replace ``INSTANCE_ID`` with - ``97b4b853-80f6-414f-ba6f-c6f455a79ae6``: + $ openstack database backup strategy create --instance-id 97b4b853-80f6-414f-ba6f-c6f455a79ae6 --swift-container my-trove-backups + +-----------------+--------------------------------------+ + | Field | Value | + +-----------------+--------------------------------------+ + | backend | swift | + | instance_id | 97b4b853-80f6-414f-ba6f-c6f455a79ae6 | + | project_id | 922b47766bcb448f83a760358337f2b4 | + | swift_container | my-trove-backups | + +-----------------+--------------------------------------+ - .. note:: +#. **Backup the database instance** - This command syntax pertains only to python-troveclient version - 1.0.6 and later. Earlier versions require you to pass in the backup - name as the first argument. + Back up the database instance by using the :command:`openstack database backup create` + command. In this example, the backup is called ``backup1``. .. code-block:: console - $ openstack database backup create INSTANCE_ID backup1 + $ openstack database backup create 97b4b853-80f6-414f-ba6f-c6f455a79ae6 backup1 +-------------+--------------------------------------+ | Property | Value | +-------------+--------------------------------------+ @@ -76,11 +79,9 @@ This example shows you how to back up and restore a MySQL database. | updated | 2014-03-18T17:09:07 | +-------------+--------------------------------------+ - Note that the command returns both the ID of the original instance - (``instance_id``) and the ID of the backup artifact (``id``). - - Later on, use the :command:`openstack database backup list` command to get this - information: + Later on, use either :command:`openstack database backup list` command or + :command:`openstack database backup show` command to check the backup + status: .. code-block:: console @@ -90,14 +91,7 @@ This example shows you how to back up and restore a MySQL database. +--------------------------------------+--------------------------------------+---------+-----------+-----------+---------------------+ | 8af30763-61fd-4aab-8fe8-57d528911138 | 97b4b853-80f6-414f-ba6f-c6f455a79ae6 | backup1 | COMPLETED | None | 2014-03-18T17:09:11 | +--------------------------------------+--------------------------------------+---------+-----------+-----------+---------------------+ - - You can get additional information about the backup by using the - :command:`openstack database backup show` command and passing in the ``BACKUP_ID``, - which is ``8af30763-61fd-4aab-8fe8-57d528911138``. - - .. code-block:: console - - $ openstack database backup show BACKUP_ID + $ openstack database backup show 8af30763-61fd-4aab-8fe8-57d528911138 +-------------+----------------------------------------------------+ | Property | Value | +-------------+----------------------------------------------------+ @@ -113,17 +107,36 @@ This example shows you how to back up and restore a MySQL database. | updated | 2014-03-18T17:09:11 | +-------------+----------------------------------------------------+ +#. **Check the backup data in Swift** + + Check the container is created and the backup data is saved as objects inside the container. + + .. code-block:: console + + $ openstack container list + +------------------+ + | Name | + +------------------+ + | my-trove-backups | + +------------------+ + $ openstack object list my-trove-backups + +--------------------------------------------------+ + | Name | + +--------------------------------------------------+ + | 8af30763-61fd-4aab-8fe8-57d528911138.xbstream.gz | + +--------------------------------------------------+ + #. **Restore a database instance** - Now assume that your ``guest1`` database instance is damaged and you + Now assume that the ``guest1`` database instance is damaged and you need to restore it. In this example, you use the :command:`openstack database instance create` command to create a new database instance called ``guest2``. - - You specify that the new ``guest2`` instance has the same flavor + - Specify that the new ``guest2`` instance has the same flavor (``10``) and the same root volume size (``2``) as the original ``guest1`` instance. - - You use the ``--backup`` argument to indicate that this new + - Use the ``--backup`` argument to indicate that this new instance is based on the backup artifact identified by ``BACKUP_ID``. In this example, replace ``BACKUP_ID`` with ``8af30763-61fd-4aab-8fe8-57d528911138``. @@ -233,3 +246,33 @@ This example shows you how to back up and restore a MySQL database. $ openstack database instance delete INSTANCE_ID +Create incremental backups +-------------------------- + +Incremental backups let you chain together a series of backups. You start with +a regular backup. Then, when you want to create a subsequent incremental +backup, you specify the parent backup. + +Restoring a database instance from an incremental backup is the same as +creating a database instance from a regular backup. the Database service +handles the process of applying the chain of incremental backups. + +Create an incremental backup based on a parent backup: + +.. code-block:: console + + $ openstack database backup create INSTANCE_ID backup1.1 --parent BACKUP_ID + +-------------+--------------------------------------+ + | Property | Value | + +-------------+--------------------------------------+ + | created | 2014-03-19T14:09:13 | + | description | None | + | id | 1d474981-a006-4f62-b25f-43d7b8a7097e | + | instance_id | 792a6a56-278f-4a01-9997-d997fa126370 | + | locationRef | None | + | name | backup1.1 | + | parent_id | 6dc3a9b7-1f3e-4954-8582-3f2e4942cddd | + | size | None | + | status | NEW | + | updated | 2014-03-19T14:09:13 | + +-------------+--------------------------------------+ diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index c26a3af4..0aee563f 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -14,7 +14,6 @@ handling complex administrative tasks. create-db.rst manage-db-and-users.rst backup-db.rst - backup-db-incremental.rst manage-db-config.rst set-up-replication.rst upgrade-datastore.rst diff --git a/releasenotes/notes/victoria-support-backup-strategy.yaml b/releasenotes/notes/victoria-support-backup-strategy.yaml new file mode 100644 index 00000000..97b9335c --- /dev/null +++ b/releasenotes/notes/victoria-support-backup-strategy.yaml @@ -0,0 +1,6 @@ +--- +features: + - The user can create backup strategy to define the configurations for + creating backups, e.g. the swift container to store the backup data. Users + can also specify the container name when creating backups which takes + precedence over the backup strategy configuration. @@ -1,5 +1,5 @@ [tox] -envlist = py37,pep8,cover,api-ref,releasenotes,bandit,fakemodetests,pylint +envlist = py37,pep8,cover,api-ref,releasenotes,bandit,fakemodetests minversion = 2.0 skipsdist = True diff --git a/trove/backup/models.py b/trove/backup/models.py index 38c847f6..95d8683f 100644 --- a/trove/backup/models.py +++ b/trove/backup/models.py @@ -23,8 +23,8 @@ from trove.backup.state import BackupState from trove.common import cfg from trove.common import clients from trove.common import exception -from trove.common.i18n import _ from trove.common import utils +from trove.common.i18n import _ from trove.datastore import models as datastore_models from trove.db.models import DatabaseModelBase from trove.quota.quota import run_with_quotas @@ -49,7 +49,7 @@ class Backup(object): @classmethod def create(cls, context, instance, name, description=None, - parent_id=None, incremental=False): + parent_id=None, incremental=False, swift_container=None): """ create db record for Backup :param cls: @@ -60,6 +60,7 @@ class Backup(object): :param parent_id: :param incremental: flag to indicate incremental backup based on previous backup + :param swift_container: Swift container name. :return: """ @@ -73,7 +74,9 @@ class Backup(object): instance_model.validate_can_perform_action() cls.validate_can_perform_action( instance_model, 'backup_create') + cls.verify_swift_auth_token(context) + if instance_model.cluster_id is not None: raise exception.ClusterInstanceOperationNotSupported() @@ -121,6 +124,7 @@ class Backup(object): 'parent': parent, 'datastore': ds.name, 'datastore_version': ds_version.name, + 'swift_container': swift_container } api.API(context).create_backup(backup_info, instance_id) return db_info @@ -295,8 +299,55 @@ class Backup(object): raise exception.SwiftConnectionError() +class BackupStrategy(object): + @classmethod + def create(cls, context, instance_id, swift_container): + try: + existing = DBBackupStrategy.find_by(tenant_id=context.project_id, + instance_id=instance_id) + existing.swift_container = swift_container + existing.save() + return existing + except exception.NotFound: + return DBBackupStrategy.create( + tenant_id=context.project_id, + instance_id=instance_id, + backend='swift', + swift_container=swift_container, + ) + + @classmethod + def list(cls, context, tenant_id, instance_id=None): + kwargs = {'tenant_id': tenant_id} + if instance_id: + kwargs['instance_id'] = instance_id + result = DBBackupStrategy.find_by_filter(**kwargs) + return result + + @classmethod + def get(cls, context, instance_id): + try: + return DBBackupStrategy.find_by(tenant_id=context.project_id, + instance_id=instance_id) + except exception.NotFound: + try: + return DBBackupStrategy.find_by(tenant_id=context.project_id, + instance_id='') + except exception.NotFound: + return None + + @classmethod + def delete(cls, context, tenant_id, instance_id): + try: + existing = DBBackupStrategy.find_by(tenant_id=tenant_id, + instance_id=instance_id) + existing.delete() + except exception.NotFound: + pass + + def persisted_models(): - return {'backups': DBBackup} + return {'backups': DBBackup, 'backup_strategy': DBBackupStrategy} class DBBackup(DatabaseModelBase): @@ -332,6 +383,13 @@ class DBBackup(DatabaseModelBase): return None @property + def container_name(self): + if self.location: + return self.location.split('/')[-2] + else: + return None + + @property def datastore(self): if self.datastore_version_id: return datastore_models.Datastore.load( @@ -366,3 +424,9 @@ class DBBackup(DatabaseModelBase): return False else: raise exception.SwiftAuthError(tenant_id=context.project_id) + + +class DBBackupStrategy(DatabaseModelBase): + """A table for backup strategy records.""" + _data_fields = ['tenant_id', 'instance_id', 'backend', 'swift_container'] + _table_name = 'backup_strategy' diff --git a/trove/backup/service.py b/trove/backup/service.py index 6e8f3bf7..095ee9c9 100644 --- a/trove/backup/service.py +++ b/trove/backup/service.py @@ -16,14 +16,17 @@ from oslo_log import log as logging from oslo_utils import strutils -from trove.backup.models import Backup from trove.backup import views +from trove.backup.models import Backup +from trove.backup.models import BackupStrategy from trove.common import apischema +from trove.common import exception from trove.common import notification -from trove.common.notification import StartNotification from trove.common import pagination from trove.common import policy +from trove.common import utils from trove.common import wsgi +from trove.common.notification import StartNotification LOG = logging.getLogger(__name__) @@ -80,12 +83,23 @@ class BackupController(wsgi.Controller): desc = data.get('description') parent = data.get('parent_id') incremental = data.get('incremental') + swift_container = data.get('swift_container') + context.notification = notification.DBaaSBackupCreate(context, request=req) + + if not swift_container: + instance_id = utils.get_id_from_href(instance) + backup_strategy = BackupStrategy.get(context, instance_id) + if backup_strategy: + swift_container = backup_strategy.swift_container + with StartNotification(context, name=name, instance_id=instance, description=desc, parent_id=parent): backup = Backup.create(context, instance, name, desc, - parent_id=parent, incremental=incremental) + parent_id=parent, incremental=incremental, + swift_container=swift_container) + return wsgi.Result(views.BackupView(backup).data(), 202) def delete(self, req, tenant_id, id): @@ -101,3 +115,56 @@ class BackupController(wsgi.Controller): with StartNotification(context, backup_id=id): Backup.delete(context, id) return wsgi.Result(None, 202) + + +class BackupStrategyController(wsgi.Controller): + schemas = apischema.backup_strategy + + def create(self, req, body, tenant_id): + LOG.info("Creating or updating a backup strategy for tenant %s, " + "body: %s", tenant_id, body) + context = req.environ[wsgi.CONTEXT_KEY] + policy.authorize_on_tenant(context, 'backup_strategy:create') + data = body['backup_strategy'] + + instance_id = data.get('instance_id', '') + swift_container = data.get('swift_container') + + backup_strategy = BackupStrategy.create(context, instance_id, + swift_container) + return wsgi.Result( + views.BackupStrategyView(backup_strategy).data(), 202) + + def index(self, req, tenant_id): + context = req.environ[wsgi.CONTEXT_KEY] + instance_id = req.GET.get('instance_id') + tenant_id = req.GET.get('project_id', context.project_id) + LOG.info("Listing backup strateies for tenant %s", tenant_id) + + if tenant_id != context.project_id and not context.is_admin: + raise exception.TroveOperationAuthError( + tenant_id=context.project_id + ) + policy.authorize_on_tenant(context, 'backup_strategy:index') + + result = BackupStrategy.list(context, tenant_id, + instance_id=instance_id) + view = views.BackupStrategiesView(result) + return wsgi.Result(view.data(), 200) + + def delete(self, req, tenant_id): + context = req.environ[wsgi.CONTEXT_KEY] + instance_id = req.GET.get('instance_id', '') + tenant_id = req.GET.get('project_id', context.project_id) + LOG.info('Deleting backup strategies for tenant %s, instance_id=%s', + tenant_id, instance_id) + + if tenant_id != context.project_id and not context.is_admin: + raise exception.TroveOperationAuthError( + tenant_id=context.project_id + ) + policy.authorize_on_tenant(context, 'backup_strategy:delete') + + BackupStrategy.delete(context, tenant_id, instance_id) + + return wsgi.Result(None, 202) diff --git a/trove/backup/views.py b/trove/backup/views.py index 9e1cfd2b..c4b1ad8f 100644 --- a/trove/backup/views.py +++ b/trove/backup/views.py @@ -54,3 +54,31 @@ class BackupViews(object): for b in self.backups: backups.append(BackupView(b).data()["backup"]) return {"backups": backups} + + +class BackupStrategyView(object): + def __init__(self, backup_strategy): + self.backup_strategy = backup_strategy + + def data(self): + result = { + "backup_strategy": { + "project_id": self.backup_strategy.tenant_id, + "instance_id": self.backup_strategy.instance_id, + 'backend': self.backup_strategy.backend, + "swift_container": self.backup_strategy.swift_container, + } + } + return result + + +class BackupStrategiesView(object): + def __init__(self, backup_strategies): + self.backup_strategies = backup_strategies + + def data(self): + backup_strategies = [] + for item in self.backup_strategies: + backup_strategies.append( + BackupStrategyView(item).data()["backup_strategy"]) + return {"backup_strategies": backup_strategies} diff --git a/trove/common/api.py b/trove/common/api.py index 0bfdf0d9..dbef09d3 100644 --- a/trove/common/api.py +++ b/trove/common/api.py @@ -15,6 +15,7 @@ import routes from trove.backup.service import BackupController +from trove.backup.service import BackupStrategyController from trove.cluster.service import ClusterController from trove.common import wsgi from trove.configuration.service import ConfigurationsController @@ -37,6 +38,7 @@ class API(wsgi.Router): self._versions_router(mapper) self._limits_router(mapper) self._backups_router(mapper) + self._backup_strategy_router(mapper) self._configurations_router(mapper) self._modules_router(mapper) @@ -192,6 +194,21 @@ class API(wsgi.Router): action="delete", conditions={'method': ['DELETE']}) + def _backup_strategy_router(self, mapper): + backup_strategy_resource = BackupStrategyController().create_resource() + mapper.connect("/{tenant_id}/backup_strategies", + controller=backup_strategy_resource, + action="create", + conditions={'method': ['POST']}) + mapper.connect("/{tenant_id}/backup_strategies", + controller=backup_strategy_resource, + action="index", + conditions={'method': ['GET']}) + mapper.connect("/{tenant_id}/backup_strategies", + controller=backup_strategy_resource, + action="delete", + conditions={'method': ['DELETE']}) + def _modules_router(self, mapper): modules_resource = ModuleController().create_resource() diff --git a/trove/common/apischema.py b/trove/common/apischema.py index 2a7b08c8..0e3d60b1 100644 --- a/trove/common/apischema.py +++ b/trove/common/apischema.py @@ -601,13 +601,33 @@ backup = { "instance": uuid, "name": non_empty_string, "parent_id": uuid, - "incremental": boolean_string + "incremental": boolean_string, + "swift_container": non_empty_string } } } } } +backup_strategy = { + "create": { + "name": "backup_strategy:create", + "type": "object", + "required": ["backup_strategy"], + "properties": { + "backup_strategy": { + "type": "object", + "additionalProperties": False, + "required": ["swift_container"], + "properties": { + "instance_id": uuid, + "swift_container": non_empty_string + } + } + }, + } +} + guest_log = { "action": { "name": "guest_log:action", diff --git a/trove/common/cfg.py b/trove/common/cfg.py index 609aa19b..2be70056 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -471,7 +471,7 @@ common_opts = [ help='Key (OpenSSL aes_cbc) for instance RPC encryption.'), cfg.StrOpt('database_service_uid', default='1001', help='The UID(GID) of database service user.'), - cfg.StrOpt('backup_docker_image', default='openstacktrove/db-backup:1.0.0', + cfg.StrOpt('backup_docker_image', default='openstacktrove/db-backup:1.0.1', help='The docker image used for backup and restore.'), cfg.ListOpt('reserved_network_cidrs', default=[], help='Network CIDRs reserved for Trove guest instance ' diff --git a/trove/common/exception.py b/trove/common/exception.py index c51a53b9..6787a204 100644 --- a/trove/common/exception.py +++ b/trove/common/exception.py @@ -457,6 +457,12 @@ class BackupDatastoreMismatchError(TroveError): " datastore of %(datastore2)s.") +class BackupTooLarge(TroveError): + message = _("Backup is too large for given flavor or volume. " + "Backup size: %(backup_size)s GBs. " + "Available size: %(disk_size)s GBs.") + + class ReplicaCreateWithUsersDatabasesError(TroveError): message = _("Cannot create a replica with users or databases.") @@ -688,12 +694,6 @@ class ClusterDatastoreNotSupported(TroveError): "%(datastore)s-%(datastore_version)s.") -class BackupTooLarge(TroveError): - message = _("Backup is too large for given flavor or volume. " - "Backup size: %(backup_size)s GBs. " - "Available size: %(disk_size)s GBs.") - - class ImageNotFound(NotFound): message = _("Image %(uuid)s cannot be found.") diff --git a/trove/common/policies/backups.py b/trove/common/policies/backups.py index 312e4950..22525ad4 100644 --- a/trove/common/policies/backups.py +++ b/trove/common/policies/backups.py @@ -12,7 +12,7 @@ from oslo_policy import policy -from trove.common.policies.base import PATH_BACKUPS, PATH_BACKUP +from trove.common.policies import base rules = [ policy.DocumentedRuleDefault( @@ -21,7 +21,7 @@ rules = [ description='Create a backup of a database instance.', operations=[ { - 'path': PATH_BACKUPS, + 'path': base.PATH_BACKUPS, 'method': 'POST' } ]), @@ -31,7 +31,7 @@ rules = [ description='Delete a backup of a database instance.', operations=[ { - 'path': PATH_BACKUP, + 'path': base.PATH_BACKUP, 'method': 'DELETE' } ]), @@ -41,7 +41,7 @@ rules = [ description='List all backups.', operations=[ { - 'path': PATH_BACKUPS, + 'path': base.PATH_BACKUPS, 'method': 'GET' } ]), @@ -51,7 +51,7 @@ rules = [ description='List backups for all the projects.', operations=[ { - 'path': PATH_BACKUPS, + 'path': base.PATH_BACKUPS, 'method': 'GET' } ]), @@ -61,10 +61,40 @@ rules = [ description='Get informations of a backup.', operations=[ { - 'path': PATH_BACKUP, + 'path': base.PATH_BACKUP, 'method': 'GET' } - ]) + ]), + policy.DocumentedRuleDefault( + name='backup_strategy:create', + check_str='rule:admin_or_owner', + description='Create a backup strategy.', + operations=[ + { + 'path': base.PATH_BACKUP_STRATEGIES, + 'method': 'POST' + } + ]), + policy.DocumentedRuleDefault( + name='backup_strategy:index', + check_str='rule:admin_or_owner', + description='List all backup strategies.', + operations=[ + { + 'path': base.PATH_BACKUP_STRATEGIES, + 'method': 'GET' + } + ]), + policy.DocumentedRuleDefault( + name='backup_strategy:delete', + check_str='rule:admin_or_owner', + description='Delete backup strategies.', + operations=[ + { + 'path': base.PATH_BACKUP_STRATEGIES, + 'method': 'DELETE' + } + ]), ] diff --git a/trove/common/policies/base.py b/trove/common/policies/base.py index 94d48312..6655f7c5 100644 --- a/trove/common/policies/base.py +++ b/trove/common/policies/base.py @@ -33,6 +33,8 @@ PATH_CLUSTER_INSTANCE = PATH_CLUSTER_INSTANCES + '/{instance}' PATH_BACKUPS = PATH_BASE + '/backups' PATH_BACKUP = PATH_BACKUPS + '/{backup}' +PATH_BACKUP_STRATEGIES = PATH_BASE + '/backup_strategies' + PATH_CONFIGS = PATH_BASE + '/configurations' PATH_CONFIG = PATH_CONFIGS + '/{config}' diff --git a/trove/common/strategies/storage/__init__.py b/trove/common/strategies/storage/__init__.py deleted file mode 100644 index 8a458a93..00000000 --- a/trove/common/strategies/storage/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -from oslo_log import log as logging - -from trove.common.strategies.strategy import Strategy - -LOG = logging.getLogger(__name__) - - -def get_storage_strategy(storage_driver, ns=__name__): - return Strategy.get_strategy(storage_driver, ns) diff --git a/trove/common/strategies/storage/base.py b/trove/common/strategies/storage/base.py deleted file mode 100644 index b349af55..00000000 --- a/trove/common/strategies/storage/base.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import abc -from trove.common.strategies.strategy import Strategy - - -class Storage(Strategy): - """Base class for Storage Strategy implementation.""" - __strategy_type__ = 'storage' - __strategy_ns__ = 'trove.common.strategies.storage' - - def __init__(self, context): - self.context = context - super(Storage, self).__init__() - - @abc.abstractmethod - def save(self, filename, stream, metadata=None): - """Persist information from the stream.""" - - @abc.abstractmethod - def load(self, location, backup_checksum): - """Load a stream from a persisted storage location.""" - - @abc.abstractmethod - def load_metadata(self, location, backup_checksum): - """Load metadata for a persisted object.""" - - @abc.abstractmethod - def save_metadata(self, location, metadata={}): - """Save metadata for a persisted object.""" diff --git a/trove/common/strategies/storage/swift.py b/trove/common/strategies/storage/swift.py deleted file mode 100644 index f2f78523..00000000 --- a/trove/common/strategies/storage/swift.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright 2013 Hewlett-Packard Development Company, L.P. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# - -import hashlib -import json - -from oslo_log import log as logging -import six - -from trove.common import cfg -from trove.common.clients import create_swift_client -from trove.common.i18n import _ -from trove.common.strategies.storage import base - -LOG = logging.getLogger(__name__) -CONF = cfg.CONF - -CHUNK_SIZE = CONF.backup_chunk_size -MAX_FILE_SIZE = CONF.backup_segment_max_size -BACKUP_CONTAINER = CONF.backup_swift_container - - -class DownloadError(Exception): - """Error running the Swift Download Command.""" - - -class SwiftDownloadIntegrityError(Exception): - """Integrity error while running the Swift Download Command.""" - - -class StreamReader(object): - """Wrap the stream from the backup process and chunk it into segements.""" - - def __init__(self, stream, filename, max_file_size=MAX_FILE_SIZE): - self.stream = stream - self.filename = filename - self.container = BACKUP_CONTAINER - self.max_file_size = max_file_size - self.segment_length = 0 - self.process = None - self.file_number = 0 - self.end_of_file = False - self.end_of_segment = False - self.segment_checksum = hashlib.md5() - - @property - def base_filename(self): - """Filename with extensions removed.""" - return self.filename.split('.')[0] - - @property - def segment(self): - return '%s_%08d' % (self.base_filename, self.file_number) - - @property - def first_segment(self): - return '%s_%08d' % (self.base_filename, 0) - - @property - def segment_path(self): - return '%s/%s' % (self.container, self.segment) - - def read(self, chunk_size=CHUNK_SIZE): - if self.end_of_segment: - self.segment_length = 0 - self.segment_checksum = hashlib.md5() - self.end_of_segment = False - - # Upload to a new file if we are starting or too large - if self.segment_length > (self.max_file_size - chunk_size): - self.file_number += 1 - self.end_of_segment = True - return '' - - chunk = self.stream.read(chunk_size) - if not chunk: - self.end_of_file = True - return '' - - self.segment_checksum.update(chunk) - self.segment_length += len(chunk) - return chunk - - -class SwiftStorage(base.Storage): - """Implementation of Storage Strategy for Swift.""" - __strategy_name__ = 'swift' - - def __init__(self, *args, **kwargs): - super(SwiftStorage, self).__init__(*args, **kwargs) - self.connection = create_swift_client(self.context) - - def save(self, filename, stream, metadata=None): - """Persist information from the stream to swift. - - The file is saved to the location <BACKUP_CONTAINER>/<filename>. - It will be a Swift Static Large Object (SLO). - The filename is defined on the backup runner manifest property - which is typically in the format '<backup_id>.<ext>.gz' - """ - - LOG.info('Saving %(filename)s to %(container)s in swift.', - {'filename': filename, 'container': BACKUP_CONTAINER}) - - # Create the container if it doesn't already exist - LOG.debug('Creating container %s.', BACKUP_CONTAINER) - self.connection.put_container(BACKUP_CONTAINER) - - # Swift Checksum is the checksum of the concatenated segment checksums - swift_checksum = hashlib.md5() - - # Wrap the output of the backup process to segment it for swift - stream_reader = StreamReader(stream, filename, MAX_FILE_SIZE) - LOG.debug('Using segment size %s', stream_reader.max_file_size) - - url = self.connection.url - # Full location where the backup manifest is stored - location = "%s/%s/%s" % (url, BACKUP_CONTAINER, filename) - - # Information about each segment upload job - segment_results = [] - - # Read from the stream and write to the container in swift - while not stream_reader.end_of_file: - LOG.debug('Saving segment %s.', stream_reader.segment) - path = stream_reader.segment_path - etag = self.connection.put_object(BACKUP_CONTAINER, - stream_reader.segment, - stream_reader) - - segment_checksum = stream_reader.segment_checksum.hexdigest() - - # Check each segment MD5 hash against swift etag - # Raise an error and mark backup as failed - if etag != segment_checksum: - LOG.error("Error saving data segment to swift. " - "ETAG: %(tag)s Segment MD5: %(checksum)s.", - {'tag': etag, 'checksum': segment_checksum}) - return False, "Error saving data to Swift!", None, location - - segment_results.append({ - 'path': path, - 'etag': etag, - 'size_bytes': stream_reader.segment_length - }) - - if six.PY3: - swift_checksum.update(segment_checksum.encode()) - else: - swift_checksum.update(segment_checksum) - - # All segments uploaded. - num_segments = len(segment_results) - LOG.debug('File uploaded in %s segments.', num_segments) - - # An SLO will be generated if the backup was more than one segment in - # length. - large_object = num_segments > 1 - - # Meta data is stored as headers - if metadata is None: - metadata = {} - metadata.update(stream.metadata()) - headers = {} - for key, value in metadata.items(): - headers[self._set_attr(key)] = value - - LOG.debug('Metadata headers: %s', str(headers)) - if large_object: - LOG.info('Creating the manifest file.') - manifest_data = json.dumps(segment_results) - LOG.debug('Manifest contents: %s', manifest_data) - # The etag returned from the manifest PUT is the checksum of the - # manifest object (which is empty); this is not the checksum we - # want. - self.connection.put_object(BACKUP_CONTAINER, - filename, - manifest_data, - query_string='multipart-manifest=put') - - # Validation checksum is the Swift Checksum - final_swift_checksum = swift_checksum.hexdigest() - else: - LOG.info('Backup fits in a single segment. Moving segment ' - '%(segment)s to %(filename)s.', - {'segment': stream_reader.first_segment, - 'filename': filename}) - segment_result = segment_results[0] - # Just rename it via a special put copy. - headers['X-Copy-From'] = segment_result['path'] - self.connection.put_object(BACKUP_CONTAINER, - filename, '', - headers=headers) - # Delete the old segment file that was copied - LOG.debug('Deleting the old segment file %s.', - stream_reader.first_segment) - self.connection.delete_object(BACKUP_CONTAINER, - stream_reader.first_segment) - final_swift_checksum = segment_result['etag'] - - # Validate the object by comparing checksums - # Get the checksum according to Swift - resp = self.connection.head_object(BACKUP_CONTAINER, filename) - # swift returns etag in double quotes - # e.g. '"dc3b0827f276d8d78312992cc60c2c3f"' - etag = resp['etag'].strip('"') - - # Raise an error and mark backup as failed - if etag != final_swift_checksum: - LOG.error( - ("Error saving data to swift. Manifest " - "ETAG: %(tag)s Swift MD5: %(checksum)s"), - {'tag': etag, 'checksum': final_swift_checksum}) - return False, "Error saving data to Swift!", None, location - - return (True, "Successfully saved data to Swift!", - final_swift_checksum, location) - - def _explodeLocation(self, location): - storage_url = "/".join(location.split('/')[:-2]) - container = location.split('/')[-2] - filename = location.split('/')[-1] - return storage_url, container, filename - - def _verify_checksum(self, etag, checksum): - etag_checksum = etag.strip('"') - if etag_checksum != checksum: - log_fmt = ("Original checksum: %(original)s does not match" - " the current checksum: %(current)s") - exc_fmt = _("Original checksum: %(original)s does not match" - " the current checksum: %(current)s") - msg_content = { - 'original': etag_checksum, - 'current': checksum} - LOG.error(log_fmt, msg_content) - raise SwiftDownloadIntegrityError(exc_fmt % msg_content) - return True - - def load(self, location, backup_checksum): - """Restore a backup from the input stream to the restore_location.""" - storage_url, container, filename = self._explodeLocation(location) - - headers, info = self.connection.get_object(container, filename, - resp_chunk_size=CHUNK_SIZE) - - if CONF.verify_swift_checksum_on_restore: - self._verify_checksum(headers.get('etag', ''), backup_checksum) - - return info - - def _get_attr(self, original): - """Get a friendly name from an object header key.""" - key = original.replace('-', '_') - key = key.replace('x_object_meta_', '') - return key - - def _set_attr(self, original): - """Return a swift friendly header key.""" - key = original.replace('_', '-') - return 'X-Object-Meta-%s' % key - - def load_metadata(self, location, backup_checksum): - """Load metadata from swift.""" - - storage_url, container, filename = self._explodeLocation(location) - - headers = self.connection.head_object(container, filename) - - if CONF.verify_swift_checksum_on_restore: - self._verify_checksum(headers.get('etag', ''), backup_checksum) - - _meta = {} - for key, value in headers.items(): - if key.startswith('x-object-meta'): - _meta[self._get_attr(key)] = value - - return _meta - - def save_metadata(self, location, metadata={}): - """Save metadata to a swift object.""" - - storage_url, container, filename = self._explodeLocation(location) - - headers = {} - for key, value in metadata.items(): - headers[self._set_attr(key)] = value - - LOG.info("Writing metadata: %s", str(headers)) - self.connection.post_object(container, filename, headers=headers) diff --git a/trove/db/sqlalchemy/mappers.py b/trove/db/sqlalchemy/mappers.py index 7bb8b15f..5ff8e434 100644 --- a/trove/db/sqlalchemy/mappers.py +++ b/trove/db/sqlalchemy/mappers.py @@ -54,6 +54,8 @@ def map(engine, models): Table('reservations', meta, autoload=True)) orm.mapper(models['backups'], Table('backups', meta, autoload=True)) + orm.mapper(models['backup_strategy'], + Table('backup_strategy', meta, autoload=True)) orm.mapper(models['security_groups'], Table('security_groups', meta, autoload=True)) orm.mapper(models['security_group_rules'], diff --git a/trove/db/sqlalchemy/migrate_repo/versions/045_add_backup_strategy.py b/trove/db/sqlalchemy/migrate_repo/versions/045_add_backup_strategy.py new file mode 100644 index 00000000..5b5deaea --- /dev/null +++ b/trove/db/sqlalchemy/migrate_repo/versions/045_add_backup_strategy.py @@ -0,0 +1,46 @@ +# Copyright 2020 Catalyst Cloud +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy.schema import Column +from sqlalchemy.schema import Index +from sqlalchemy.schema import MetaData +from sqlalchemy.schema import UniqueConstraint + +from trove.db.sqlalchemy.migrate_repo.schema import create_tables +from trove.db.sqlalchemy.migrate_repo.schema import DateTime +from trove.db.sqlalchemy.migrate_repo.schema import String +from trove.db.sqlalchemy.migrate_repo.schema import Table + +meta = MetaData() + +backup_strategy = Table( + 'backup_strategy', + meta, + Column('id', String(36), primary_key=True, nullable=False), + Column('tenant_id', String(36), nullable=False), + Column('instance_id', String(36), nullable=False, default=''), + Column('backend', String(255), nullable=False), + Column('swift_container', String(255), nullable=True), + Column('created', DateTime()), + UniqueConstraint( + 'tenant_id', 'instance_id', + name='UQ_backup_strategy_tenant_id_instance_id'), + Index("backup_strategy_tenant_id_instance_id", "tenant_id", "instance_id"), +) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + create_tables([backup_strategy]) diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py index 77b10d81..87abb82d 100644 --- a/trove/guestagent/datastore/mysql_common/service.py +++ b/trove/guestagent/datastore/mysql_common/service.py @@ -741,8 +741,19 @@ class BaseMySqlApp(object): user_token = context.auth_token auth_url = CONF.service_credentials.auth_url user_tenant = context.project_id - metadata = f'datastore:{backup_info["datastore"]},' \ - f'datastore_version:{backup_info["datastore_version"]}' + + swift_metadata = ( + f'datastore:{backup_info["datastore"]},' + f'datastore_version:{backup_info["datastore_version"]}' + ) + swift_params = f'--swift-extra-metadata={swift_metadata}' + swift_container = backup_info.get('swift_container', + CONF.backup_swift_container) + if backup_info.get('swift_container'): + swift_params = ( + f'{swift_params} ' + f'--swift-container {swift_container}' + ) command = ( f'/usr/bin/python3 main.py --backup --backup-id={backup_id} ' @@ -751,7 +762,7 @@ class BaseMySqlApp(object): f'--db-host=127.0.0.1 ' f'--os-token={user_token} --os-auth-url={auth_url} ' f'--os-tenant-id={user_tenant} ' - f'--swift-extra-metadata={metadata} ' + f'{swift_params} ' f'{incremental}' ) @@ -792,6 +803,7 @@ class BaseMySqlApp(object): 'state': BackupState.COMPLETED, }) else: + LOG.error(f'Cannot parse backup output: {result}') backup_state.update({ 'success': False, 'state': BackupState.FAILED, diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index afbac20d..7adfb337 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -1371,8 +1371,8 @@ class BackupTasks(object): return container, prefix @classmethod - def delete_files_from_swift(cls, context, filename): - container = CONF.backup_swift_container + def delete_files_from_swift(cls, context, container, filename): + container = container or CONF.backup_swift_container client = clients.create_swift_client(context) obj = client.head_object(container, filename) if 'x-static-large-object' in obj: @@ -1404,7 +1404,9 @@ class BackupTasks(object): try: filename = backup.filename if filename: - BackupTasks.delete_files_from_swift(context, filename) + BackupTasks.delete_files_from_swift(context, + backup.container_name, + filename) except ValueError: _delete(backup) except ClientException as e: diff --git a/trove/tests/api/instances_actions.py b/trove/tests/api/instances_actions.py index f8d9f0af..608e827e 100644 --- a/trove/tests/api/instances_actions.py +++ b/trove/tests/api/instances_actions.py @@ -267,14 +267,14 @@ class RebootTestBase(ActionTestBase): poll_until(is_finished_rebooting, time_out=TIME_OUT_TIME) - def wait_for_status(self, status, timeout=60): + def wait_for_status(self, status, timeout=60, sleep_time=5): def is_status(): instance = self.instance if instance.status in status: return True return False - poll_until(is_status, time_out=timeout) + poll_until(is_status, time_out=timeout, sleep_time=sleep_time) @test(groups=[tests.DBAAS_API_INSTANCE_ACTIONS], @@ -323,7 +323,9 @@ class StopTests(RebootTestBase): def test_stop_mysql(self): """Stops MySQL by admin.""" instance_info.dbaas_admin.management.stop(self.instance_id) - self.wait_for_status(['SHUTDOWN'], timeout=60) + + # The instance status will only be updated by guest agent. + self.wait_for_status(['SHUTDOWN'], timeout=90, sleep_time=10) @test(depends_on=[test_stop_mysql]) def test_volume_info_while_mysql_is_down(self): diff --git a/trove/tests/scenario/runners/guest_log_runners.py b/trove/tests/scenario/runners/guest_log_runners.py index ebc9fb01..f4c11c26 100644 --- a/trove/tests/scenario/runners/guest_log_runners.py +++ b/trove/tests/scenario/runners/guest_log_runners.py @@ -741,8 +741,9 @@ class GuestLogRunner(TestRunner): self.admin_client, log_name, expected_type=guest_log.LogType.SYS.name, - expected_status=guest_log.LogStatus.Partial.name, - expected_published=1, expected_pending=1, + expected_status=[guest_log.LogStatus.Partial.name, + guest_log.LogStatus.Published.name], + expected_published=1, expected_pending=None, is_admin=True) def run_test_log_publish_again_sys(self): @@ -751,9 +752,10 @@ class GuestLogRunner(TestRunner): self.admin_client, log_name, expected_type=guest_log.LogType.SYS.name, - expected_status=guest_log.LogStatus.Partial.name, + expected_status=[guest_log.LogStatus.Partial.name, + guest_log.LogStatus.Published.name], expected_published=self._get_last_log_published(log_name) + 1, - expected_pending=1, + expected_pending=None, is_admin=True) def run_test_log_generator_sys(self): diff --git a/trove/tests/unittests/backup/test_backup_models.py b/trove/tests/unittests/backup/test_backup_models.py index 4cc9d113..7ff6066c 100644 --- a/trove/tests/unittests/backup/test_backup_models.py +++ b/trove/tests/unittests/backup/test_backup_models.py @@ -557,3 +557,48 @@ class OrderingTests(trove_testtools.TestCase): actual = [b.name for b in backups] expected = [u'one', u'two', u'three', u'four'] self.assertEqual(expected, actual) + + +class TestBackupStrategy(trove_testtools.TestCase): + def setUp(self): + super(TestBackupStrategy, self).setUp() + util.init_db() + self.context, self.instance_id = _prep_conf(timeutils.utcnow()) + + def test_create(self): + db_backstg = models.BackupStrategy.create(self.context, + self.instance_id, + 'test-container') + self.addCleanup(models.BackupStrategy.delete, self.context, + self.context.project_id, self.instance_id) + + self.assertEqual('test-container', db_backstg.swift_container) + + def test_list(self): + models.BackupStrategy.create(self.context, self.instance_id, + 'test_list') + self.addCleanup(models.BackupStrategy.delete, self.context, + self.context.project_id, self.instance_id) + + db_backstgs = models.BackupStrategy.list(self.context, + self.context.project_id, + self.instance_id).all() + + self.assertEqual(1, len(db_backstgs)) + self.assertEqual('test_list', db_backstgs[0].swift_container) + + def test_delete(self): + models.BackupStrategy.create(self.context, self.instance_id, + 'test_delete') + db_backstgs = models.BackupStrategy.list(self.context, + self.context.project_id, + self.instance_id).all() + self.assertEqual(1, len(db_backstgs)) + + models.BackupStrategy.delete(self.context, self.context.project_id, + self.instance_id) + + db_backstgs = models.BackupStrategy.list(self.context, + self.context.project_id, + self.instance_id).all() + self.assertEqual(0, len(db_backstgs)) diff --git a/trove/tests/unittests/taskmanager/test_models.py b/trove/tests/unittests/taskmanager/test_models.py index 07530874..e9dad0d6 100644 --- a/trove/tests/unittests/taskmanager/test_models.py +++ b/trove/tests/unittests/taskmanager/test_models.py @@ -14,15 +14,15 @@ import os from tempfile import NamedTemporaryFile from unittest import mock - -from cinderclient import exceptions as cinder_exceptions -from cinderclient.v2 import volumes as cinderclient_volumes -import cinderclient.v2.client as cinderclient from unittest.mock import call from unittest.mock import MagicMock from unittest.mock import Mock from unittest.mock import patch from unittest.mock import PropertyMock + +from cinderclient import exceptions as cinder_exceptions +from cinderclient.v2 import volumes as cinderclient_volumes +import cinderclient.v2.client as cinderclient import neutronclient.v2_0.client as neutronclient from novaclient import exceptions as nova_exceptions import novaclient.v2.flavors @@ -999,7 +999,7 @@ class BackupTasksTest(trove_testtools.TestCase): self.backup.id = 'backup_id' self.backup.name = 'backup_test', self.backup.description = 'test desc' - self.backup.location = 'http://xxx/z_CLOUD/12e48.xbstream.gz' + self.backup.location = 'http://xxx/z_CLOUD/container/12e48.xbstream.gz' self.backup.instance_id = 'instance id' self.backup.created = 'yesterday' self.backup.updated = 'today' @@ -1049,6 +1049,18 @@ class BackupTasksTest(trove_testtools.TestCase): "backup should be in DELETE_FAILED status" ) + @patch('trove.common.clients.create_swift_client') + def test_delete_backup_delete_swift(self, mock_swift_client): + client_mock = MagicMock() + mock_swift_client.return_value = client_mock + + taskmanager_models.BackupTasks.delete_backup(mock.ANY, self.backup.id) + + client_mock.head_object.assert_called_once_with('container', + '12e48.xbstream.gz') + client_mock.delete_object.assert_called_once_with('container', + '12e48.xbstream.gz') + def test_parse_manifest(self): manifest = 'container/prefix' cont, prefix = taskmanager_models.BackupTasks._parse_manifest(manifest) |