Support incremental backup for MariaDB

Enable the tests in CI as well.

Change-Id: Ie9706d26355bd325baf50ec874f05e6904768a1a
(cherry picked from commit 605ff34608e2db53673764fe406dd94306b69c5b)

8 files changed, 153 insertions, 14 deletions

diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 0b4e9041..3d4b28de 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -217,6 +217,7 @@ function configure_trove {
     iniset_conditional $TROVE_CONF DEFAULT resize_time_out $TROVE_RESIZE_TIME_OUT
     iniset_conditional $TROVE_CONF DEFAULT usage_timeout $TROVE_USAGE_TIMEOUT
     iniset_conditional $TROVE_CONF DEFAULT state_change_wait_time $TROVE_STATE_CHANGE_WAIT_TIME
+    iniset_conditional $TROVE_CONF DEFAULT reboot_time_out 300
 
     configure_keystone_authtoken_middleware $TROVE_CONF trove
     iniset $TROVE_CONF service_credentials username trove
@@ -239,7 +240,6 @@ function configure_trove {
     iniset $TROVE_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin
     iniset $TROVE_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin
     iniset $TROVE_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin
-    iniset $TROVE_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin
     iniset $TROVE_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin
 
     iniset $TROVE_CONF cassandra tcp_ports 7000,7001,7199,9042,9160
@@ -278,7 +278,6 @@ function configure_trove {
     iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_nova_client trove.common.clients_admin.nova_client_trove_admin
     iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_cinder_client trove.common.clients_admin.cinder_client_trove_admin
     iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_neutron_client trove.common.clients_admin.neutron_client_trove_admin
-    iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_swift_client trove.common.clients_admin.swift_client_trove_admin
     iniset $TROVE_GUESTAGENT_CONF DEFAULT remote_glance_client trove.common.clients_admin.glance_client_trove_admin
 
     # To avoid 'Connection timed out' error of sudo command inside the guest agent
diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py
index 68288eb6..1fd7342a 100644
--- a/trove/guestagent/backup/backupagent.py
+++ b/trove/guestagent/backup/backupagent.py
@@ -166,6 +166,7 @@ class BackupAgent(object):
 
             LOG.info("Restoring instance from backup %(id)s to "
                      "%(restore_location)s", backup_info)
+
             content_size = runner.restore()
             LOG.info("Restore from backup %(id)s completed successfully "
                      "to %(restore_location)s", backup_info)
diff --git a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py
index d32c69b4..2ff310ca 100644
--- a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py
+++ b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py
@@ -15,6 +15,7 @@ import re
 
 from oslo_log import log as logging
 
+from trove.common.i18n import _
 from trove.guestagent.datastore.mysql import service as mysql_service
 from trove.guestagent.datastore.mysql_common import service as common_service
 from trove.guestagent.strategies.backup import base
@@ -29,13 +30,13 @@ class MariaBackup(base.BackupRunner):
 
     @property
     def user_and_pass(self):
-        return (' --user=%(user)s --password=%(password)s --host=127.0.0.1 ' %
+        return ('--user=%(user)s --password=%(password)s --host=127.0.0.1' %
                 {'user': common_service.ADMIN_USER_NAME,
                  'password': mysql_service.MySqlApp.get_auth_password()})
 
     @property
     def cmd(self):
-        cmd = ('sudo mariabackup --backup --stream=xbstream' +
+        cmd = ('sudo mariabackup --backup --stream=xbstream ' +
                self.user_and_pass + ' 2>' + BACKUP_LOG)
         return cmd + self.zip_cmd + self.encrypt_cmd
 
@@ -61,10 +62,49 @@ class MariaBackup(base.BackupRunner):
 
         return True
 
+    def metadata(self):
+        LOG.debug('Getting metadata for backup %s', self.base_filename)
+
+        meta = {}
+        lsn = re.compile(r"The latest check point \(for incremental\): "
+                         r"'(\d+)'")
+        with open(BACKUP_LOG, 'r') as backup_log:
+            output = backup_log.read()
+            match = lsn.search(output)
+            if match:
+                meta = {'lsn': match.group(1)}
+
+        LOG.info("Metadata for backup %s: %s", self.base_filename, meta)
+        return meta
+
     @property
     def filename(self):
         return '%s.xbstream' % self.base_filename
 
 
 class MariaBackupIncremental(MariaBackup):
-    pass
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get('lsn'):
+            raise AttributeError(_('lsn attribute missing, bad parent?'))
+        super(MariaBackupIncremental, self).__init__(*args, **kwargs)
+        self.parent_location = kwargs.get('parent_location')
+        self.parent_checksum = kwargs.get('parent_checksum')
+
+    @property
+    def cmd(self):
+        cmd = (
+            'sudo mariabackup --backup --stream=xbstream'
+            ' --incremental-lsn=%(lsn)s ' +
+            self.user_and_pass +
+            ' 2>' +
+            BACKUP_LOG
+        )
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def metadata(self):
+        meta = super(MariaBackupIncremental, self).metadata()
+        meta.update({
+            'parent_location': self.parent_location,
+            'parent_checksum': self.parent_checksum,
+        })
+        return meta
diff --git a/trove/guestagent/strategies/backup/mysql_impl.py b/trove/guestagent/strategies/backup/mysql_impl.py
index 1386af3d..8d5b844a 100644
--- a/trove/guestagent/strategies/backup/mysql_impl.py
+++ b/trove/guestagent/strategies/backup/mysql_impl.py
@@ -100,7 +100,7 @@ class InnoBackupEx(base.BackupRunner):
         return True
 
     def metadata(self):
-        LOG.debug('Getting metadata from backup.')
+        LOG.debug('Getting metadata for backup %s', self.base_filename)
         meta = {}
         lsn = re.compile(r"The latest check point \(for incremental\): "
                          r"'(\d+)'")
@@ -109,7 +109,7 @@ class InnoBackupEx(base.BackupRunner):
             match = lsn.search(output)
             if match:
                 meta = {'lsn': match.group(1)}
-        LOG.info("Metadata for backup: %s.", str(meta))
+        LOG.info("Metadata for backup %s: %s", self.base_filename, meta)
         return meta
 
     @property
diff --git a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py
index 99339cbb..48292595 100644
--- a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py
+++ b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py
@@ -11,22 +11,32 @@
 #    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #    See the License for the specific language governing permissions and
 #    limitations under the License.
+import glob
+import os
 
 from oslo_log import log as logging
 
+from trove.common import cfg
+from trove.common import utils
 from trove.guestagent.common import operating_system
 from trove.guestagent.datastore.experimental.mariadb import service
 from trove.guestagent.datastore.mysql_common import service as mysql_service
+from trove.guestagent.strategies.restore import base
 from trove.guestagent.strategies.restore import mysql_impl
 
 LOG = logging.getLogger(__name__)
+PREPARE_LOG = '/tmp/innoprepare.log'
 
 
-class MariaBackup(mysql_impl.InnoBackupEx):
+class MariaBackup(base.RestoreRunner, mysql_impl.MySQLRestoreMixin):
     __strategy_name__ = 'mariabackup'
     base_restore_cmd = ('sudo mbstream -x -C %(restore_location)s '
                         '2>/tmp/xbstream_extract.log')
 
+    def __init__(self, *args, **kwargs):
+        self._app = None
+        super(MariaBackup, self).__init__(*args, **kwargs)
+
     @property
     def app(self):
         if self._app is None:
@@ -35,6 +45,14 @@ class MariaBackup(mysql_impl.InnoBackupEx):
             )
         return self._app
 
+    def pre_restore(self):
+        self.app.stop_db()
+        LOG.debug("Cleaning out restore location: %s.", self.restore_location)
+        operating_system.chmod(self.restore_location,
+                               operating_system.FileMode.SET_FULL,
+                               as_root=True)
+        utils.clean_out(self.restore_location)
+
     def post_restore(self):
         operating_system.chown(self.restore_location, 'mysql', None,
                                force=True, as_root=True)
@@ -45,6 +63,11 @@ class MariaBackup(mysql_impl.InnoBackupEx):
         self.app.start_mysql()
         LOG.debug("Finished post restore.")
 
+    def _delete_old_binlogs(self):
+        files = glob.glob(os.path.join(self.restore_location, "ib_logfile*"))
+        for f in files:
+            os.unlink(f)
+
     def check_process(self):
         LOG.debug('Checking return code of mbstream restore process.')
         return_code = self.process.wait()
@@ -56,4 +79,80 @@ class MariaBackup(mysql_impl.InnoBackupEx):
 
 
 class MariaBackupIncremental(MariaBackup):
-    pass
+    __strategy_name__ = 'mariabackupincremental'
+    incremental_prep = ('sudo mariabackup --prepare '
+                        '--target-dir=%(restore_location)s '
+                        '%(incremental_args)s '
+                        '2>/tmp/innoprepare.log')
+
+    def __init__(self, *args, **kwargs):
+        super(MariaBackupIncremental, self).__init__(*args, **kwargs)
+        self.content_length = 0
+
+    def _incremental_restore_cmd(self, incremental_dir):
+        """Return a command for a restore with a incremental location."""
+        args = {'restore_location': incremental_dir}
+        return (self.decrypt_cmd +
+                self.unzip_cmd +
+                (self.base_restore_cmd % args))
+
+    def _incremental_prepare_cmd(self, incremental_dir):
+        if incremental_dir is not None:
+            incremental_arg = '--incremental-dir=%s' % incremental_dir
+        else:
+            incremental_arg = ''
+
+        args = {
+            'restore_location': self.restore_location,
+            'incremental_args': incremental_arg,
+        }
+
+        return self.incremental_prep % args
+
+    def _incremental_prepare(self, incremental_dir):
+        prepare_cmd = self._incremental_prepare_cmd(incremental_dir)
+
+        LOG.debug("Running mariabackup prepare: %s.", prepare_cmd)
+        utils.execute(prepare_cmd, shell=True)
+        LOG.debug("mariabackup prepare finished successfully.")
+
+    def _incremental_restore(self, location, checksum):
+        """Recursively apply backups from all parents.
+        If we are the parent then we restore to the restore_location and
+        we apply the logs to the restore_location only.
+        Otherwise if we are an incremental we restore to a subfolder to
+        prevent stomping on the full restore data. Then we run apply log
+        with the '--incremental-dir' flag
+        """
+        metadata = self.storage.load_metadata(location, checksum)
+        incremental_dir = None
+        if 'parent_location' in metadata:
+            LOG.info("Restoring parent: %(parent_location)s"
+                     " checksum: %(parent_checksum)s.", metadata)
+            parent_location = metadata['parent_location']
+            parent_checksum = metadata['parent_checksum']
+            # Restore parents recursively so backup are applied sequentially
+            self._incremental_restore(parent_location, parent_checksum)
+            # for *this* backup set the incremental_dir
+            # just use the checksum for the incremental path as it is
+            # sufficiently unique /var/lib/mysql/<checksum>
+            incremental_dir = os.path.join(
+                cfg.get_configuration_property('mount_point'), checksum)
+            operating_system.create_directory(incremental_dir, as_root=True)
+            command = self._incremental_restore_cmd(incremental_dir)
+        else:
+            # The parent (full backup) use the same command from InnobackupEx
+            # super class and do not set an incremental_dir.
+            command = self.restore_cmd
+
+        self.content_length += self._unpack(location, checksum, command)
+        self._incremental_prepare(incremental_dir)
+
+        # Delete unpacked incremental backup metadata
+        if incremental_dir:
+            operating_system.remove(incremental_dir, force=True, as_root=True)
+
+    def _run_restore(self):
+        """Run incremental restore."""
+        self._incremental_restore(self.location, self.checksum)
+        return self.content_length
diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py
index cf4529ab..eea8c778 100644
--- a/trove/guestagent/strategies/restore/mysql_impl.py
+++ b/trove/guestagent/strategies/restore/mysql_impl.py
@@ -211,8 +211,7 @@ class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin):
 
     def pre_restore(self):
         self.app.stop_db()
-        LOG.info("Cleaning out restore location: %s.",
-                 self.restore_location)
+        LOG.debug("Cleaning out restore location: %s.", self.restore_location)
         operating_system.chmod(self.restore_location, FileMode.SET_FULL,
                                as_root=True)
         utils.clean_out(self.restore_location)
@@ -313,7 +312,7 @@ class InnoBackupExIncremental(InnoBackupEx):
         prepare_cmd = self._incremental_prepare_cmd(incremental_dir)
         LOG.debug("Running innobackupex prepare: %s.", prepare_cmd)
         utils.execute(prepare_cmd, shell=True)
-        LOG.info("Innobackupex prepare finished successfully.")
+        LOG.debug("Innobackupex prepare finished successfully.")
 
     def _incremental_restore(self, location, checksum):
         """Recursively apply backups from all parents.
diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py
index bf8a1024..c1b3d220 100755
--- a/trove/taskmanager/models.py
+++ b/trove/taskmanager/models.py
@@ -1092,7 +1092,8 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin):
         action.execute()
 
     def create_backup(self, backup_info):
-        LOG.info("Initiating backup for instance %s.", self.id)
+        LOG.info("Initiating backup for instance %s, backup_info: %s", self.id,
+                 backup_info)
         self.guest.create_backup(backup_info)
 
     def backup_required_for_replication(self):
diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py
index c662028a..c386fafd 100644
--- a/trove/tests/int_tests.py
+++ b/trove/tests/int_tests.py
@@ -319,7 +319,7 @@ register(
     ["mariadb_supported"],
     single=[common_groups,
             backup_groups,
-            # backup_incremental_groups,
+            backup_incremental_groups,
             configuration_groups,
             database_actions_groups,
             root_actions_groups,