Support mysql 8.0

* MySQL 5.7 and MySQL 8.0 need different percona-xtrabackup package version.
  Added Percona XtraBackup 8 support for MySQL 8.x backup and restore.
* Construct different backup container image names for MySQL 5.7 and MySQL 8.0
  based on the default option value.
* Two docker images are uploaded for backup/restore:
  openstacktrove/db-backup-mysql5.7:1.0.0 and
  openstacktrove/db-backup-mysql8.0:1.0.0. Trove guest agent can automatically
  choose the approriate one based on the datastore version.
* Added option "secure-file-priv=NULL" in MySQL config template to fix
  https://github.com/docker-library/mysql/issues/541.
* Stop using IDENTIFIED BY in GRANT clause (also REVOKE). Starting with MySQL 8
  creating a user implicitly using the GRANT command is not supported.

Story: #2008275
Task: #41143

Change-Id: Ibdec63324b1b39ba9b8a38dbe529da17bbb06767

4 files changed, 148 insertions, 6 deletions

diff --git a/backup/Dockerfile b/backup/Dockerfile
index a5e4e7e0..f60149c7 100644
--- a/backup/Dockerfile
+++ b/backup/Dockerfile
@@ -1,9 +1,8 @@
 FROM ubuntu:18.04
 LABEL maintainer="anlin.kong@gmail.com"
 
-ARG DATASTORE="mysql"
+ARG DATASTORE="mysql5.7"
 ARG APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated"
-ARG PERCONA_XTRABACKUP_VERSION=24
 
 RUN export DEBIAN_FRONTEND="noninteractive" \
     && export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
@@ -16,7 +15,7 @@ RUN apt-get update \
 
 COPY . /opt/trove/backup
 WORKDIR /opt/trove/backup
-RUN ./install.sh $DATASTORE ${PERCONA_XTRABACKUP_VERSION}
+RUN ./install.sh $DATASTORE
 
 RUN apt-get update \
     && apt-get install $APTOPTS build-essential python3-setuptools python3-all python3-all-dev python3-pip libffi-dev libssl-dev libxml2-dev libxslt1-dev libyaml-dev \
diff --git a/backup/drivers/xtrabackup.py b/backup/drivers/xtrabackup.py
new file mode 100644
index 00000000..6874326f
--- /dev/null
+++ b/backup/drivers/xtrabackup.py
@@ -0,0 +1,133 @@
+# Copyright 2020 Catalyst Cloud
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import re
+
+from oslo_concurrency import processutils
+from oslo_config import cfg
+from oslo_log import log as logging
+
+from backup.drivers import mysql_base
+
+LOG = logging.getLogger(__name__)
+CONF = cfg.CONF
+
+
+class XtraBackup(mysql_base.MySQLBaseRunner):
+    """Implementation of Backup and Restore for XtraBackup 8.0.
+
+    According to
+    https://www.percona.com/doc/percona-xtrabackup/8.0/index.html#user-s-manual,
+    Percona XtraBackup 8.0 does not support making backups of databases created
+    in versions prior to 8.0 of MySQL.
+
+    Percona XtraBackup 8.0.12 supports backup and restore processing for
+    versions of MySQL 8.x.
+
+    innobackupex was removed in Percona XtraBackup 8.0.
+    """
+    backup_log = '/tmp/xtrabackup.log'
+    prepare_log = '/tmp/prepare.log'
+    restore_cmd = ('xbstream -x -C %(restore_location)s --parallel=2'
+                   ' 2>/tmp/xbstream_extract.log')
+    prepare_cmd = (f'xtrabackup '
+                   f'--target-dir=%(restore_location)s '
+                   f'--prepare 2>{prepare_log}')
+
+    @property
+    def cmd(self):
+        cmd = (f'xtrabackup --backup --stream=xbstream --parallel=2 '
+               f'--datadir={self.datadir} {self.user_and_pass} '
+               f'2>{self.backup_log}')
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def check_restore_process(self):
+        """Check whether xbstream restore is successful."""
+        LOG.info('Checking return code of xbstream restore process.')
+        return_code = self.process.wait()
+        if return_code != 0:
+            LOG.error('xbstream exited with %s', return_code)
+            return False
+
+        with open('/tmp/xbstream_extract.log', 'r') as xbstream_log:
+            for line in xbstream_log:
+                # Ignore empty lines
+                if not line.strip():
+                    continue
+
+                LOG.error('xbstream restore failed with: %s',
+                          line.rstrip('\n'))
+                return False
+
+        return True
+
+    def post_restore(self):
+        """Prepare after data restore."""
+        LOG.info("Running prepare command: %s.", self.prepare_command)
+        processutils.execute(self.prepare_command, shell=True)
+
+        LOG.info("Checking prepare log")
+        with open(self.prepare_log, 'r') as prepare_log:
+            output = prepare_log.read()
+            if not output:
+                msg = "Empty prepare log file"
+                raise Exception(msg)
+
+            last_line = output.splitlines()[-1].strip()
+            if not re.search('completed OK!', last_line):
+                msg = "Prepare did not complete successfully"
+                raise Exception(msg)
+
+
+class XtraBackupIncremental(XtraBackup):
+    """XtraBackup incremental backup."""
+    prepare_log = '/tmp/prepare.log'
+    incremental_prep = (f'xtrabackup --prepare --apply-log-only'
+                        f' --target-dir=%(restore_location)s'
+                        f' %(incremental_args)s'
+                        f' 2>{prepare_log}')
+
+    def __init__(self, *args, **kwargs):
+        if not kwargs.get('lsn'):
+            raise AttributeError('lsn attribute missing')
+        self.parent_location = kwargs.pop('parent_location', '')
+        self.parent_checksum = kwargs.pop('parent_checksum', '')
+
+        super(XtraBackupIncremental, self).__init__(*args, **kwargs)
+
+    @property
+    def cmd(self):
+        cmd = (f'xtrabackup --backup --stream=xbstream '
+               f'--incremental --incremental-lsn=%(lsn)s '
+               f'--datadir={self.datadir} {self.user_and_pass} '
+               f'2>{self.backup_log}')
+        return cmd + self.zip_cmd + self.encrypt_cmd
+
+    def get_metadata(self):
+        _meta = super(XtraBackupIncremental, self).get_metadata()
+
+        _meta.update({
+            'parent_location': self.parent_location,
+            'parent_checksum': self.parent_checksum,
+        })
+        return _meta
+
+    def run_restore(self):
+        """Run incremental restore.
+
+        https://www.percona.com/doc/percona-xtrabackup/8.0/backup_scenarios/incremental_backup.html
+        """
+        LOG.debug('Running incremental restore')
+        self.incremental_restore(self.location, self.checksum)
+        return self.restore_content_length
diff --git a/backup/install.sh b/backup/install.sh
index a2aad105..26f5a51e 100755
--- a/backup/install.sh
+++ b/backup/install.sh
@@ -5,12 +5,20 @@ export APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1
 APTOPTS="-y -qq --no-install-recommends --allow-unauthenticated"
 
 case "$1" in
-"mysql")
+"mysql5.7")
     curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb
     dpkg -i percona-release.deb
     percona-release enable-only tools release
     apt-get update
-    apt-get install $APTOPTS percona-xtrabackup-$2
+    apt-get install $APTOPTS percona-xtrabackup-24
+    rm -f percona-release.deb
+    ;;
+"mysql8.0")
+    curl -sSL https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb -o percona-release.deb
+    dpkg -i percona-release.deb
+    percona-release enable-only tools release
+    apt-get update
+    apt-get install $APTOPTS percona-xtrabackup-80
     rm -f percona-release.deb
     ;;
 "mariadb")
diff --git a/backup/main.py b/backup/main.py
index a42dc4ba..04b38eca 100644
--- a/backup/main.py
+++ b/backup/main.py
@@ -36,7 +36,7 @@ cli_opts = [
     cfg.StrOpt(
         'driver',
         default='innobackupex',
-        choices=['innobackupex', 'mariabackup', 'pg_basebackup']
+        choices=['innobackupex', 'mariabackup', 'pg_basebackup', 'xtrabackup']
     ),
     cfg.BoolOpt('backup'),
     cfg.StrOpt('backup-encryption-key'),
@@ -68,6 +68,8 @@ driver_mapping = {
     'mariabackup_inc': 'backup.drivers.mariabackup.MariaBackupIncremental',
     'pg_basebackup': 'backup.drivers.postgres.PgBasebackup',
     'pg_basebackup_inc': 'backup.drivers.postgres.PgBasebackupIncremental',
+    'xtrabackup': 'backup.drivers.xtrabackup.XtraBackup',
+    'xtrabackup_inc': 'backup.drivers.xtrabackup.XtraBackupIncremental'
 }
 storage_mapping = {
     'swift': 'backup.storage.swift.SwiftStorage',