From 2afa3b039b327221950507d18905441bc4f3eb5a Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Thu, 28 Nov 2019 10:27:35 +1300 Subject: Fix mariadb CI - trove-scenario-mariadb-single - Trove supports MariaDB 10.4 - From MariaDB 10.3, Mariabackup is used instead of Percona XtraBackup for backup functionality - Some log improvements Change-Id: Ibaa6fd7273b98451097b32fb6b881008a236be9f (cherry picked from commit 11b0b8d6f2888f74cbae509fe6c81b0b324d0032) --- .../ubuntu-xenial-mariadb/install.d/30-mariadb | 5 +- integration/scripts/trovestack | 2 +- trove/common/cfg.py | 18 ++--- trove/common/strategies/storage/__init__.py | 1 - trove/guestagent/backup/backupagent.py | 26 +++----- trove/guestagent/datastore/manager.py | 5 +- trove/guestagent/datastore/mysql_common/manager.py | 17 +++-- trove/guestagent/datastore/mysql_common/service.py | 58 ++++++++++------- trove/guestagent/strategies/backup/__init__.py | 1 - .../strategies/backup/experimental/mariadb_impl.py | 76 +++++++++++++++++----- .../replication/experimental/mariadb_gtid.py | 26 ++++---- .../strategies/replication/mysql_base.py | 1 + .../restore/experimental/mariadb_impl.py | 63 +++++++++++++----- trove/guestagent/strategies/restore/mysql_impl.py | 6 +- trove/taskmanager/manager.py | 11 ++-- trove/taskmanager/models.py | 12 ++-- trove/templates/mariadb/config.template | 2 - trove/tests/int_tests.py | 3 +- trove/tests/unittests/guestagent/test_dbaas.py | 12 ++-- 19 files changed, 221 insertions(+), 124 deletions(-) diff --git a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb b/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb index 42799364..50eb9256 100755 --- a/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb +++ b/integration/scripts/files/elements/ubuntu-xenial-mariadb/install.d/30-mariadb @@ -18,13 +18,14 @@ curl -sS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb -apt-get update +apt-get install -y -qq apt-transport-https ca-certificates +apt-get update -qq # Disable password prompt debconf-set-selections <<< "mariadb-server mysql-server/root_password password ''" debconf-set-selections <<< "mariadb-server mysql-server/root_password_again password ''" -apt-get install -y --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common percona-xtrabackup-24 +apt-get install -y -qq --allow-unauthenticated mariadb-server mariadb-client galera-4 libmariadb3 mariadb-backup mariadb-common cat </etc/mysql/conf.d/no_perf_schema.cnf [mysqld] diff --git a/integration/scripts/trovestack b/integration/scripts/trovestack index ecc88f0f..c24647fe 100755 --- a/integration/scripts/trovestack +++ b/integration/scripts/trovestack @@ -563,7 +563,7 @@ function cmd_set_datastore() { VERSION="5.6" elif [ "$DATASTORE_TYPE" == "mariadb" ]; then PACKAGES=${PACKAGES:-"mariadb-server"} - VERSION="10.1" + VERSION="10.4" elif [ "$DATASTORE_TYPE" == "mongodb" ]; then PACKAGES=${PACKAGES:-"mongodb-org"} VERSION="3.2" diff --git a/trove/common/cfg.py b/trove/common/cfg.py index aaaf189f..3a627cbe 100644 --- a/trove/common/cfg.py +++ b/trove/common/cfg.py @@ -1430,7 +1430,13 @@ mariadb_opts = [ help='List of UDP ports and/or port ranges to open ' 'in the security group (only applicable ' 'if trove_security_groups_support is True).'), - cfg.StrOpt('backup_strategy', default='MariaDBInnoBackupEx', + cfg.StrOpt('backup_namespace', + default='trove.guestagent.strategies.backup.experimental' + '.mariadb_impl', + help='Namespace to load backup strategies from.', + deprecated_name='backup_namespace', + deprecated_group='DEFAULT'), + cfg.StrOpt('backup_strategy', default='MariaBackup', help='Default strategy to perform backups.', deprecated_name='backup_strategy', deprecated_group='DEFAULT'), @@ -1451,12 +1457,6 @@ mariadb_opts = [ cfg.IntOpt('usage_timeout', default=400, help='Maximum time (in seconds) to wait for a Guest to become ' 'active.'), - cfg.StrOpt('backup_namespace', - default='trove.guestagent.strategies.backup.experimental' - '.mariadb_impl', - help='Namespace to load backup strategies from.', - deprecated_name='backup_namespace', - deprecated_group='DEFAULT'), cfg.StrOpt('restore_namespace', default='trove.guestagent.strategies.restore.experimental' '.mariadb_impl', @@ -1468,8 +1468,8 @@ mariadb_opts = [ cfg.StrOpt('device_path', default='/dev/vdb', help='Device path for volume if volume support is enabled.'), cfg.DictOpt('backup_incremental_strategy', - default={'MariaDBInnoBackupEx': - 'MariaDBInnoBackupExIncremental'}, + default={'MariaBackup': + 'MariaBackupIncremental'}, help='Incremental Backup Runner based on the default ' 'strategy. For strategies that do not implement an ' 'incremental backup, the runner will use the default full ' diff --git a/trove/common/strategies/storage/__init__.py b/trove/common/strategies/storage/__init__.py index 55488ee3..8a458a93 100644 --- a/trove/common/strategies/storage/__init__.py +++ b/trove/common/strategies/storage/__init__.py @@ -22,5 +22,4 @@ LOG = logging.getLogger(__name__) def get_storage_strategy(storage_driver, ns=__name__): - LOG.debug("Getting storage strategy: %s.", storage_driver) return Strategy.get_strategy(storage_driver, ns) diff --git a/trove/guestagent/backup/backupagent.py b/trove/guestagent/backup/backupagent.py index 8220abc8..68288eb6 100644 --- a/trove/guestagent/backup/backupagent.py +++ b/trove/guestagent/backup/backupagent.py @@ -123,14 +123,13 @@ class BackupAgent(object): sent=timeutils.utcnow_ts( microsecond=True), **backup_state) - LOG.debug("Updated state for %s to %s.", - backup_id, backup_state) + LOG.info("Updated state for %s to %s.", backup_id, backup_state) def execute_backup(self, context, backup_info, runner=RUNNER, extra_opts=EXTRA_OPTS, incremental_runner=INCREMENTAL_RUNNER): - LOG.debug("Running backup %(id)s.", backup_info) + LOG.info("Running backup %(id)s.", backup_info) storage = get_storage_strategy( CONF.storage_strategy, CONF.storage_namespace)(context) @@ -154,12 +153,8 @@ class BackupAgent(object): parent_metadata, extra_opts) def execute_restore(self, context, backup_info, restore_location): - try: - LOG.debug("Getting Restore Runner %(type)s.", backup_info) restore_runner = self._get_restore_runner(backup_info['type']) - - LOG.debug("Getting Storage Strategy.") storage = get_storage_strategy( CONF.storage_strategy, CONF.storage_namespace)(context) @@ -168,16 +163,15 @@ class BackupAgent(object): checksum=backup_info['checksum'], restore_location=restore_location) backup_info['restore_location'] = restore_location - LOG.debug("Restoring instance from backup %(id)s to " - "%(restore_location)s.", backup_info) - content_size = runner.restore() - LOG.debug("Restore from backup %(id)s completed successfully " - "to %(restore_location)s.", backup_info) - LOG.debug("Restore size: %s.", content_size) + LOG.info("Restoring instance from backup %(id)s to " + "%(restore_location)s", backup_info) + content_size = runner.restore() + LOG.info("Restore from backup %(id)s completed successfully " + "to %(restore_location)s", backup_info) + LOG.debug("Restore size: %s", content_size) except Exception: - LOG.exception("Error restoring backup %(id)s.", backup_info) + LOG.exception("Error restoring backup %(id)s", backup_info) raise - else: - LOG.debug("Restored backup %(id)s.", backup_info) + LOG.debug("Restored backup %(id)s", backup_info) diff --git a/trove/guestagent/datastore/manager.py b/trove/guestagent/datastore/manager.py index c91d5517..d6d47379 100644 --- a/trove/guestagent/datastore/manager.py +++ b/trove/guestagent/datastore/manager.py @@ -111,8 +111,8 @@ class Manager(periodic_task.PeriodicTasks): try: return repl_strategy.get_instance(self.manager) except Exception as ex: - LOG.debug("Cannot get replication instance for '%(manager)s': " - "%(msg)s", {'manager': self.manager, 'msg': str(ex)}) + LOG.warning("Cannot get replication instance for '%(manager)s': " + "%(msg)s", {'manager': self.manager, 'msg': str(ex)}) return None @@ -315,6 +315,7 @@ class Manager(periodic_task.PeriodicTasks): except Exception as ex: LOG.exception("An error occurred applying modules: " "%s", str(ex)) + # The following block performs single-instance initialization. # Failures will be recorded, but won't stop the provisioning # or change the instance state. diff --git a/trove/guestagent/datastore/mysql_common/manager.py b/trove/guestagent/datastore/mysql_common/manager.py index 535fef03..83dea547 100644 --- a/trove/guestagent/datastore/mysql_common/manager.py +++ b/trove/guestagent/datastore/mysql_common/manager.py @@ -184,7 +184,8 @@ class MySqlManager(manager.Manager): return self.mysql_admin().disable_root() def _perform_restore(self, backup_info, context, restore_location, app): - LOG.info("Restoring database from backup %s.", backup_info['id']) + LOG.info("Restoring database from backup %s, backup_info: %s", + backup_info['id'], backup_info) try: backup.restore(context, backup_info, restore_location) except Exception: @@ -202,9 +203,12 @@ class MySqlManager(manager.Manager): app = self.mysql_app(self.mysql_app_status.get()) app.install_if_needed(packages) if device_path: - # stop and do not update database + LOG.info('Prepare the storage for %s', device_path) + app.stop_db( - do_not_start_on_reboot=self.volume_do_not_start_on_reboot) + do_not_start_on_reboot=self.volume_do_not_start_on_reboot + ) + device = volume.VolumeDevice(device_path) # unmount if device is already mounted device.unmount_device(device_path) @@ -219,13 +223,15 @@ class MySqlManager(manager.Manager): service.MYSQL_OWNER, recursive=False, as_root=True) - LOG.debug("Mounted the volume at %s.", mount_point) + LOG.debug("Mounted the volume at %s", mount_point) # We need to temporarily update the default my.cnf so that # mysql will start after the volume is mounted. Later on it # will be changed based on the config template # (see MySqlApp.secure()) and restart. app.set_data_dir(mount_point + '/data') app.start_mysql() + + LOG.info('Finish to prepare the storage for %s', device_path) if backup_info: self._perform_restore(backup_info, context, mount_point + "/data", app) @@ -337,7 +343,8 @@ class MySqlManager(manager.Manager): def get_replication_snapshot(self, context, snapshot_info, replica_source_config=None): - LOG.debug("Getting replication snapshot.") + LOG.info("Getting replication snapshot, snapshot_info: %s", + snapshot_info) app = self.mysql_app(self.mysql_app_status.get()) self.replication.enable_as_master(app, replica_source_config) diff --git a/trove/guestagent/datastore/mysql_common/service.py b/trove/guestagent/datastore/mysql_common/service.py index 92b84578..8bd1ad66 100644 --- a/trove/guestagent/datastore/mysql_common/service.py +++ b/trove/guestagent/datastore/mysql_common/service.py @@ -95,20 +95,28 @@ def clear_expired_password(): out, err = utils.execute("cat", secret_file, run_as_root=True, root_helper="sudo") except exception.ProcessExecutionError: - LOG.exception("/root/.mysql_secret does not exist.") - return - m = re.match('# The random password set for the root user at .*: (.*)', - out) - if m: - try: - out, err = utils.execute("mysqladmin", "-p%s" % m.group(1), - "password", "", run_as_root=True, - root_helper="sudo") - except exception.ProcessExecutionError: - LOG.exception("Cannot change mysql password.") - return - operating_system.remove(secret_file, force=True, as_root=True) - LOG.debug("Expired password removed.") + LOG.warning("/root/.mysql_secret does not exist.") + else: + m = re.match('# The random password set for the root user at .*: (.*)', + out) + if m: + try: + out, err = utils.execute("mysqladmin", "-p%s" % m.group(1), + "password", "", run_as_root=True, + root_helper="sudo") + except exception.ProcessExecutionError: + LOG.exception("Cannot change mysql password.") + return + operating_system.remove(secret_file, force=True, as_root=True) + LOG.debug("Expired password removed.") + + # The root user password will be changed in app.secure_root() later on + LOG.debug('Initializae the root password to empty') + try: + utils.execute("mysqladmin", "--user=root", "password", "", + run_as_root=True, root_helper="sudo") + except Exception: + LOG.exception("Failed to initializae the root password") def load_mysqld_options(): @@ -145,17 +153,19 @@ class BaseMySqlAppStatus(service.BaseDbStatus): def _get_actual_db_status(self): try: - out, err = utils.execute_with_timeout( + utils.execute_with_timeout( "/usr/bin/mysqladmin", "ping", run_as_root=True, root_helper="sudo", log_output_on_error=True) - LOG.info("MySQL Service Status is RUNNING.") + LOG.debug("MySQL Service Status is RUNNING.") return rd_instance.ServiceStatuses.RUNNING except exception.ProcessExecutionError: - LOG.exception("Failed to get database status.") + LOG.warning("Failed to get database status.") try: - out, err = utils.execute_with_timeout("/bin/ps", "-C", - "mysqld", "h") + out, _ = utils.execute_with_timeout( + "/bin/ps", "-C", "mysqld", "h", + log_output_on_error=True + ) pid = out.split()[0] # TODO(rnirmal): Need to create new statuses for instances # where the mysql service is up, but unresponsive @@ -163,7 +173,7 @@ class BaseMySqlAppStatus(service.BaseDbStatus): {'pid': pid}) return rd_instance.ServiceStatuses.BLOCKED except exception.ProcessExecutionError: - LOG.exception("Process execution failed.") + LOG.warning("Process execution failed.") mysql_args = load_mysqld_options() pid_file = mysql_args.get('pid_file', ['/var/run/mysqld/mysqld.pid'])[0] @@ -298,6 +308,7 @@ class BaseMySqlAdmin(object): mydb.character_set, mydb.collate) t = text(str(cd)) + LOG.debug('Creating database, command: %s', str(cd)) client.execute(t) def create_user(self, users): @@ -319,6 +330,7 @@ class BaseMySqlAdmin(object): g = sql_query.Grant(permissions='ALL', database=mydb.name, user=user.name, host=user.host) t = text(str(g)) + LOG.debug('Creating user, command: %s', str(g)) client.execute(t) def delete_database(self, database): @@ -569,11 +581,12 @@ class BaseKeepAliveConnection(interfaces.PoolListener): else: raise # MariaDB seems to timeout the client in a different - # way than MySQL and PXC, which manifests itself as - # an invalid packet sequence. Handle it as well. + # way than MySQL and PXC except pymysql_err.InternalError as ex: if "Packet sequence number wrong" in str(ex): raise exc.DisconnectionError() + elif 'Connection was killed' in str(ex): + raise exc.DisconnectionError() else: raise @@ -717,6 +730,7 @@ class BaseMySqlApp(object): def secure(self, config_contents): LOG.debug("Securing MySQL now.") clear_expired_password() + LOG.debug("Generating admin password.") admin_password = utils.generate_random_password() engine = sqlalchemy.create_engine( diff --git a/trove/guestagent/strategies/backup/__init__.py b/trove/guestagent/strategies/backup/__init__.py index 60438a2e..53992085 100644 --- a/trove/guestagent/strategies/backup/__init__.py +++ b/trove/guestagent/strategies/backup/__init__.py @@ -22,5 +22,4 @@ LOG = logging.getLogger(__name__) def get_backup_strategy(backup_driver, ns=__name__): - LOG.debug("Getting backup strategy: %s.", backup_driver) return Strategy.get_strategy(backup_driver, ns) diff --git a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py index 4a8a33c3..d32c69b4 100644 --- a/trove/guestagent/strategies/backup/experimental/mariadb_impl.py +++ b/trove/guestagent/strategies/backup/experimental/mariadb_impl.py @@ -1,28 +1,70 @@ -# Copyright 2016 Tesora Inc. -# All Rights Reserved. +# Copyright 2019 Catalyst Cloud Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re -from trove.guestagent.datastore.experimental.mariadb.service import MariaDBApp -from trove.guestagent.datastore.mysql.service import MySqlAppStatus -from trove.guestagent.strategies.backup import mysql_impl +from oslo_log import log as logging +from trove.guestagent.datastore.mysql import service as mysql_service +from trove.guestagent.datastore.mysql_common import service as common_service +from trove.guestagent.strategies.backup import base -class MariaDBInnoBackupEx(mysql_impl.InnoBackupEx): +LOG = logging.getLogger(__name__) +BACKUP_LOG = '/tmp/mariabackup.log' - def _build_app(self): - return MariaDBApp(MySqlAppStatus.get()) +class MariaBackup(base.BackupRunner): + """Implementation of Backup Strategy for mariabackup.""" + __strategy_name__ = 'mariabackup' -class MariaDBInnoBackupExIncremental(MariaDBInnoBackupEx): + @property + def user_and_pass(self): + return (' --user=%(user)s --password=%(password)s --host=127.0.0.1 ' % + {'user': common_service.ADMIN_USER_NAME, + 'password': mysql_service.MySqlApp.get_auth_password()}) + + @property + def cmd(self): + cmd = ('sudo mariabackup --backup --stream=xbstream' + + self.user_and_pass + ' 2>' + BACKUP_LOG) + return cmd + self.zip_cmd + self.encrypt_cmd + + def check_process(self): + """Check the output of mariabackup command for 'completed OK!'. + + Return True if no error, otherwise return False. + """ + LOG.debug('Checking mariabackup process output.') + + with open(BACKUP_LOG, 'r') as backup_log: + output = backup_log.read() + if not output: + LOG.error("mariabackup log file empty.") + return False + + LOG.debug(output) + + last_line = output.splitlines()[-1].strip() + if not re.search('completed OK!', last_line): + LOG.error("mariabackup command failed.") + return False + + return True + + @property + def filename(self): + return '%s.xbstream' % self.base_filename + + +class MariaBackupIncremental(MariaBackup): pass diff --git a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py index c3b82a44..30e73948 100644 --- a/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py +++ b/trove/guestagent/strategies/replication/experimental/mariadb_gtid.py @@ -17,16 +17,10 @@ from oslo_log import log as logging from trove.common import cfg -from trove.guestagent.backup.backupagent import BackupAgent from trove.guestagent.strategies import backup from trove.guestagent.strategies.replication import mysql_base -AGENT = BackupAgent() CONF = cfg.CONF - -REPL_BACKUP_NAMESPACE = 'trove.guestagent.strategies.backup' \ - '.experimental.mariadb_impl' - LOG = logging.getLogger(__name__) @@ -35,17 +29,27 @@ class MariaDBGTIDReplication(mysql_base.MysqlReplicationBase): @property def repl_backup_runner(self): - return backup.get_backup_strategy('MariaDBInnoBackupEx', - REPL_BACKUP_NAMESPACE) + return backup.get_backup_strategy( + CONF.mariadb.backup_strategy, + CONF.mariadb.backup_namespace + ) @property def repl_incr_backup_runner(self): - return backup.get_backup_strategy('MariaDBInnoBackupExIncremental', - REPL_BACKUP_NAMESPACE) + strategy = CONF.mariadb.backup_incremental_strategy.get( + CONF.mariadb.backup_strategy, CONF.mariadb.backup_strategy + ) + + return backup.get_backup_strategy( + strategy, + CONF.mariadb.backup_namespace + ) @property def repl_backup_extra_opts(self): - return CONF.backup_runner_options.get('MariaDBInnoBackupEx', '') + return CONF.backup_runner_options.get( + CONF.mariadb.backup_strategy, '' + ) def connect_to_master(self, service, snapshot): logging_config = snapshot['log_position'] diff --git a/trove/guestagent/strategies/replication/mysql_base.py b/trove/guestagent/strategies/replication/mysql_base.py index 549198a5..6a3843af 100644 --- a/trove/guestagent/strategies/replication/mysql_base.py +++ b/trove/guestagent/strategies/replication/mysql_base.py @@ -107,6 +107,7 @@ class MysqlReplicationBase(base.Replication): incremental_runner=self.repl_incr_backup_runner) else: LOG.debug("Using existing backup created for previous replica.") + LOG.debug("Replication snapshot %(snapshot_id)s used for replica " "number %(replica_number)d.", {'snapshot_id': snapshot_id, diff --git a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py index 9060d140..99339cbb 100644 --- a/trove/guestagent/strategies/restore/experimental/mariadb_impl.py +++ b/trove/guestagent/strategies/restore/experimental/mariadb_impl.py @@ -1,28 +1,59 @@ -# Copyright 2016 Tesora Inc. -# All Rights Reserved. +# Copyright 2019 Catalyst Cloud Ltd. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -from trove.guestagent.datastore.experimental.mariadb.service import MariaDBApp -from trove.guestagent.datastore.mysql.service import MySqlAppStatus +from oslo_log import log as logging + +from trove.guestagent.common import operating_system +from trove.guestagent.datastore.experimental.mariadb import service +from trove.guestagent.datastore.mysql_common import service as mysql_service from trove.guestagent.strategies.restore import mysql_impl +LOG = logging.getLogger(__name__) + + +class MariaBackup(mysql_impl.InnoBackupEx): + __strategy_name__ = 'mariabackup' + base_restore_cmd = ('sudo mbstream -x -C %(restore_location)s ' + '2>/tmp/xbstream_extract.log') + + @property + def app(self): + if self._app is None: + self._app = service.MariaDBApp( + mysql_service.BaseMySqlAppStatus.get() + ) + return self._app + + def post_restore(self): + operating_system.chown(self.restore_location, 'mysql', None, + force=True, as_root=True) + + # When using Mariabackup from versions prior to MariaDB 10.2.10, you + # would also have to remove any pre-existing InnoDB redo log files. + self._delete_old_binlogs() + self.app.start_mysql() + LOG.debug("Finished post restore.") -class MariaDBInnoBackupEx(mysql_impl.InnoBackupEx): + def check_process(self): + LOG.debug('Checking return code of mbstream restore process.') + return_code = self.process.wait() + if return_code != 0: + LOG.error('mbstream exited with %s', return_code) + return False - def _build_app(self): - return MariaDBApp(MySqlAppStatus.get()) + return True -class MariaDBInnoBackupExIncremental(MariaDBInnoBackupEx): +class MariaBackupIncremental(MariaBackup): pass diff --git a/trove/guestagent/strategies/restore/mysql_impl.py b/trove/guestagent/strategies/restore/mysql_impl.py index f42ad59a..cf4529ab 100644 --- a/trove/guestagent/strategies/restore/mysql_impl.py +++ b/trove/guestagent/strategies/restore/mysql_impl.py @@ -57,7 +57,7 @@ class MySQLRestoreMixin(object): def mysql_is_not_running(self): try: utils.execute_with_timeout("/usr/bin/pgrep", "mysqld") - LOG.info("MySQL is still running.") + LOG.debug("MySQL is still running.") return False except exception.ProcessExecutionError: LOG.debug("MySQL is not running.") @@ -218,7 +218,7 @@ class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin): utils.clean_out(self.restore_location) def _run_prepare(self): - LOG.debug("Running innobackupex prepare: %s.", self.prepare_cmd) + LOG.info("Running innobackupex prepare: %s.", self.prepare_cmd) self.prep_retcode = utils.execute(self.prepare_cmd, shell=True) LOG.info("Innobackupex prepare finished successfully.") @@ -247,7 +247,7 @@ class InnoBackupEx(base.RestoreRunner, MySQLRestoreMixin): LOG.debug('Checking return code of xbstream restore process.') return_code = self.process.wait() if return_code != 0: - LOG.erro('xbstream exited with %s', return_code) + LOG.error('xbstream exited with %s', return_code) return False LOG.debug('Checking xbstream restore process stderr output.') diff --git a/trove/taskmanager/manager.py b/trove/taskmanager/manager.py index ad1a04c9..b103704b 100644 --- a/trove/taskmanager/manager.py +++ b/trove/taskmanager/manager.py @@ -330,8 +330,8 @@ class Manager(periodic_task.PeriodicTasks): master_instance_tasks = BuiltInstanceTasks.load(context, slave_of_id) server_group = master_instance_tasks.server_group scheduler_hints = srv_grp.ServerGroup.convert_to_hint(server_group) - LOG.info("Using scheduler hints %s for creating instance %s", - scheduler_hints, instance_id) + LOG.debug("Using scheduler hints %s for creating instance %s", + scheduler_hints, instance_id) try: for replica_index in range(0, len(ids)): @@ -344,14 +344,17 @@ class Manager(periodic_task.PeriodicTasks): snapshot = instance_tasks.get_replication_master_snapshot( context, slave_of_id, flavor, replica_backup_id, replica_number=replica_number) + replica_backup_id = snapshot['dataset']['snapshot_id'] replica_backup_created = (replica_backup_id is not None) + instance_tasks.create_instance( flavor, image_id, databases, users, datastore_manager, packages, volume_size, replica_backup_id, availability_zone, root_passwords[replica_index], nics, overrides, None, snapshot, volume_type, modules, scheduler_hints) + replicas.append(instance_tasks) except Exception: # if it's the first replica, then we shouldn't continue @@ -390,8 +393,8 @@ class Manager(periodic_task.PeriodicTasks): scheduler_hints = srv_grp.ServerGroup.build_scheduler_hint( context, locality, instance_id ) - LOG.info("Using scheduler hints %s for creating instance %s", - scheduler_hints, instance_id) + LOG.debug("Using scheduler hints %s for creating instance %s", + scheduler_hints, instance_id) instance_tasks = FreshInstanceTasks.load(context, instance_id) instance_tasks.create_instance( diff --git a/trove/taskmanager/models.py b/trove/taskmanager/models.py index 403d8dce..bf8a1024 100755 --- a/trove/taskmanager/models.py +++ b/trove/taskmanager/models.py @@ -424,7 +424,8 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): error_message = '' error_details = '' try: - LOG.info("Waiting for instance %s up and running", self.id) + LOG.info("Waiting for instance %s up and running with " + "timeout %ss", self.id, timeout) utils.poll_until(self._service_is_active, sleep_time=CONF.usage_sleep_time, time_out=timeout) @@ -621,7 +622,8 @@ class FreshInstanceTasks(FreshInstance, NotifyMixin, ConfigurationMixin): if backup: backup_id = backup.id else: - LOG.debug('Skipping replication backup, as none is required.') + LOG.debug('Will skip replication master backup') + snapshot_info = { 'name': "Replication snapshot for %s" % self.id, 'description': "Backup image used to initialize " @@ -1094,7 +1096,7 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin): self.guest.create_backup(backup_info) def backup_required_for_replication(self): - LOG.debug("Seeing if replication backup is required for instance %s.", + LOG.debug("Check if replication backup is required for instance %s.", self.id) return self.guest.backup_required_for_replication() @@ -1106,7 +1108,9 @@ class BuiltInstanceTasks(BuiltInstance, NotifyMixin, ConfigurationMixin): rep_source_config = self._render_replica_source_config(flavor) result = self.guest.get_replication_snapshot( snapshot_info, rep_source_config.config_contents) - LOG.debug("Got replication snapshot from guest successfully.") + + LOG.info("Finnished getting replication snapshot for " + "instance %s", self.id) return result except Exception: LOG.exception("Failed to get replication snapshot from %s.", diff --git a/trove/templates/mariadb/config.template b/trove/templates/mariadb/config.template index 8507e0f3..aa10164e 100644 --- a/trove/templates/mariadb/config.template +++ b/trove/templates/mariadb/config.template @@ -6,8 +6,6 @@ socket = /var/run/mysqld/mysqld.sock nice = 0 [mysqld] -ignore_builtin_innodb -plugin_load=innodb=ha_innodb.so user = mysql port = 3306 basedir = /usr diff --git a/trove/tests/int_tests.py b/trove/tests/int_tests.py index 6e598ece..c662028a 100644 --- a/trove/tests/int_tests.py +++ b/trove/tests/int_tests.py @@ -318,7 +318,8 @@ register( register( ["mariadb_supported"], single=[common_groups, - backup_incremental_groups, + backup_groups, + # backup_incremental_groups, configuration_groups, database_actions_groups, root_actions_groups, diff --git a/trove/tests/unittests/guestagent/test_dbaas.py b/trove/tests/unittests/guestagent/test_dbaas.py index b2de1527..2d4b88b5 100644 --- a/trove/tests/unittests/guestagent/test_dbaas.py +++ b/trove/tests/unittests/guestagent/test_dbaas.py @@ -158,7 +158,7 @@ class DbaasTest(trove_testtools.TestCase): with patch.object(mysql_common_service.utils, 'execute', return_value=(secret_content, None)): mysql_common_service.clear_expired_password() - self.assertEqual(2, mysql_common_service.utils.execute.call_count) + self.assertEqual(3, mysql_common_service.utils.execute.call_count) self.assertEqual(1, mock_remove.call_count) @patch.object(operating_system, 'remove') @@ -166,7 +166,7 @@ class DbaasTest(trove_testtools.TestCase): with patch.object(mysql_common_service.utils, 'execute', return_value=('', None)): mysql_common_service.clear_expired_password() - self.assertEqual(1, mysql_common_service.utils.execute.call_count) + self.assertEqual(2, mysql_common_service.utils.execute.call_count) mock_remove.assert_not_called() @patch.object(operating_system, 'remove') @@ -184,16 +184,14 @@ class DbaasTest(trove_testtools.TestCase): self.assertEqual(2, mysql_common_service.utils.execute.call_count) mock_remove.assert_not_called() - @patch('trove.guestagent.datastore.mysql_common.service.LOG') @patch.object(operating_system, 'remove') @patch.object(mysql_common_service.utils, 'execute', - side_effect=ProcessExecutionError) + side_effect=[ProcessExecutionError, (None, None)]) def test_fail_retrieve_secret_content_clear_expired_password(self, mock_execute, - mock_remove, - mock_logging): + mock_remove): mysql_common_service.clear_expired_password() - self.assertEqual(1, mock_execute.call_count) + self.assertEqual(2, mock_execute.call_count) mock_remove.assert_not_called() @patch.object(operating_system, 'read_file', -- cgit v1.2.1