diff options
author | Jan Lindström <jan.lindstrom@skysql.com> | 2014-08-06 15:39:15 +0300 |
---|---|---|
committer | Jan Lindström <jan.lindstrom@skysql.com> | 2014-08-26 15:43:46 +0300 |
commit | df4dd593f29aec8e2116aec1775ad4b8833d8c93 (patch) | |
tree | becae67f02054e15ead58e929e91c044f0b7aa15 /scripts | |
parent | e974b564389af8251c2ba51060e6129e45431586 (diff) | |
download | mariadb-git-df4dd593f29aec8e2116aec1775ad4b8833d8c93.tar.gz |
MDEV-6247: Merge 10.0-galera to 10.1.
Merged lp:maria/maria-10.0-galera up to revision 3879.
Added a new functions to handler API to forcefully abort_transaction,
producing fake_trx_id, get_checkpoint and set_checkpoint for XA. These
were added for future possiblity to add more storage engines that
could use galera replication.
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/CMakeLists.txt | 10 | ||||
-rw-r--r-- | scripts/mysqld_multi.sh | 8 | ||||
-rw-r--r-- | scripts/mysqld_safe.sh | 122 | ||||
-rwxr-xr-x | scripts/wsrep_sst_common | 157 | ||||
-rw-r--r-- | scripts/wsrep_sst_common.sh | 157 | ||||
-rwxr-xr-x | scripts/wsrep_sst_mysqldump | 131 | ||||
-rw-r--r-- | scripts/wsrep_sst_mysqldump.sh | 131 | ||||
-rwxr-xr-x | scripts/wsrep_sst_rsync | 335 | ||||
-rw-r--r-- | scripts/wsrep_sst_rsync.sh | 335 | ||||
-rwxr-xr-x | scripts/wsrep_sst_xtrabackup | 715 | ||||
-rwxr-xr-x | scripts/wsrep_sst_xtrabackup-v2 | 930 | ||||
-rw-r--r-- | scripts/wsrep_sst_xtrabackup-v2.sh | 930 | ||||
-rw-r--r-- | scripts/wsrep_sst_xtrabackup.sh | 715 |
13 files changed, 4673 insertions, 3 deletions
diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index d3a7861269c..1f0f28061ed 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -327,6 +327,15 @@ IF(WIN32) INSTALL_SCRIPT(${CMAKE_CURRENT_BINARY_DIR}/${file}.pl COMPONENT Server_Scripts) ENDFOREACH() ELSE() + IF(WITH_WSREP) + SET(WSREP_BINARIES + wsrep_sst_common + wsrep_sst_mysqldump + wsrep_sst_rsync + wsrep_sst_xtrabackup + wsrep_sst_xtrabackup-v2 + ) + ENDIF() # Configure this one, for testing, but do not install it. CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/mysql_config.pl.in ${CMAKE_CURRENT_BINARY_DIR}/mysql_config.pl ESCAPE_QUOTES @ONLY) @@ -346,6 +355,7 @@ ELSE() mysqldumpslow mysqld_multi mysqld_safe + ${WSREP_BINARIES} ) FOREACH(file ${BIN_SCRIPTS}) IF(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${file}.sh) diff --git a/scripts/mysqld_multi.sh b/scripts/mysqld_multi.sh index cd1b6fc18b7..93ab717c8ae 100644 --- a/scripts/mysqld_multi.sh +++ b/scripts/mysqld_multi.sh @@ -38,7 +38,7 @@ use Getopt::Long; use POSIX qw(strftime getcwd); $|=1; -$VER="2.16"; +$VER="2.20"; my @defaults_options; # Leading --no-defaults, --defaults-file, etc. @@ -138,6 +138,7 @@ sub main print "will be disabled\nand some will be enabled.\n\n"; } + init_log() if (!defined($opt_log)); $groupids = $ARGV[1]; if ($opt_version) { @@ -163,7 +164,6 @@ sub main !($ARGV[0] =~ m/^stop$/i) && !($ARGV[0] =~ m/^report$/i))); - init_log() if (!defined($opt_log)); if (!$opt_no_log) { w2log("$my_progname log file version $VER; run: ", @@ -218,6 +218,10 @@ sub quote_shell_word return $option; } +#### +#### get options for a group +#### + sub defaults_for_group { my ($group) = @_; diff --git a/scripts/mysqld_safe.sh b/scripts/mysqld_safe.sh index 9d0dd0e68ac..4e7b098c8d8 100644 --- a/scripts/mysqld_safe.sh +++ b/scripts/mysqld_safe.sh @@ -189,6 +189,89 @@ shell_quote_string() { echo "$1" | sed -e 's,\([^a-zA-Z0-9/_.=-]\),\\\1,g' } +wsrep_pick_url() { + [ $# -eq 0 ] && return 0 + + log_error "WSREP: 'wsrep_urls' is DEPRECATED! Use wsrep_cluster_address to specify multiple addresses instead." + + if ! which nc >/dev/null; then + log_error "ERROR: nc tool not found in PATH! Make sure you have it installed." + return 1 + fi + + local url + # Assuming URL in the form scheme://host:port + # If host and port are not NULL, the liveness of URL is assumed to be tested + # If port part is absent, the url is returned literally and unconditionally + # If every URL has port but none is reachable, nothing is returned + for url in `echo $@ | sed s/,/\ /g` 0; do + local host=`echo $url | cut -d \: -f 2 | sed s/^\\\/\\\///` + local port=`echo $url | cut -d \: -f 3` + [ -z "$port" ] && break + nc -z "$host" $port >/dev/null && break + done + + if [ "$url" == "0" ]; then + log_error "ERROR: none of the URLs in '$@' is reachable." + return 1 + fi + + echo $url +} + +# Run mysqld with --wsrep-recover and parse recovered position from log. +# Position will be stored in wsrep_start_position_opt global. +wsrep_start_position_opt="" +wsrep_recover_position() { + local mysqld_cmd="$@" + local euid=$(id -u) + local ret=0 + + local wr_logfile=$(mktemp $DATADIR/wsrep_recovery.XXXXXX) + + # safety checks + if [ -z $wr_logfile ]; then + log_error "WSREP: mktemp failed" + return 1 + fi + + if [ -f $wr_logfile ]; then + [ "$euid" = "0" ] && chown $user $wr_logfile + chmod 600 $wr_logfile + else + log_error "WSREP: mktemp failed" + return 1 + fi + + local wr_pidfile="$DATADIR/"`@HOSTNAME@`"-recover.pid" + + local wr_options="--log_error='$wr_logfile' --pid-file='$wr_pidfile'" + + log_notice "WSREP: Running position recovery with $wr_options" + + eval_log_error "$mysqld_cmd --wsrep_recover $wr_options" + + local rp="$(grep 'WSREP: Recovered position:' $wr_logfile)" + if [ -z "$rp" ]; then + local skipped="$(grep WSREP $wr_logfile | grep 'skipping position recovery')" + if [ -z "$skipped" ]; then + log_error "WSREP: Failed to recover position: '`cat $wr_logfile`'" + ret=1 + else + log_notice "WSREP: Position recovery skipped" + fi + else + local start_pos="$(echo $rp | sed 's/.*WSREP\:\ Recovered\ position://' \ + | sed 's/^[ \t]*//')" + log_notice "WSREP: Recovered position $start_pos" + wsrep_start_position_opt="--wsrep_start_position=$start_pos" + fi + + [ $ret -eq 0 ] && rm $wr_logfile + + return $ret +} + parse_arguments() { # We only need to pass arguments through to the server if we don't # handle them here. So, we collect unrecognized options (passed on @@ -245,6 +328,14 @@ parse_arguments() { --timezone=*) TZ="$val"; export TZ; ;; --flush[-_]caches) flush_caches=1 ;; --numa[-_]interleave) numa_interleave=1 ;; + --wsrep[-_]urls=*) wsrep_urls="$val"; ;; + --wsrep[-_]provider=*) + if test -n "$val" && test "$val" != "none" + then + wsrep_restart=1 + fi + append_arg_to_args "$arg" + ;; --help) usage ;; @@ -860,13 +951,28 @@ max_fast_restarts=5 # flag whether a usable sleep command exists have_sleep=1 +# maximum number of wsrep restarts +max_wsrep_restarts=0 + while true do rm -f "$pid_file" # Some extra safety start_time=`date +%M%S` - eval_log_error "$cmd" + # this sets wsrep_start_position_opt + wsrep_recover_position "$cmd" + + [ $? -ne 0 ] && exit 1 # + + [ -n "$wsrep_urls" ] && url=`wsrep_pick_url $wsrep_urls` # check connect address + + if [ -z "$url" ] + then + eval_log_error "$cmd $wsrep_start_position_opt $nohup_redir" + else + eval_log_error "$cmd $wsrep_start_position_opt --wsrep_cluster_address=$url $nohup_redir" + fi if [ $want_syslog -eq 0 -a ! -f "$err_log" ]; then touch "$err_log" # hypothetical: log was renamed but not @@ -936,6 +1042,20 @@ do I=`expr $I + 1` done fi + + if [ -n "$wsrep_restart" ] + then + if [ $wsrep_restart -le $max_wsrep_restarts ] + then + wsrep_restart=`expr $wsrep_restart + 1` + log_notice "WSREP: sleeping 15 seconds before restart" + sleep 15 + else + log_notice "WSREP: not restarting wsrep node automatically" + break + fi + fi + log_notice "mysqld restarted" if test -n "$CRASH_SCRIPT" then diff --git a/scripts/wsrep_sst_common b/scripts/wsrep_sst_common new file mode 100755 index 00000000000..88f5d80f53a --- /dev/null +++ b/scripts/wsrep_sst_common @@ -0,0 +1,157 @@ +# Copyright (C) 2012-2014 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a common command line parser to be sourced by other SST scripts + +set -u + +WSREP_SST_OPT_BYPASS=0 +WSREP_SST_OPT_BINLOG="" +WSREP_SST_OPT_DATA="" +WSREP_SST_OPT_AUTH="" + +while [ $# -gt 0 ]; do +case "$1" in + '--address') + readonly WSREP_SST_OPT_ADDR="$2" + shift + ;; + '--auth') + WSREP_SST_OPT_AUTH="$2" + shift + ;; + '--bypass') + WSREP_SST_OPT_BYPASS=1 + ;; + '--datadir') + readonly WSREP_SST_OPT_DATA="$2" + shift + ;; + '--defaults-file') + readonly WSREP_SST_OPT_CONF="$2" + shift + ;; + '--host') + readonly WSREP_SST_OPT_HOST="$2" + shift + ;; + '--local-port') + readonly WSREP_SST_OPT_LPORT="$2" + shift + ;; + '--parent') + readonly WSREP_SST_OPT_PARENT="$2" + shift + ;; + '--password') + WSREP_SST_OPT_PSWD="$2" + shift + ;; + '--port') + readonly WSREP_SST_OPT_PORT="$2" + shift + ;; + '--role') + readonly WSREP_SST_OPT_ROLE="$2" + shift + ;; + '--socket') + readonly WSREP_SST_OPT_SOCKET="$2" + shift + ;; + '--user') + WSREP_SST_OPT_USER="$2" + shift + ;; + '--gtid') + readonly WSREP_SST_OPT_GTID="$2" + shift + ;; + '--binlog') + WSREP_SST_OPT_BINLOG="$2" + shift + ;; + *) # must be command + # usage + # exit 1 + ;; +esac +shift +done +readonly WSREP_SST_OPT_BYPASS +readonly WSREP_SST_OPT_BINLOG + +# State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727. +if my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth";then + if [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ];then + WSREP_SST_OPT_AUTH=$(my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2) + fi +fi + +if [ -n "${WSREP_SST_OPT_DATA:-}" ] +then + SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" +else + SST_PROGRESS_FILE="" +fi + + +wsrep_log() +{ + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + echo "$tst WSREP_SST: " >&2 +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + +wsrep_cleanup_progress_file() +{ + [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null +} + +wsrep_check_program() +{ + local prog=$1 + + if ! which $prog >/dev/null + then + echo "'$prog' not found in PATH" + return 2 # no such file or directory + fi +} + +wsrep_check_programs() +{ + local ret=0 + + while [ $# -gt 0 ] + do + wsrep_check_program $1 || ret=$? + shift + done + + return $ret +} diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh new file mode 100644 index 00000000000..88f5d80f53a --- /dev/null +++ b/scripts/wsrep_sst_common.sh @@ -0,0 +1,157 @@ +# Copyright (C) 2012-2014 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a common command line parser to be sourced by other SST scripts + +set -u + +WSREP_SST_OPT_BYPASS=0 +WSREP_SST_OPT_BINLOG="" +WSREP_SST_OPT_DATA="" +WSREP_SST_OPT_AUTH="" + +while [ $# -gt 0 ]; do +case "$1" in + '--address') + readonly WSREP_SST_OPT_ADDR="$2" + shift + ;; + '--auth') + WSREP_SST_OPT_AUTH="$2" + shift + ;; + '--bypass') + WSREP_SST_OPT_BYPASS=1 + ;; + '--datadir') + readonly WSREP_SST_OPT_DATA="$2" + shift + ;; + '--defaults-file') + readonly WSREP_SST_OPT_CONF="$2" + shift + ;; + '--host') + readonly WSREP_SST_OPT_HOST="$2" + shift + ;; + '--local-port') + readonly WSREP_SST_OPT_LPORT="$2" + shift + ;; + '--parent') + readonly WSREP_SST_OPT_PARENT="$2" + shift + ;; + '--password') + WSREP_SST_OPT_PSWD="$2" + shift + ;; + '--port') + readonly WSREP_SST_OPT_PORT="$2" + shift + ;; + '--role') + readonly WSREP_SST_OPT_ROLE="$2" + shift + ;; + '--socket') + readonly WSREP_SST_OPT_SOCKET="$2" + shift + ;; + '--user') + WSREP_SST_OPT_USER="$2" + shift + ;; + '--gtid') + readonly WSREP_SST_OPT_GTID="$2" + shift + ;; + '--binlog') + WSREP_SST_OPT_BINLOG="$2" + shift + ;; + *) # must be command + # usage + # exit 1 + ;; +esac +shift +done +readonly WSREP_SST_OPT_BYPASS +readonly WSREP_SST_OPT_BINLOG + +# State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727. +if my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth";then + if [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ];then + WSREP_SST_OPT_AUTH=$(my_print_defaults -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2) + fi +fi + +if [ -n "${WSREP_SST_OPT_DATA:-}" ] +then + SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" +else + SST_PROGRESS_FILE="" +fi + + +wsrep_log() +{ + # echo everything to stderr so that it gets into common error log + # deliberately made to look different from the rest of the log + local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)" + echo "$tst WSREP_SST: " >&2 +} + +wsrep_log_error() +{ + wsrep_log "[ERROR] $*" +} + +wsrep_log_info() +{ + wsrep_log "[INFO] $*" +} + +wsrep_cleanup_progress_file() +{ + [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null +} + +wsrep_check_program() +{ + local prog=$1 + + if ! which $prog >/dev/null + then + echo "'$prog' not found in PATH" + return 2 # no such file or directory + fi +} + +wsrep_check_programs() +{ + local ret=0 + + while [ $# -gt 0 ] + do + wsrep_check_program $1 || ret=$? + shift + done + + return $ret +} diff --git a/scripts/wsrep_sst_mysqldump b/scripts/wsrep_sst_mysqldump new file mode 100755 index 00000000000..d4093dbcaef --- /dev/null +++ b/scripts/wsrep_sst_mysqldump @@ -0,0 +1,131 @@ +#!/bin/bash -e +# Copyright (C) 2009 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a reference script for mysqldump-based state snapshot tansfer + +# This variable is not used in mysqldump sst, so better initialize it +# to avoid shell's "parameter not set" message. +WSREP_SST_OPT_CONF="" + +. $(dirname $0)/wsrep_sst_common + +EINVAL=22 +PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin + +local_ip() +{ + [ "$1" = "127.0.0.1" ] && return 0 + [ "$1" = "localhost" ] && return 0 + [ "$1" = "$(hostname -s)" ] && return 0 + [ "$1" = "$(hostname -f)" ] && return 0 + [ "$1" = "$(hostname -d)" ] && return 0 + + # Now if ip program is not found in the path, we can't return 0 since + # it would block any address. Thankfully grep should fail in this case + ip route get "$1" | grep local >/dev/null && return 0 + + return 1 +} + +if test -z "$WSREP_SST_OPT_USER"; then wsrep_log_error "USER cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_HOST"; then wsrep_log_error "HOST cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_PORT"; then wsrep_log_error "PORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_LPORT"; then wsrep_log_error "LPORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_SOCKET";then wsrep_log_error "SOCKET cannot be nil";exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_GTID"; then wsrep_log_error "GTID cannot be nil"; exit $EINVAL; fi + +if local_ip $WSREP_SST_OPT_HOST && \ + [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ] +then + wsrep_log_error \ + "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address." + exit $EINVAL +fi + +# Check client version +if ! mysql --version | grep 'Distrib 10' >/dev/null +then + mysql --version >&2 + wsrep_log_error "this operation requires MySQL client version 10 or newer" + exit $EINVAL +fi + +# For Bug:1293798 +if [ -z "$WSREP_SST_OPT_PSWD" -a -n "$WSREP_SST_OPT_AUTH" ]; then + WSREP_SST_OPT_USER=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f1) + WSREP_SST_OPT_PSWD=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f2) +fi +AUTH="-u$WSREP_SST_OPT_USER" +if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi + +STOP_WSREP="SET wsrep_on=OFF;" + +# NOTE: we don't use --routines here because we're dumping mysql.proc table +MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \ +--add-drop-database --add-drop-table --skip-add-locks --create-options \ +--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ +--skip-comments --flush-privileges --all-databases" + +# mysqldump cannot restore CSV tables, fix this issue +CSV_TABLES_FIX=" +set sql_mode=''; + +USE mysql; + +SET @cond = (SELECT (SUPPORT = 'YES' or SUPPORT = 'DEFAULT') FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE = 'csv'); + +SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"General log\"', 'SET @dummy = 0'); + +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DROP PREPARE stmt; + +SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"Slow log\"', 'SET @dummy = 0'); + +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DROP PREPARE stmt;" + +SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" + +MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\ +"--disable-reconnect --connect_timeout=10" + +# need to disable logging when loading the dump +# reason is that dump contains ALTER TABLE for log tables, and +# this causes an error if logging is enabled +GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@GENERAL_LOG"` +SLOW_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@SLOW_QUERY_LOG"` +$MYSQL -e"$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF" +$MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF" + +# commands to restore log settings +RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;" +RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;" + +if [ $WSREP_SST_OPT_BYPASS -eq 0 ] +then + (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \ + && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \ + && echo $SET_START_POSITION \ + || echo "SST failed to complete;") | $MYSQL +else + wsrep_log_info "Bypassing state dump." + echo $SET_START_POSITION | $MYSQL +fi + +# diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh new file mode 100644 index 00000000000..d4093dbcaef --- /dev/null +++ b/scripts/wsrep_sst_mysqldump.sh @@ -0,0 +1,131 @@ +#!/bin/bash -e +# Copyright (C) 2009 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a reference script for mysqldump-based state snapshot tansfer + +# This variable is not used in mysqldump sst, so better initialize it +# to avoid shell's "parameter not set" message. +WSREP_SST_OPT_CONF="" + +. $(dirname $0)/wsrep_sst_common + +EINVAL=22 +PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin + +local_ip() +{ + [ "$1" = "127.0.0.1" ] && return 0 + [ "$1" = "localhost" ] && return 0 + [ "$1" = "$(hostname -s)" ] && return 0 + [ "$1" = "$(hostname -f)" ] && return 0 + [ "$1" = "$(hostname -d)" ] && return 0 + + # Now if ip program is not found in the path, we can't return 0 since + # it would block any address. Thankfully grep should fail in this case + ip route get "$1" | grep local >/dev/null && return 0 + + return 1 +} + +if test -z "$WSREP_SST_OPT_USER"; then wsrep_log_error "USER cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_HOST"; then wsrep_log_error "HOST cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_PORT"; then wsrep_log_error "PORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_LPORT"; then wsrep_log_error "LPORT cannot be nil"; exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_SOCKET";then wsrep_log_error "SOCKET cannot be nil";exit $EINVAL; fi +if test -z "$WSREP_SST_OPT_GTID"; then wsrep_log_error "GTID cannot be nil"; exit $EINVAL; fi + +if local_ip $WSREP_SST_OPT_HOST && \ + [ "$WSREP_SST_OPT_PORT" = "$WSREP_SST_OPT_LPORT" ] +then + wsrep_log_error \ + "destination address '$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT' matches source address." + exit $EINVAL +fi + +# Check client version +if ! mysql --version | grep 'Distrib 10' >/dev/null +then + mysql --version >&2 + wsrep_log_error "this operation requires MySQL client version 10 or newer" + exit $EINVAL +fi + +# For Bug:1293798 +if [ -z "$WSREP_SST_OPT_PSWD" -a -n "$WSREP_SST_OPT_AUTH" ]; then + WSREP_SST_OPT_USER=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f1) + WSREP_SST_OPT_PSWD=$(echo $WSREP_SST_OPT_AUTH | cut -d: -f2) +fi +AUTH="-u$WSREP_SST_OPT_USER" +if test -n "$WSREP_SST_OPT_PSWD"; then AUTH="$AUTH -p$WSREP_SST_OPT_PSWD"; fi + +STOP_WSREP="SET wsrep_on=OFF;" + +# NOTE: we don't use --routines here because we're dumping mysql.proc table +MYSQLDUMP="mysqldump $AUTH -S$WSREP_SST_OPT_SOCKET \ +--add-drop-database --add-drop-table --skip-add-locks --create-options \ +--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \ +--skip-comments --flush-privileges --all-databases" + +# mysqldump cannot restore CSV tables, fix this issue +CSV_TABLES_FIX=" +set sql_mode=''; + +USE mysql; + +SET @cond = (SELECT (SUPPORT = 'YES' or SUPPORT = 'DEFAULT') FROM INFORMATION_SCHEMA.ENGINES WHERE ENGINE = 'csv'); + +SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS general_log ( event_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL, server_id int(10) unsigned NOT NULL, command_type varchar(64) NOT NULL, argument mediumtext NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"General log\"', 'SET @dummy = 0'); + +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DROP PREPARE stmt; + +SET @stmt = IF (@cond = '1', 'CREATE TABLE IF NOT EXISTS slow_log ( start_time timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6), user_host mediumtext NOT NULL, query_time time(6) NOT NULL, lock_time time(6) NOT NULL, rows_sent int(11) NOT NULL, rows_examined int(11) NOT NULL, db varchar(512) NOT NULL, last_insert_id int(11) NOT NULL, insert_id int(11) NOT NULL, server_id int(10) unsigned NOT NULL, sql_text mediumtext NOT NULL, thread_id bigint(21) unsigned NOT NULL) ENGINE=CSV DEFAULT CHARSET=utf8 COMMENT=\"Slow log\"', 'SET @dummy = 0'); + +PREPARE stmt FROM @stmt; +EXECUTE stmt; +DROP PREPARE stmt;" + +SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" + +MYSQL="mysql $AUTH -h$WSREP_SST_OPT_HOST -P$WSREP_SST_OPT_PORT "\ +"--disable-reconnect --connect_timeout=10" + +# need to disable logging when loading the dump +# reason is that dump contains ALTER TABLE for log tables, and +# this causes an error if logging is enabled +GENERAL_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@GENERAL_LOG"` +SLOW_LOG_OPT=`$MYSQL --skip-column-names -e"$STOP_WSREP SELECT @@SLOW_QUERY_LOG"` +$MYSQL -e"$STOP_WSREP SET GLOBAL GENERAL_LOG=OFF" +$MYSQL -e"$STOP_WSREP SET GLOBAL SLOW_QUERY_LOG=OFF" + +# commands to restore log settings +RESTORE_GENERAL_LOG="SET GLOBAL GENERAL_LOG=$GENERAL_LOG_OPT;" +RESTORE_SLOW_QUERY_LOG="SET GLOBAL SLOW_QUERY_LOG=$SLOW_LOG_OPT;" + +if [ $WSREP_SST_OPT_BYPASS -eq 0 ] +then + (echo $STOP_WSREP && $MYSQLDUMP && echo $CSV_TABLES_FIX \ + && echo $RESTORE_GENERAL_LOG && echo $RESTORE_SLOW_QUERY_LOG \ + && echo $SET_START_POSITION \ + || echo "SST failed to complete;") | $MYSQL +else + wsrep_log_info "Bypassing state dump." + echo $SET_START_POSITION | $MYSQL +fi + +# diff --git a/scripts/wsrep_sst_rsync b/scripts/wsrep_sst_rsync new file mode 100755 index 00000000000..86bf557662d --- /dev/null +++ b/scripts/wsrep_sst_rsync @@ -0,0 +1,335 @@ +#!/bin/bash -ue + +# Copyright (C) 2010-2014 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a reference script for rsync-based state snapshot tansfer + +RSYNC_PID= +RSYNC_CONF= +OS=$(uname) +[ "$OS" == "Darwin" ] && export -n LD_LIBRARY_PATH + +# Setting the path for lsof on CentOS +export PATH="/usr/sbin:/sbin:$PATH" + +. $(dirname $0)/wsrep_sst_common + +wsrep_check_programs rsync + +cleanup_joiner() +{ + wsrep_log_info "Joiner cleanup." + local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0) + [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \ + || : + rm -rf "$RSYNC_CONF" + rm -rf "$MAGIC_FILE" + rm -rf "$RSYNC_PID" + wsrep_log_info "Joiner cleanup done." + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_cleanup_progress_file + fi +} + +check_pid() +{ + local pid_file=$1 + [ -r "$pid_file" ] && ps -p $(cat $pid_file) >/dev/null 2>&1 +} + +check_pid_and_port() +{ + local pid_file=$1 + local rsync_pid=$2 + local rsync_port=$3 + + if ! which lsof > /dev/null; then + wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed." + exit 2 # ENOENT + fi + + local port_info=$(lsof -i :$rsync_port -Pn 2>/dev/null | \ + grep "(LISTEN)") + local is_rsync=$(echo $port_info | \ + grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null) + + if [ -n "$port_info" -a -z "$is_rsync" ]; then + wsrep_log_error "rsync daemon port '$rsync_port' has been taken" + exit 16 # EBUSY + fi + check_pid $pid_file && \ + [ -n "$port_info" ] && [ -n "$is_rsync" ] && \ + [ $(cat $pid_file) -eq $rsync_pid ] +} + +MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" +rm -rf "$MAGIC_FILE" + +BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" +BINLOG_N_FILES=1 +rm -f "$BINLOG_TAR_FILE" || : + +if ! [ -z $WSREP_SST_OPT_BINLOG ] +then + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) +fi + +WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} +# if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf +if [ -z "$WSREP_LOG_DIR" ]; then + SCRIPT_DIR="$(cd $(dirname "$0"); pwd -P)" + WSREP_LOG_DIR=$($SCRIPT_DIR/my_print_defaults --defaults-file \ + "$WSREP_SST_OPT_CONF" mysqld server mysqld-10.0 mariadb mariadb-10.0 \ + | grep -- '--innodb[-_]log[-_]group[-_]home[-_]dir=' \ + | cut -b 29- ) +fi + +if [ -n "$WSREP_LOG_DIR" ]; then + # handle both relative and absolute paths + WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$WSREP_LOG_DIR"; cd $WSREP_LOG_DIR; pwd -P) +else + # default to datadir + WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P) +fi + +# Old filter - include everything except selected +# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \ +# --exclude '*.conf' --exclude core --exclude 'galera.*' \ +# --exclude grastate.txt --exclude '*.pem' \ +# --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index') + +# New filter - exclude everything except dirs (schemas) and innodb files +FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' + -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*') + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" + rm -rf "$FLUSHED" + + # Use deltaxfer only for WAN + inv=$(basename $0) + [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \ + || WHOLE_FILE_OPT="--whole-file" + + echo "flush tables" + + # wait for tables flushed and state ID written to the file + while [ ! -r "$FLUSHED" ] && ! grep -q ':' "$FLUSHED" >/dev/null 2>&1 + do + sleep 0.2 + done + + STATE="$(cat $FLUSHED)" + rm -rf "$FLUSHED" + + sync + + if ! [ -z $WSREP_SST_OPT_BINLOG ] + then + # Prepare binlog files + pushd $BINLOG_DIRNAME &> /dev/null + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + binlog_files="" + for ii in $binlog_files_full + do + binlog_files="$binlog_files $(basename $ii)" + done + if ! [ -z "$binlog_files" ] + then + wsrep_log_info "Preparing binlog files for transfer:" + tar -cvf $BINLOG_TAR_FILE $binlog_files >&2 + fi + popd &> /dev/null + fi + + # first, the normal directories, so that we can detect incompatible protocol + RC=0 + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA/" \ + rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$? + + if [ "$RC" -ne 0 ]; then + wsrep_log_error "rsync returned code $RC:" + + case $RC in + 12) RC=71 # EPROTO + wsrep_log_error \ + "rsync server on the other end has incompatible protocol. " \ + "Make sure you have the same version of rsync on all nodes." + ;; + 22) RC=12 # ENOMEM + ;; + *) RC=255 # unknown error + ;; + esac + exit $RC + fi + + # second, we transfer InnoDB log files + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \ + rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" + exit 255 # unknown error + fi + + # then, we parallelize the transfer of database directories, use . so that pathconcatenation works + pushd "$WSREP_SST_OPT_DATA" >/dev/null + + count=1 + [ "$OS" == "Linux" ] && count=$(grep -c processor /proc/cpuinfo) + [ "$OS" == "Darwin" -o "$OS" == "FreeBSD" ] && count=$(sysctl -n hw.ncpu) + + find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -I{} -0 -P $count \ + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --recursive --delete --quiet \ + $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \ + rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$? + + popd >/dev/null + + if [ $RC -ne 0 ]; then + wsrep_log_error "find/rsync returned code $RC:" + exit 255 # unknown error + fi + + else # BYPASS + wsrep_log_info "Bypassing state dump." + STATE="$WSREP_SST_OPT_GTID" + fi + + echo "continue" # now server can resume updating data + + echo "$STATE" > "$MAGIC_FILE" + rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR + + echo "done $STATE" + +elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ] +then + wsrep_check_programs lsof + + touch $SST_PROGRESS_FILE + MYSQLD_PID=$WSREP_SST_OPT_PARENT + + MODULE="rsync_sst" + + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" + + if check_pid $RSYNC_PID + then + wsrep_log_error "rsync daemon already running." + exit 114 # EALREADY + fi + rm -rf "$RSYNC_PID" + + ADDR=$WSREP_SST_OPT_ADDR + RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') + if [ -z "$RSYNC_PORT" ] + then + RSYNC_PORT=4444 + ADDR="$(echo $ADDR | awk -F ':' '{ print $1 }'):$RSYNC_PORT" + fi + + trap "exit 32" HUP PIPE + trap "exit 3" INT TERM ABRT + trap cleanup_joiner EXIT + + RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +cat << EOF > "$RSYNC_CONF" +pid file = $RSYNC_PID +use chroot = no +read only = no +timeout = 300 +[$MODULE] + path = $WSREP_SST_OPT_DATA +[$MODULE-log_dir] + path = $WSREP_LOG_DIR +EOF + +# rm -rf "$DATA"/ib_logfile* # we don't want old logs around + + # listen at all interfaces (for firewalled setups) + rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" & + RSYNC_REAL_PID=$! + + until check_pid_and_port $RSYNC_PID $RSYNC_REAL_PID $RSYNC_PORT + do + sleep 0.2 + done + + echo "ready $ADDR/$MODULE" + + # wait for SST to complete by monitoring magic file + while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \ + ps -p $MYSQLD_PID >/dev/null + do + sleep 1 + done + + if ! ps -p $MYSQLD_PID >/dev/null + then + wsrep_log_error \ + "Parent mysqld process (PID:$MYSQLD_PID) terminated unexpectedly." + exit 32 + fi + + if ! [ -z $WSREP_SST_OPT_BINLOG ] + then + + pushd $BINLOG_DIRNAME &> /dev/null + if [ -f $BINLOG_TAR_FILE ] + then + # Clean up old binlog files first + rm -f ${BINLOG_FILENAME}.* + wsrep_log_info "Extracting binlog files:" + tar -xvf $BINLOG_TAR_FILE >&2 + for ii in $(ls -1 ${BINLOG_FILENAME}.*) + do + echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index + done + fi + popd &> /dev/null + fi + if [ -r "$MAGIC_FILE" ] + then + cat "$MAGIC_FILE" # output UUID:seqno + else + # this message should cause joiner to abort + echo "rsync process ended without creating '$MAGIC_FILE'" + fi + wsrep_cleanup_progress_file +# cleanup_joiner +else + wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" + exit 22 # EINVAL +fi + +rm -f $BINLOG_TAR_FILE || : + +exit 0 diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh new file mode 100644 index 00000000000..86bf557662d --- /dev/null +++ b/scripts/wsrep_sst_rsync.sh @@ -0,0 +1,335 @@ +#!/bin/bash -ue + +# Copyright (C) 2010-2014 Codership Oy +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# This is a reference script for rsync-based state snapshot tansfer + +RSYNC_PID= +RSYNC_CONF= +OS=$(uname) +[ "$OS" == "Darwin" ] && export -n LD_LIBRARY_PATH + +# Setting the path for lsof on CentOS +export PATH="/usr/sbin:/sbin:$PATH" + +. $(dirname $0)/wsrep_sst_common + +wsrep_check_programs rsync + +cleanup_joiner() +{ + wsrep_log_info "Joiner cleanup." + local PID=$(cat "$RSYNC_PID" 2>/dev/null || echo 0) + [ "0" != "$PID" ] && kill $PID && sleep 0.5 && kill -9 $PID >/dev/null 2>&1 \ + || : + rm -rf "$RSYNC_CONF" + rm -rf "$MAGIC_FILE" + rm -rf "$RSYNC_PID" + wsrep_log_info "Joiner cleanup done." + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_cleanup_progress_file + fi +} + +check_pid() +{ + local pid_file=$1 + [ -r "$pid_file" ] && ps -p $(cat $pid_file) >/dev/null 2>&1 +} + +check_pid_and_port() +{ + local pid_file=$1 + local rsync_pid=$2 + local rsync_port=$3 + + if ! which lsof > /dev/null; then + wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed." + exit 2 # ENOENT + fi + + local port_info=$(lsof -i :$rsync_port -Pn 2>/dev/null | \ + grep "(LISTEN)") + local is_rsync=$(echo $port_info | \ + grep -w '^rsync[[:space:]]\+'"$rsync_pid" 2>/dev/null) + + if [ -n "$port_info" -a -z "$is_rsync" ]; then + wsrep_log_error "rsync daemon port '$rsync_port' has been taken" + exit 16 # EBUSY + fi + check_pid $pid_file && \ + [ -n "$port_info" ] && [ -n "$is_rsync" ] && \ + [ $(cat $pid_file) -eq $rsync_pid ] +} + +MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" +rm -rf "$MAGIC_FILE" + +BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" +BINLOG_N_FILES=1 +rm -f "$BINLOG_TAR_FILE" || : + +if ! [ -z $WSREP_SST_OPT_BINLOG ] +then + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) +fi + +WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} +# if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf +if [ -z "$WSREP_LOG_DIR" ]; then + SCRIPT_DIR="$(cd $(dirname "$0"); pwd -P)" + WSREP_LOG_DIR=$($SCRIPT_DIR/my_print_defaults --defaults-file \ + "$WSREP_SST_OPT_CONF" mysqld server mysqld-10.0 mariadb mariadb-10.0 \ + | grep -- '--innodb[-_]log[-_]group[-_]home[-_]dir=' \ + | cut -b 29- ) +fi + +if [ -n "$WSREP_LOG_DIR" ]; then + # handle both relative and absolute paths + WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$WSREP_LOG_DIR"; cd $WSREP_LOG_DIR; pwd -P) +else + # default to datadir + WSREP_LOG_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P) +fi + +# Old filter - include everything except selected +# FILTER=(--exclude '*.err' --exclude '*.pid' --exclude '*.sock' \ +# --exclude '*.conf' --exclude core --exclude 'galera.*' \ +# --exclude grastate.txt --exclude '*.pem' \ +# --exclude '*.[0-9][0-9][0-9][0-9][0-9][0-9]' --exclude '*.index') + +# New filter - exclude everything except dirs (schemas) and innodb files +FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes' + -f '+ /wsrep_sst_binlog.tar' -f '+ /ib_lru_dump' -f '+ /ibdata*' -f '+ /*/' -f '- /*') + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed" + rm -rf "$FLUSHED" + + # Use deltaxfer only for WAN + inv=$(basename $0) + [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \ + || WHOLE_FILE_OPT="--whole-file" + + echo "flush tables" + + # wait for tables flushed and state ID written to the file + while [ ! -r "$FLUSHED" ] && ! grep -q ':' "$FLUSHED" >/dev/null 2>&1 + do + sleep 0.2 + done + + STATE="$(cat $FLUSHED)" + rm -rf "$FLUSHED" + + sync + + if ! [ -z $WSREP_SST_OPT_BINLOG ] + then + # Prepare binlog files + pushd $BINLOG_DIRNAME &> /dev/null + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + binlog_files="" + for ii in $binlog_files_full + do + binlog_files="$binlog_files $(basename $ii)" + done + if ! [ -z "$binlog_files" ] + then + wsrep_log_info "Preparing binlog files for transfer:" + tar -cvf $BINLOG_TAR_FILE $binlog_files >&2 + fi + popd &> /dev/null + fi + + # first, the normal directories, so that we can detect incompatible protocol + RC=0 + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT "${FILTER[@]}" "$WSREP_SST_OPT_DATA/" \ + rsync://$WSREP_SST_OPT_ADDR >&2 || RC=$? + + if [ "$RC" -ne 0 ]; then + wsrep_log_error "rsync returned code $RC:" + + case $RC in + 12) RC=71 # EPROTO + wsrep_log_error \ + "rsync server on the other end has incompatible protocol. " \ + "Make sure you have the same version of rsync on all nodes." + ;; + 22) RC=12 # ENOMEM + ;; + *) RC=255 # unknown error + ;; + esac + exit $RC + fi + + # second, we transfer InnoDB log files + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --dirs --delete --quiet \ + $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '- **' "$WSREP_LOG_DIR/" \ + rsync://$WSREP_SST_OPT_ADDR-log_dir >&2 || RC=$? + + if [ $RC -ne 0 ]; then + wsrep_log_error "rsync innodb_log_group_home_dir returned code $RC:" + exit 255 # unknown error + fi + + # then, we parallelize the transfer of database directories, use . so that pathconcatenation works + pushd "$WSREP_SST_OPT_DATA" >/dev/null + + count=1 + [ "$OS" == "Linux" ] && count=$(grep -c processor /proc/cpuinfo) + [ "$OS" == "Darwin" -o "$OS" == "FreeBSD" ] && count=$(sysctl -n hw.ncpu) + + find . -maxdepth 1 -mindepth 1 -type d -print0 | xargs -I{} -0 -P $count \ + rsync --owner --group --perms --links --specials \ + --ignore-times --inplace --recursive --delete --quiet \ + $WHOLE_FILE_OPT --exclude '*/ib_logfile*' "$WSREP_SST_OPT_DATA"/{}/ \ + rsync://$WSREP_SST_OPT_ADDR/{} >&2 || RC=$? + + popd >/dev/null + + if [ $RC -ne 0 ]; then + wsrep_log_error "find/rsync returned code $RC:" + exit 255 # unknown error + fi + + else # BYPASS + wsrep_log_info "Bypassing state dump." + STATE="$WSREP_SST_OPT_GTID" + fi + + echo "continue" # now server can resume updating data + + echo "$STATE" > "$MAGIC_FILE" + rsync --archive --quiet --checksum "$MAGIC_FILE" rsync://$WSREP_SST_OPT_ADDR + + echo "done $STATE" + +elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ] +then + wsrep_check_programs lsof + + touch $SST_PROGRESS_FILE + MYSQLD_PID=$WSREP_SST_OPT_PARENT + + MODULE="rsync_sst" + + RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" + + if check_pid $RSYNC_PID + then + wsrep_log_error "rsync daemon already running." + exit 114 # EALREADY + fi + rm -rf "$RSYNC_PID" + + ADDR=$WSREP_SST_OPT_ADDR + RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') + if [ -z "$RSYNC_PORT" ] + then + RSYNC_PORT=4444 + ADDR="$(echo $ADDR | awk -F ':' '{ print $1 }'):$RSYNC_PORT" + fi + + trap "exit 32" HUP PIPE + trap "exit 3" INT TERM ABRT + trap cleanup_joiner EXIT + + RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +cat << EOF > "$RSYNC_CONF" +pid file = $RSYNC_PID +use chroot = no +read only = no +timeout = 300 +[$MODULE] + path = $WSREP_SST_OPT_DATA +[$MODULE-log_dir] + path = $WSREP_LOG_DIR +EOF + +# rm -rf "$DATA"/ib_logfile* # we don't want old logs around + + # listen at all interfaces (for firewalled setups) + rsync --daemon --no-detach --port $RSYNC_PORT --config "$RSYNC_CONF" & + RSYNC_REAL_PID=$! + + until check_pid_and_port $RSYNC_PID $RSYNC_REAL_PID $RSYNC_PORT + do + sleep 0.2 + done + + echo "ready $ADDR/$MODULE" + + # wait for SST to complete by monitoring magic file + while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \ + ps -p $MYSQLD_PID >/dev/null + do + sleep 1 + done + + if ! ps -p $MYSQLD_PID >/dev/null + then + wsrep_log_error \ + "Parent mysqld process (PID:$MYSQLD_PID) terminated unexpectedly." + exit 32 + fi + + if ! [ -z $WSREP_SST_OPT_BINLOG ] + then + + pushd $BINLOG_DIRNAME &> /dev/null + if [ -f $BINLOG_TAR_FILE ] + then + # Clean up old binlog files first + rm -f ${BINLOG_FILENAME}.* + wsrep_log_info "Extracting binlog files:" + tar -xvf $BINLOG_TAR_FILE >&2 + for ii in $(ls -1 ${BINLOG_FILENAME}.*) + do + echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index + done + fi + popd &> /dev/null + fi + if [ -r "$MAGIC_FILE" ] + then + cat "$MAGIC_FILE" # output UUID:seqno + else + # this message should cause joiner to abort + echo "rsync process ended without creating '$MAGIC_FILE'" + fi + wsrep_cleanup_progress_file +# cleanup_joiner +else + wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'" + exit 22 # EINVAL +fi + +rm -f $BINLOG_TAR_FILE || : + +exit 0 diff --git a/scripts/wsrep_sst_xtrabackup b/scripts/wsrep_sst_xtrabackup new file mode 100755 index 00000000000..6b33eabee23 --- /dev/null +++ b/scripts/wsrep_sst_xtrabackup @@ -0,0 +1,715 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + + + + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +tcert="" +tpem="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +incremental=0 +ecmd="" +rlimit="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +uextra=0 + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=innobackupex +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + if [[ $encrypt -eq 2 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=0 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="nc -dl ${TSST_PORT}" + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 ]] && ! socat -V | grep -q OPENSSL;then + wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" + encrypt=0 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" + fi + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "tar") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + incremental=$(parse_cnf sst incremental 0) + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use_extra 0) +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="xbstream -x" + else + strmcmd="xbstream -c \${INFO_FILE} \${IST_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - --recursive-unlink -h" + else + strmcmd="tar cf - \${INFO_FILE} \${IST_FILE}" + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n $XTRABACKUP_PID ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + rm -f $XTRABACKUP_PID + fi + rm -f ${DATA}/${IST_FILE} + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + rm -f "$XTRABACKUP_PID" +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + if [[ $incremental -eq 1 ]];then + echo "ready ${ADDR}/${MODULE}/$lsn" + else + echo "ready ${ADDR}/${MODULE}" + fi +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +if [[ ! -x `which innobackupex` ]];then + wsrep_log_error "innobackupex not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports +get_stream +get_transfer + +INNOEXTRA="" +INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" +INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + TMPDIR="${TMPDIR:-/tmp}" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOEXTRA+=" --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOEXTRA+=" --password=${AUTH[1]}" + elif [ "${AUTH[0]}" != "(null)" ]; then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + if [[ -n $lsn ]];then + INNOEXTRA+=" --incremental --incremental-lsn=$lsn " + fi + + check_extra + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT}" + + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + set +e + timeit "Donor-Transfer" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in ${TMPDIR} + XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + STATE="${WSREP_SST_OPT_GTID}" + echo "continue" # now server can resume updating data + echo "${STATE}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + pushd ${DATA} 1>/dev/null + set +e + if [[ $encrypt -eq 1 ]];then + tcmd=" $ecmd | $tcmd" + fi + timeit "Donor-IST-Unencrypted-transfer" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while streaming data to joiner node: " \ + "exit codes: ${RC[@]}" + exit 1 + fi + done + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + touch $SST_PROGRESS_FILE + + if [[ ! -e ${DATA}/ibdata1 ]];then + incremental=0 + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Incremental SST enabled" + #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF --basedir=/pxc --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1) + lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') + wsrep_log_info "Recovered LSN: $lsn" + fi + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + if [[ $incremental -eq 1 ]];then + BDATA=$DATA + DATA=$(mktemp -d) + MAGIC_FILE="${DATA}/${INFO_FILE}" + fi + + get_keys + set +e + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + strmcmd=" $ecmd | $strmcmd" + fi + + pushd ${DATA} 1>/dev/null + timeit "Joiner-Recv-Unencrypted" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + popd 1>/dev/null + + set -e + + if [[ $sfmt == 'xbstream' ]];then + # Special handling till lp:1193240 is fixed" + if [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "Xbstream failed" + wsrep_log_error "Data directory ${DATA} may not be empty: lp:1193240" \ + "Manual intervention required in that case" + exit 32 + fi + fi + + wait %% # join for wait_for_listen thread + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [ ! -r "${MAGIC_FILE}" ] + then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${DATA}/**/*)" + exit 32 + fi + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${DATA}/${IST_FILE}" ] + then + wsrep_log_info "Proceeding with SST" + wsrep_log_info "Removing existing ib_logfile files" + if [[ $incremental -ne 1 ]];then + rm -f ${DATA}/ib_logfile* + else + rm -f ${BDATA}/ib_logfile* + fi + + get_proc + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + wsrep_log_info "Removing existing ibdata1 file" + rm -f ${DATA}/ibdata1 + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + if [[ $incremental -eq 1 ]];then + # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. + INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \ + --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Cleaning up ${DATA} after incremental SST" + [[ -d ${DATA} ]] && rm -rf ${DATA} + DATA=$BDATA + fi + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + + cat "${MAGIC_FILE}" # output UUID:seqno + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 diff --git a/scripts/wsrep_sst_xtrabackup-v2 b/scripts/wsrep_sst_xtrabackup-v2 new file mode 100755 index 00000000000..f4d133d4f8e --- /dev/null +++ b/scripts/wsrep_sst_xtrabackup-v2 @@ -0,0 +1,930 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + + + + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +tcert="" +tpem="" +tkey="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +incremental=0 +ecmd="" +rlimit="" +# Initially +stagemsg="${WSREP_SST_OPT_ROLE}" +cpat="" +speciald=0 +ib_home_dir="" +ib_log_dir="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +STATDIR="" +uextra=0 +disver="" + +tmpopts="" +itmpdir="" +xtmpdir="" + +scomp="" +sdecomp="" + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=innobackupex +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + # $encrypt -eq 1 is for internal purposes only + if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=-1 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi + + stagemsg+="-XB-Encrypted" +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="nc -dl ${TSST_PORT}" + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q WITH_OPENSSL;then + wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" + encrypt=-1 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-2" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" + fi + elif [[ $encrypt -eq 3 ]];then + wsrep_log_info "Using openssl based encryption with socat: with key and crt" + if [[ -z $tpem || -z $tkey ]];then + wsrep_log_error "Both certificate and key files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-3" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with certificate $tpem, key $tkey" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,key=${tkey},verify=0${sockopt} stdio" + else + wsrep_log_info "Encrypting with certificate $tpem, key $tkey" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,key=${tkey},verify=0${sockopt}" + fi + + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "xbstream") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + incremental=$(parse_cnf sst incremental 0) + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + scomp=$(parse_cnf sst compressor "") + sdecomp=$(parse_cnf sst decompressor "") + + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use-extra 0) + speciald=$(parse_cnf sst sst-special-dirs 1) + iopts=$(parse_cnf sst inno-backup-opts "") + iapts=$(parse_cnf sst inno-apply-opts "") + impts=$(parse_cnf sst inno-move-opts "") + stimeout=$(parse_cnf sst sst-initial-timeout 100) +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="xbstream -x" + else + strmcmd="xbstream -c \${INFO_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - " + else + strmcmd="tar cf - \${INFO_FILE} " + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi + if [[ -n ${STATDIR:-} ]];then + [[ -d $STATDIR ]] && rm -rf $STATDIR + fi +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n ${XTRABACKUP_PID:-} ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + fi + rm -f ${DATA}/${IST_FILE} || true + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm -f $progress || true + fi + + wsrep_log_info "Cleaning up temporary directories" + + if [[ -n $xtmpdir ]];then + [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true + fi + + if [[ -n $itmpdir ]];then + [[ -d $itmpdir ]] && rm -rf $itmpdir || true + fi +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" + rm -f "$XTRABACKUP_PID" || true +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + if [[ $incremental -eq 1 ]];then + echo "ready ${ADDR}/${MODULE}/$lsn" + else + echo "ready ${ADDR}/${MODULE}" + fi +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +recv_joiner() +{ + local dir=$1 + local msg=$2 + local tmt=$3 + local ltcmd + + pushd ${dir} 1>/dev/null + set +e + + if [[ $tmt -gt 0 && -x `which timeout` ]];then + if timeout --help | grep -q -- '-k';then + ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" + else + ltcmd="timeout $tmt $tcmd" + fi + timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + else + timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + fi + + set -e + popd 1>/dev/null + + if [[ ${RC[0]} -eq 124 ]];then + wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" + exit 32 + fi + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [ ! -r "${MAGIC_FILE}" ];then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${dir}/*)" + exit 32 + fi +} + + +send_donor() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + +} + +if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then + wsrep_log_error "innobackupex not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports +get_stream +get_transfer + +if ${INNOBACKUPEX_BIN} /tmp --help | grep -q -- '--version-check'; then + disver="--no-version-check" +fi + + +INNOEXTRA="" +INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" +INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" +INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then + xtmpdir=$(mktemp -d) + tmpopts=" --tmpdir=$xtmpdir " + wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" + fi + + itmpdir=$(mktemp -d) + wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOEXTRA+=" --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOEXTRA+=" --password=${AUTH[1]}" + elif [ "${AUTH[0]}" != "(null)" ]; then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + if [[ -n $lsn ]];then + INNOEXTRA+=" --incremental --incremental-lsn=$lsn " + fi + + check_extra + + wsrep_log_info "Streaming GTID file before SST" + + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + + ttcmd="$tcmd" + + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + + + send_donor $DATA "${stagemsg}-gtid" + + tcmd="$ttcmd" + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + wsrep_log_info "Sleeping before data transfer for SST" + sleep 10 + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + + set +e + timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in $xtmpdir + XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + echo "continue" # now server can resume updating data + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + strmcmd+=" \${IST_FILE}" + + send_donor $DATA "${stagemsg}-IST" + + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE + + if [[ $speciald -eq 1 ]];then + ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") + ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") + if [[ -z $ib_home_dir && -z $ib_log_dir ]];then + speciald=0 + fi + fi + + stagemsg="Joiner-Recv" + + if [[ ! -e ${DATA}/ibdata1 ]];then + incremental=0 + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Incremental SST enabled: NOT SUPPORTED yet" + lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') + wsrep_log_info "Recovered LSN: $lsn" + fi + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + rm -f "${DATA}/${IST_FILE}" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + if [[ $incremental -eq 1 ]];then + BDATA=$DATA + DATA=$(mktemp -d) + MAGIC_FILE="${DATA}/${INFO_FILE}" + fi + + get_keys + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + if [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $ecmd | $strmcmd" + else + strmcmd=" $ecmd | $strmcmd" + fi + elif [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $strmcmd" + fi + + STATDIR=$(mktemp -d) + MAGIC_FILE="${STATDIR}/${INFO_FILE}" + recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${STATDIR}/${IST_FILE}" ] + then + wsrep_log_info "Proceeding with SST" + + if [[ $speciald -eq 1 && -d ${DATA}/.sst ]];then + wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous SST" + fi + + if [[ $incremental -ne 1 ]];then + if [[ $speciald -eq 1 ]];then + wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + find $ib_home_dir $ib_log_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + else + wsrep_log_info "Cleaning the existing datadir" + find $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + fi + tempdir=$(parse_cnf mysqld log-bin "") + if [[ -n ${tempdir:-} ]];then + binlog_dir=$(dirname $tempdir) + binlog_file=$(basename $tempdir) + if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + pattern="$binlog_dir/$binlog_file\.[0-9]+$" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ + rm $binlog_dir/*.index || true + fi + fi + + else + wsrep_log_info "Removing existing ib_logfile files" + rm -f ${BDATA}/ib_logfile* + fi + + + if [[ $speciald -eq 1 ]];then + mkdir -p ${DATA}/.sst + TDATA=${DATA} + DATA="${DATA}/.sst" + fi + + + MAGIC_FILE="${DATA}/${INFO_FILE}" + recv_joiner $DATA "${stagemsg}-SST" 0 + + get_proc + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + + if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) + + # To avoid comparing data directory and BINLOG_DIRNAME + mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true + + pushd $BINLOG_DIRNAME &>/dev/null + for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do + echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index + done + popd &> /dev/null + + fi + + if [[ $incremental -eq 1 ]];then + # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. + INNOAPPLY="${INNOBACKUPEX_BIN} $disver --defaults-file=${WSREP_SST_OPT_CONF} \ + --ibbackup=xtrabackup_56 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + + if [[ $speciald -eq 1 ]];then + MAGIC_FILE="${TDATA}/${INFO_FILE}" + set +e + rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log + set -e + wsrep_log_info "Moving the backup to ${TDATA}" + timeit "Xtrabackup move stage" "$INNOMOVE" + if [[ $? -eq 0 ]];then + wsrep_log_info "Move successful, removing ${DATA}" + rm -rf $DATA + DATA=${TDATA} + else + wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" + wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + fi + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Cleaning up ${DATA} after incremental SST" + [[ -d ${DATA} ]] && rm -rf ${DATA} + DATA=$BDATA + fi + + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + cat "${MAGIC_FILE}" # output UUID:seqno + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh new file mode 100644 index 00000000000..f4d133d4f8e --- /dev/null +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -0,0 +1,930 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + + + + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +tcert="" +tpem="" +tkey="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +incremental=0 +ecmd="" +rlimit="" +# Initially +stagemsg="${WSREP_SST_OPT_ROLE}" +cpat="" +speciald=0 +ib_home_dir="" +ib_log_dir="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +STATDIR="" +uextra=0 +disver="" + +tmpopts="" +itmpdir="" +xtmpdir="" + +scomp="" +sdecomp="" + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=innobackupex +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + # $encrypt -eq 1 is for internal purposes only + if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=-1 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi + + stagemsg+="-XB-Encrypted" +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="nc -dl ${TSST_PORT}" + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 || $encrypt -eq 3 ]] && ! socat -V | grep -q WITH_OPENSSL;then + wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" + encrypt=-1 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat: with crt and pem" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-2" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" + fi + elif [[ $encrypt -eq 3 ]];then + wsrep_log_info "Using openssl based encryption with socat: with key and crt" + if [[ -z $tpem || -z $tkey ]];then + wsrep_log_error "Both certificate and key files required" + exit 22 + fi + stagemsg+="-OpenSSL-Encrypted-3" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with certificate $tpem, key $tkey" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,key=${tkey},verify=0${sockopt} stdio" + else + wsrep_log_info "Encrypting with certificate $tpem, key $tkey" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,key=${tkey},verify=0${sockopt}" + fi + + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "xbstream") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') + incremental=$(parse_cnf sst incremental 0) + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + scomp=$(parse_cnf sst compressor "") + sdecomp=$(parse_cnf sst decompressor "") + + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use-extra 0) + speciald=$(parse_cnf sst sst-special-dirs 1) + iopts=$(parse_cnf sst inno-backup-opts "") + iapts=$(parse_cnf sst inno-apply-opts "") + impts=$(parse_cnf sst inno-move-opts "") + stimeout=$(parse_cnf sst sst-initial-timeout 100) +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="xbstream -x" + else + strmcmd="xbstream -c \${INFO_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - " + else + strmcmd="tar cf - \${INFO_FILE} " + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi + if [[ -n ${STATDIR:-} ]];then + [[ -d $STATDIR ]] && rm -rf $STATDIR + fi +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n ${XTRABACKUP_PID:-} ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + fi + rm -f ${DATA}/${IST_FILE} || true + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm -f $progress || true + fi + + wsrep_log_info "Cleaning up temporary directories" + + if [[ -n $xtmpdir ]];then + [[ -d $xtmpdir ]] && rm -rf $xtmpdir || true + fi + + if [[ -n $itmpdir ]];then + [[ -d $itmpdir ]] && rm -rf $itmpdir || true + fi +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID" + rm -f "$XTRABACKUP_PID" || true +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + if [[ $incremental -eq 1 ]];then + echo "ready ${ADDR}/${MODULE}/$lsn" + else + echo "ready ${ADDR}/${MODULE}" + fi +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +recv_joiner() +{ + local dir=$1 + local msg=$2 + local tmt=$3 + local ltcmd + + pushd ${dir} 1>/dev/null + set +e + + if [[ $tmt -gt 0 && -x `which timeout` ]];then + if timeout --help | grep -q -- '-k';then + ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" + else + ltcmd="timeout $tmt $tcmd" + fi + timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + else + timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + fi + + set -e + popd 1>/dev/null + + if [[ ${RC[0]} -eq 124 ]];then + wsrep_log_error "Possible timeout in receving first data from donor in gtid stage" + exit 32 + fi + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [ ! -r "${MAGIC_FILE}" ];then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${dir}/*)" + exit 32 + fi +} + + +send_donor() +{ + local dir=$1 + local msg=$2 + + pushd ${dir} 1>/dev/null + set +e + timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + +} + +if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then + wsrep_log_error "innobackupex not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports +get_stream +get_transfer + +if ${INNOBACKUPEX_BIN} /tmp --help | grep -q -- '--version-check'; then + disver="--no-version-check" +fi + + +INNOEXTRA="" +INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" +INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log" +INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + + if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then + xtmpdir=$(mktemp -d) + tmpopts=" --tmpdir=$xtmpdir " + wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory" + fi + + itmpdir=$(mktemp -d) + wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOEXTRA+=" --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOEXTRA+=" --password=${AUTH[1]}" + elif [ "${AUTH[0]}" != "(null)" ]; then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + if [[ -n $lsn ]];then + INNOEXTRA+=" --incremental --incremental-lsn=$lsn " + fi + + check_extra + + wsrep_log_info "Streaming GTID file before SST" + + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + + ttcmd="$tcmd" + + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + + + send_donor $DATA "${stagemsg}-gtid" + + tcmd="$ttcmd" + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + wsrep_log_info "Sleeping before data transfer for SST" + sleep 10 + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + + set +e + timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in $xtmpdir + XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + echo "continue" # now server can resume updating data + echo "${WSREP_SST_OPT_GTID}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $scomp ]];then + tcmd=" $ecmd | $scomp | $tcmd " + else + tcmd=" $ecmd | $tcmd " + fi + elif [[ -n $scomp ]];then + tcmd=" $scomp | $tcmd " + fi + strmcmd+=" \${IST_FILE}" + + send_donor $DATA "${stagemsg}-IST" + + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + [[ -n $SST_PROGRESS_FILE ]] && touch $SST_PROGRESS_FILE + + if [[ $speciald -eq 1 ]];then + ib_home_dir=$(parse_cnf mysqld innodb-data-home-dir "") + ib_log_dir=$(parse_cnf mysqld innodb-log-group-home-dir "") + if [[ -z $ib_home_dir && -z $ib_log_dir ]];then + speciald=0 + fi + fi + + stagemsg="Joiner-Recv" + + if [[ ! -e ${DATA}/ibdata1 ]];then + incremental=0 + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Incremental SST enabled: NOT SUPPORTED yet" + lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') + wsrep_log_info "Recovered LSN: $lsn" + fi + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + rm -f "${DATA}/${IST_FILE}" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + if [[ $incremental -eq 1 ]];then + BDATA=$DATA + DATA=$(mktemp -d) + MAGIC_FILE="${DATA}/${INFO_FILE}" + fi + + get_keys + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + if [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $ecmd | $strmcmd" + else + strmcmd=" $ecmd | $strmcmd" + fi + elif [[ -n $sdecomp ]];then + strmcmd=" $sdecomp | $strmcmd" + fi + + STATDIR=$(mktemp -d) + MAGIC_FILE="${STATDIR}/${INFO_FILE}" + recv_joiner $STATDIR "${stagemsg}-gtid" $stimeout + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${STATDIR}/${IST_FILE}" ] + then + wsrep_log_info "Proceeding with SST" + + if [[ $speciald -eq 1 && -d ${DATA}/.sst ]];then + wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous SST" + fi + + if [[ $incremental -ne 1 ]];then + if [[ $speciald -eq 1 ]];then + wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories" + find $ib_home_dir $ib_log_dir $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + else + wsrep_log_info "Cleaning the existing datadir" + find $DATA -mindepth 1 -regex $cpat -prune -o -exec rm -rfv {} 1>&2 \+ + fi + tempdir=$(parse_cnf mysqld log-bin "") + if [[ -n ${tempdir:-} ]];then + binlog_dir=$(dirname $tempdir) + binlog_file=$(basename $tempdir) + if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + pattern="$binlog_dir/$binlog_file\.[0-9]+$" + wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" + find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ + rm $binlog_dir/*.index || true + fi + fi + + else + wsrep_log_info "Removing existing ib_logfile files" + rm -f ${BDATA}/ib_logfile* + fi + + + if [[ $speciald -eq 1 ]];then + mkdir -p ${DATA}/.sst + TDATA=${DATA} + DATA="${DATA}/.sst" + fi + + + MAGIC_FILE="${DATA}/${INFO_FILE}" + recv_joiner $DATA "${stagemsg}-SST" 0 + + get_proc + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + + if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + + BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) + BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) + + # To avoid comparing data directory and BINLOG_DIRNAME + mv $DATA/${BINLOG_FILENAME}.* $BINLOG_DIRNAME/ 2>/dev/null || true + + pushd $BINLOG_DIRNAME &>/dev/null + for bfiles in $(ls -1 ${BINLOG_FILENAME}.*);do + echo ${BINLOG_DIRNAME}/${bfiles} >> ${BINLOG_FILENAME}.index + done + popd &> /dev/null + + fi + + if [[ $incremental -eq 1 ]];then + # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. + INNOAPPLY="${INNOBACKUPEX_BIN} $disver --defaults-file=${WSREP_SST_OPT_CONF} \ + --ibbackup=xtrabackup_56 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + + if [[ $speciald -eq 1 ]];then + MAGIC_FILE="${TDATA}/${INFO_FILE}" + set +e + rm $TDATA/innobackup.prepare.log $TDATA/innobackup.move.log + set -e + wsrep_log_info "Moving the backup to ${TDATA}" + timeit "Xtrabackup move stage" "$INNOMOVE" + if [[ $? -eq 0 ]];then + wsrep_log_info "Move successful, removing ${DATA}" + rm -rf $DATA + DATA=${TDATA} + else + wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis" + wsrep_log_error "Check ${DATA}/innobackup.move.log for details" + fi + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Cleaning up ${DATA} after incremental SST" + [[ -d ${DATA} ]] && rm -rf ${DATA} + DATA=$BDATA + fi + + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + cat "${MAGIC_FILE}" # output UUID:seqno + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh new file mode 100644 index 00000000000..6b33eabee23 --- /dev/null +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -0,0 +1,715 @@ +#!/bin/bash -ue +# Copyright (C) 2013 Percona Inc +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; see the file COPYING. If not, write to the +# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston +# MA 02110-1301 USA. + +# Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html +# Make sure to read that before proceeding! + + + + +. $(dirname $0)/wsrep_sst_common + +ealgo="" +ekey="" +ekeyfile="" +encrypt=0 +nproc=1 +ecode=0 +XTRABACKUP_PID="" +SST_PORT="" +REMOTEIP="" +tcert="" +tpem="" +sockopt="" +progress="" +ttime=0 +totime=0 +lsn="" +incremental=0 +ecmd="" +rlimit="" + +sfmt="tar" +strmcmd="" +tfmt="" +tcmd="" +rebuild=0 +rebuildcmd="" +payload=0 +pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " +pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " +uextra=0 + +if which pv &>/dev/null && pv --help | grep -q FORMAT;then + pvopts+=$pvformat +fi +pcmd="pv $pvopts" +declare -a RC + +INNOBACKUPEX_BIN=innobackupex +readonly AUTH=(${WSREP_SST_OPT_AUTH//:/ }) +DATA="${WSREP_SST_OPT_DATA}" +INFO_FILE="xtrabackup_galera_info" +IST_FILE="xtrabackup_ist" +MAGIC_FILE="${DATA}/${INFO_FILE}" + +# Setting the path for ss and ip +export PATH="/usr/sbin:/sbin:$PATH" + +timeit(){ + local stage=$1 + shift + local cmd="$@" + local x1 x2 took extcode + + if [[ $ttime -eq 1 ]];then + x1=$(date +%s) + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + x2=$(date +%s) + took=$(( x2-x1 )) + wsrep_log_info "NOTE: $stage took $took seconds" + totime=$(( totime+took )) + else + wsrep_log_info "Evaluating $cmd" + eval "$cmd" + extcode=$? + fi + return $extcode +} + +get_keys() +{ + if [[ $encrypt -eq 2 ]];then + return + fi + + if [[ $encrypt -eq 0 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then + wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html " + fi + return + fi + + if [[ $sfmt == 'tar' ]];then + wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format" + encrypt=0 + return + fi + + wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4" + + if [[ -z $ealgo ]];then + wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out" + exit 3 + fi + + if [[ -z $ekey && ! -r $ekeyfile ]];then + wsrep_log_error "FATAL: Either key or keyfile must be readable" + exit 3 + fi + + if [[ -z $ekey ]];then + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile" + else + ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey" + fi + + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + ecmd+=" -d" + fi +} + +get_transfer() +{ + if [[ -z $SST_PORT ]];then + TSST_PORT=4444 + else + TSST_PORT=$SST_PORT + fi + + if [[ $tfmt == 'nc' ]];then + if [[ ! -x `which nc` ]];then + wsrep_log_error "nc(netcat) not found in path: $PATH" + exit 2 + fi + wsrep_log_info "Using netcat as streamer" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="nc -dl ${TSST_PORT}" + else + tcmd="nc ${REMOTEIP} ${TSST_PORT}" + fi + else + tfmt='socat' + wsrep_log_info "Using socat as streamer" + if [[ ! -x `which socat` ]];then + wsrep_log_error "socat not found in path: $PATH" + exit 2 + fi + + if [[ $encrypt -eq 2 ]] && ! socat -V | grep -q OPENSSL;then + wsrep_log_info "NOTE: socat is not openssl enabled, falling back to plain transfer" + encrypt=0 + fi + + if [[ $encrypt -eq 2 ]];then + wsrep_log_info "Using openssl based encryption with socat" + if [[ -z $tpem || -z $tcert ]];then + wsrep_log_error "Both PEM and CRT files required" + exit 22 + fi + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + wsrep_log_info "Decrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=$tpem,cafile=${tcert}${sockopt} stdio" + else + wsrep_log_info "Encrypting with PEM $tpem, CA: $tcert" + tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=$tpem,cafile=${tcert}${sockopt}" + fi + else + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" + else + tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" + fi + fi + fi + +} + +parse_cnf() +{ + local group=$1 + local var=$2 + reval=$(my_print_defaults -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-) + if [[ -z $reval ]];then + [[ -n $3 ]] && reval=$3 + fi + echo $reval +} + +get_footprint() +{ + pushd $WSREP_SST_OPT_DATA 1>/dev/null + payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }') + if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then + # QuickLZ has around 50% compression ratio + # When compression/compaction used, the progress is only an approximate. + payload=$(( payload*1/2 )) + fi + popd 1>/dev/null + pcmd+=" -s $payload" + adjust_progress +} + +adjust_progress() +{ + if [[ -n $progress && $progress != '1' ]];then + if [[ -e $progress ]];then + pcmd+=" 2>>$progress" + else + pcmd+=" 2>$progress" + fi + elif [[ -z $progress && -n $rlimit ]];then + # When rlimit is non-zero + pcmd="pv -q" + fi + + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + wsrep_log_info "Rate-limiting SST to $rlimit" + pcmd+=" -L \$rlimit" + fi +} + +read_cnf() +{ + sfmt=$(parse_cnf sst streamfmt "tar") + tfmt=$(parse_cnf sst transferfmt "socat") + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + encrypt=$(parse_cnf sst encrypt 0) + sockopt=$(parse_cnf sst sockopt "") + progress=$(parse_cnf sst progress "") + rebuild=$(parse_cnf sst rebuild 0) + ttime=$(parse_cnf sst time 0) + incremental=$(parse_cnf sst incremental 0) + ealgo=$(parse_cnf xtrabackup encrypt "") + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + if [[ -z $ealgo ]];then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + fi + rlimit=$(parse_cnf sst rlimit "") + uextra=$(parse_cnf sst use_extra 0) +} + +get_stream() +{ + if [[ $sfmt == 'xbstream' ]];then + wsrep_log_info "Streaming with xbstream" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="xbstream -x" + else + strmcmd="xbstream -c \${INFO_FILE} \${IST_FILE}" + fi + else + sfmt="tar" + wsrep_log_info "Streaming with tar" + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + strmcmd="tar xfi - --recursive-unlink -h" + else + strmcmd="tar cf - \${INFO_FILE} \${IST_FILE}" + fi + + fi +} + +get_proc() +{ + set +e + nproc=$(grep -c processor /proc/cpuinfo) + [[ -z $nproc || $nproc -eq 0 ]] && nproc=1 + set -e +} + +sig_joiner_cleanup() +{ + wsrep_log_error "Removing $MAGIC_FILE file due to signal" + rm -f "$MAGIC_FILE" +} + +cleanup_joiner() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + if [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then + wsrep_log_info "Removing the sst_in_progress file" + wsrep_cleanup_progress_file + fi + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi +} + +check_pid() +{ + local pid_file="$1" + [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1 +} + +cleanup_donor() +{ + # Since this is invoked just after exit NNN + local estatus=$? + if [[ $estatus -ne 0 ]];then + wsrep_log_error "Cleanup after exit with status:$estatus" + fi + + if [[ -n $XTRABACKUP_PID ]];then + if check_pid $XTRABACKUP_PID + then + wsrep_log_error "xtrabackup process is still running. Killing... " + kill_xtrabackup + fi + + rm -f $XTRABACKUP_PID + fi + rm -f ${DATA}/${IST_FILE} + + if [[ -n $progress && -p $progress ]];then + wsrep_log_info "Cleaning up fifo file $progress" + rm $progress + fi +} + +kill_xtrabackup() +{ + local PID=$(cat $XTRABACKUP_PID) + [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || : + rm -f "$XTRABACKUP_PID" +} + +setup_ports() +{ + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $2 }') + REMOTEIP=$(echo $WSREP_SST_OPT_ADDR | awk -F ':' '{ print $1 }') + lsn=$(echo $WSREP_SST_OPT_ADDR | awk -F '[:/]' '{ print $4 }') + else + SST_PORT=$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $2 }') + fi +} + +# waits ~10 seconds for nc to open the port and then reports ready +# (regardless of timeout) +wait_for_listen() +{ + local PORT=$1 + local ADDR=$2 + local MODULE=$3 + for i in {1..50} + do + ss -p state listening "( sport = :$PORT )" | grep -qE 'socat|nc' && break + sleep 0.2 + done + if [[ $incremental -eq 1 ]];then + echo "ready ${ADDR}/${MODULE}/$lsn" + else + echo "ready ${ADDR}/${MODULE}" + fi +} + +check_extra() +{ + local use_socket=1 + if [[ $uextra -eq 1 ]];then + if my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then + local eport=$(my_print_defaults -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + if [[ -n $eport ]];then + # Xtrabackup works only locally. + # Hence, setting host to 127.0.0.1 unconditionally. + wsrep_log_info "SST through extra_port $eport" + INNOEXTRA+=" --host=127.0.0.1 --port=$eport " + use_socket=0 + else + wsrep_log_error "Extra port $eport null, failing" + exit 1 + fi + else + wsrep_log_info "Thread pool not set, ignore the option use_extra" + fi + fi + if [[ $use_socket -eq 1 ]] && [[ -n "${WSREP_SST_OPT_SOCKET}" ]];then + INNOEXTRA+=" --socket=${WSREP_SST_OPT_SOCKET}" + fi +} + +if [[ ! -x `which innobackupex` ]];then + wsrep_log_error "innobackupex not in path: $PATH" + exit 2 +fi + +rm -f "${MAGIC_FILE}" + +if [[ ! ${WSREP_SST_OPT_ROLE} == 'joiner' && ! ${WSREP_SST_OPT_ROLE} == 'donor' ]];then + wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}" + exit 22 +fi + +read_cnf +setup_ports +get_stream +get_transfer + +INNOEXTRA="" +INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log" +INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log" + +if [ "$WSREP_SST_OPT_ROLE" = "donor" ] +then + trap cleanup_donor EXIT + + if [ $WSREP_SST_OPT_BYPASS -eq 0 ] + then + TMPDIR="${TMPDIR:-/tmp}" + + if [ "${AUTH[0]}" != "(null)" ]; then + INNOEXTRA+=" --user=${AUTH[0]}" + fi + + if [ ${#AUTH[*]} -eq 2 ]; then + INNOEXTRA+=" --password=${AUTH[1]}" + elif [ "${AUTH[0]}" != "(null)" ]; then + # Empty password, used for testing, debugging etc. + INNOEXTRA+=" --password=" + fi + + get_keys + if [[ $encrypt -eq 1 ]];then + if [[ -n $ekey ]];then + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey " + else + INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile " + fi + fi + + if [[ -n $lsn ]];then + INNOEXTRA+=" --incremental --incremental-lsn=$lsn " + fi + + check_extra + + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT}" + + if [[ -n $progress ]];then + get_footprint + tcmd="$pcmd | $tcmd" + elif [[ -n $rlimit ]];then + adjust_progress + tcmd="$pcmd | $tcmd" + fi + + set +e + timeit "Donor-Transfer" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + + if [ ${RC[0]} -ne 0 ]; then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + "Check ${DATA}/innobackup.backup.log" + exit 22 + elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "$tcmd finished with error: ${RC[1]}" + exit 22 + fi + + # innobackupex implicitly writes PID to fixed location in ${TMPDIR} + XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid" + + + else # BYPASS FOR IST + + wsrep_log_info "Bypassing the SST for IST" + STATE="${WSREP_SST_OPT_GTID}" + echo "continue" # now server can resume updating data + echo "${STATE}" > "${MAGIC_FILE}" + echo "1" > "${DATA}/${IST_FILE}" + get_keys + pushd ${DATA} 1>/dev/null + set +e + if [[ $encrypt -eq 1 ]];then + tcmd=" $ecmd | $tcmd" + fi + timeit "Donor-IST-Unencrypted-transfer" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )" + set -e + popd 1>/dev/null + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while streaming data to joiner node: " \ + "exit codes: ${RC[@]}" + exit 1 + fi + done + fi + + echo "done ${WSREP_SST_OPT_GTID}" + wsrep_log_info "Total time on donor: $totime seconds" + +elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ] +then + [[ -e $SST_PROGRESS_FILE ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" + touch $SST_PROGRESS_FILE + + if [[ ! -e ${DATA}/ibdata1 ]];then + incremental=0 + fi + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Incremental SST enabled" + #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF --basedir=/pxc --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1) + lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ') + wsrep_log_info "Recovered LSN: $lsn" + fi + + sencrypted=1 + nthreads=1 + + MODULE="xtrabackup_sst" + + # May need xtrabackup_checkpoints later on + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile + + ADDR=${WSREP_SST_OPT_ADDR} + if [ -z "${SST_PORT}" ] + then + SST_PORT=4444 + ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" + fi + + wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & + + trap sig_joiner_cleanup HUP PIPE INT TERM + trap cleanup_joiner EXIT + + if [[ -n $progress ]];then + adjust_progress + tcmd+=" | $pcmd" + fi + + if [[ $incremental -eq 1 ]];then + BDATA=$DATA + DATA=$(mktemp -d) + MAGIC_FILE="${DATA}/${INFO_FILE}" + fi + + get_keys + set +e + if [[ $encrypt -eq 1 && $sencrypted -eq 1 ]];then + strmcmd=" $ecmd | $strmcmd" + fi + + pushd ${DATA} 1>/dev/null + timeit "Joiner-Recv-Unencrypted" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" + popd 1>/dev/null + + set -e + + if [[ $sfmt == 'xbstream' ]];then + # Special handling till lp:1193240 is fixed" + if [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then + wsrep_log_error "Xbstream failed" + wsrep_log_error "Data directory ${DATA} may not be empty: lp:1193240" \ + "Manual intervention required in that case" + exit 32 + fi + fi + + wait %% # join for wait_for_listen thread + + for ecode in "${RC[@]}";do + if [[ $ecode -ne 0 ]];then + wsrep_log_error "Error while getting data from donor node: " \ + "exit codes: ${RC[@]}" + exit 32 + fi + done + + if [ ! -r "${MAGIC_FILE}" ] + then + # this message should cause joiner to abort + wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'" + wsrep_log_info "Contents of datadir" + wsrep_log_info "$(ls -l ${DATA}/**/*)" + exit 32 + fi + + if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null + then + wsrep_log_error "Parent mysqld process (PID:${WSREP_SST_OPT_PARENT}) terminated unexpectedly." + exit 32 + fi + + if [ ! -r "${DATA}/${IST_FILE}" ] + then + wsrep_log_info "Proceeding with SST" + wsrep_log_info "Removing existing ib_logfile files" + if [[ $incremental -ne 1 ]];then + rm -f ${DATA}/ib_logfile* + else + rm -f ${BDATA}/ib_logfile* + fi + + get_proc + + # Rebuild indexes for compact backups + if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then + wsrep_log_info "Index compaction detected" + rebuild=1 + fi + + if [[ $rebuild -eq 1 ]];then + nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) + wsrep_log_info "Rebuilding during prepare with $nthreads threads" + rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + fi + + if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then + + wsrep_log_info "Compressed qpress files found" + + if [[ ! -x `which qpress` ]];then + wsrep_log_error "qpress not found in path: $PATH" + exit 22 + fi + + if [[ -n $progress ]] && pv --help | grep -q 'line-mode';then + count=$(find ${DATA} -type f -name '*.qp' | wc -l) + count=$(( count*2 )) + if pv --help | grep -q FORMAT;then + pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" + else + pvopts="-f -s $count -l -N Decompression" + fi + pcmd="pv $pvopts" + adjust_progress + dcmd="$pcmd | xargs -n 2 qpress -T${nproc}d" + else + dcmd="xargs -n 2 qpress -T${nproc}d" + fi + + wsrep_log_info "Removing existing ibdata1 file" + rm -f ${DATA}/ibdata1 + + # Decompress the qpress files + wsrep_log_info "Decompression with $nproc threads" + timeit "Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + extcode=$? + + if [[ $extcode -eq 0 ]];then + wsrep_log_info "Removing qpress files after decompression" + find ${DATA} -type f -name '*.qp' -delete + if [[ $? -ne 0 ]];then + wsrep_log_error "Something went wrong with deletion of qpress files. Investigate" + fi + else + wsrep_log_error "Decompression failed. Exit code: $extcode" + exit 22 + fi + fi + + if [[ $incremental -eq 1 ]];then + # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues. + INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \ + --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log" + fi + + wsrep_log_info "Preparing the backup at ${DATA}" + timeit "Xtrabackup prepare stage" "$INNOAPPLY" + + if [[ $incremental -eq 1 ]];then + wsrep_log_info "Cleaning up ${DATA} after incremental SST" + [[ -d ${DATA} ]] && rm -rf ${DATA} + DATA=$BDATA + fi + + if [ $? -ne 0 ]; + then + wsrep_log_error "${INNOBACKUPEX_BIN} finished with errors. Check ${DATA}/innobackup.prepare.log" + exit 22 + fi + else + wsrep_log_info "${IST_FILE} received from donor: Running IST" + fi + + if [[ ! -r ${MAGIC_FILE} ]];then + wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable" + exit 2 + fi + + cat "${MAGIC_FILE}" # output UUID:seqno + wsrep_log_info "Total time on joiner: $totime seconds" +fi + +exit 0 |