summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorJulius Goryavsky <julius.goryavsky@mariadb.com>2022-02-22 10:45:06 +0100
committerJulius Goryavsky <julius.goryavsky@mariadb.com>2022-02-22 10:45:06 +0100
commit17e0f5224c8339ec08707a6ad0397bbf8c19bbd3 (patch)
tree90159c2bb04d45edc14565be33bafedf5694a60e /scripts
parent571eb9d7759a198e57f1efd03e50e0b15c627631 (diff)
downloadmariadb-git-17e0f5224c8339ec08707a6ad0397bbf8c19bbd3.tar.gz
MDEV-27524: Incorrect binlogs after Galera SST using rsync and mariabackup
This commit adds correct handling of binlogs for SST using rsync or mariabackup. Before this fix, binlogs were handled incorrectly - - only one (last) binary log file was transferred during SST, which then led to various failures (for example, when trying to list all events from the binary log). These bugs were long masked by flaws in the primitive binlogs handling code in the SST scripts, which causing binary logs files to be erased after transfer or not added to the binlog index on the joiner node. Now the correct transfer of all binary logs (not just the last of the binary log files) has been implemented both for the rsync (at the script level) and for the mariabackup (at the level of the main utility code). This commit also adds a new sst_max_binlogs=<n> parameter, which can be located in the [sst] section or in the [xtrabackup] section (historically, supported for mariabackup only, not for rsync), or in one of the server sections. This parameter specifies the number of binary log files to be sent to the joiner node during SST. This option is added for compatibility with old SST scripting behavior, which can be emulated by setting the sst_max_binlogs=1 (although in general this can cause problems for the reasons described above). In addition, setting the sst_max_binlogs=0 can be used to suppress the transmission of binary logs to the joiner nodes during SST (although sometimes a single file with the current binary log can still be transmitted to the joiner, even with sst_max_binlogs=0, because this sometimes necessary in modes that involve the use of GTIDs with Galera). Also, this commit ensures correct handling of paths to various innodb files and directories in the SST scripts, and fixes some problems with this that existed in mariabackup utility (which were associated with incorrect handling of the innodb_data_dir parameter in some scenarios). In addition, this commit contains the following enhancements: 1) Added tests for mtr, which check the correct work with binlogs after SST (using rsync and mariabackup); 2) Added correct handling of slashes at the end of all paths that the SST script receives as parameters; 3) Improved parsing code for --mysqld-args parameters. Now it correctly processes the sequence "--" after the name of the one-letter option; 4) Checking the secret signature during joiner authentication is made independent of presence of bash (as a unix shell) in the system and diff utility no longer needed to check certificates compliance; 5) All directories that are necessary for the correct placement of various logs are automatically created by SST scripts in advance (before running mariabackup on the joiner node); 6) Removal of old binary logs on joiner is done using the binlog index (if it exists) (not only by fixed pattern that based on the current binlog name, as before); 7) Paths for placing binary logs are correctly processed if they are set as relative paths (to the datadir); 8) SST scripts are made even more resistant to spaces in filenames (now for binlogs); 9) In case of failure, SST scripts now always end with an exit code other than zero; 10) SST script for rsync now correctly create a tar file with the binlogs, even if the paths to them (in the binlog index file) are specified as a mix of absolute and relative paths, and even if they do not match with the datadir path specified in the current configuration settings.
Diffstat (limited to 'scripts')
-rw-r--r--scripts/wsrep_sst_common.sh230
-rw-r--r--scripts/wsrep_sst_mariabackup.sh238
-rw-r--r--scripts/wsrep_sst_mysqldump.sh7
-rw-r--r--scripts/wsrep_sst_rsync.sh417
-rw-r--r--scripts/wsrep_sst_xtrabackup-v2.sh242
5 files changed, 721 insertions, 413 deletions
diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index deebe7cf820..f1d332a9822 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2017-2022 MariaDB
# Copyright (C) 2012-2015 Codership Oy
#
# This program is free software; you can redistribute it and/or modify
@@ -22,6 +22,62 @@ set -ue
# Setting the path for some utilities on CentOS
export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin"
+trim_string()
+{
+ if [ -n "$BASH_VERSION" ]; then
+ local pattern="[![:space:]${2:-}]"
+ local x="${1#*$pattern}"
+ local z=${#1}
+ x=${#x}
+ if [ $x -ne $z ]; then
+ local y="${1%$pattern*}"
+ y=${#y}
+ x=$(( z-x-1 ))
+ y=$(( y-x+1 ))
+ printf '%s' "${1:$x:$y}"
+ else
+ printf ''
+ fi
+ else
+ local pattern="[[:space:]${2:-}]"
+ echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g"
+ fi
+}
+
+trim_dir()
+{
+ local t=$(trim_string "$1")
+ if [ "$t" != '/' ]; then
+ if [ "${t%/}" != "$t" ]; then
+ t=$(trim_string "${t%/}")
+ fi
+ else
+ t='.'
+ fi
+ if [ -n "$BASH_VERSION" ]; then
+ printf '%s' "$t"
+ else
+ echo "$t"
+ fi
+}
+
+to_minuses()
+{
+ local x="$1"
+ local t="${1#*_}"
+ local r=""
+ while [ "$t" != "$x" ]; do
+ r="$r${x%%_*}-"
+ x="$t"
+ t="${t#*_}"
+ done
+ if [ -n "$BASH_VERSION" ]; then
+ printf '%s' "$r$x"
+ else
+ echo "$r$x"
+ fi
+}
+
WSREP_SST_OPT_BYPASS=0
WSREP_SST_OPT_BINLOG=""
WSREP_SST_OPT_BINLOG_INDEX=""
@@ -44,9 +100,9 @@ WSREP_SST_OPT_ADDR_PORT=""
WSREP_SST_OPT_HOST=""
WSREP_SST_OPT_HOST_UNESCAPED=""
WSREP_SST_OPT_HOST_ESCAPED=""
-INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}"
-INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}"
-INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}"
+INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}")
+INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}")
+INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}")
INNODB_FORCE_RECOVERY=""
INNOEXTRA=""
@@ -138,22 +194,22 @@ case "$1" in
;;
'--datadir')
# Let's remove the trailing slash:
- readonly WSREP_SST_OPT_DATA="${2%/}"
+ readonly WSREP_SST_OPT_DATA=$(trim_dir "$2")
shift
;;
'--innodb-data-home-dir')
# Let's remove the trailing slash:
- readonly INNODB_DATA_HOME_DIR="${2%/}"
+ readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2")
shift
;;
'--innodb-log-group-home-dir')
# Let's remove the trailing slash:
- readonly INNODB_LOG_GROUP_HOME="${2%/}"
+ readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2")
shift
;;
'--innodb-undo-directory')
# Let's remove the trailing slash:
- readonly INNODB_UNDO_DIR="${2%/}"
+ readonly INNODB_UNDO_DIR=$(trim_dir "$2")
shift
;;
'--defaults-file')
@@ -247,6 +303,7 @@ case "$1" in
'--mysqld-args')
original_cmd=""
shift
+ cmd_tail=0
while [ $# -gt 0 ]; do
lname="${1#--}"
# "--" is interpreted as the end of the list of options:
@@ -261,7 +318,7 @@ case "$1" in
shift
done
fi
- break;
+ break
fi
# Make sure the argument does not start with "--", otherwise it
# is a long option, which is processed after this "if":
@@ -301,15 +358,25 @@ case "$1" in
if [ "${2#-}" = "$2" ]; then
shift
value="$1"
+ elif [ "$2" = '--' ]; then
+ shift
+ if [ $# -gt 1 ]; then
+ cmd_tail=1
+ shift
+ value="$1"
+ fi
fi
fi
- if [ $option = 'h' ]; then
+ if [ "$option" = 'h' ]; then
if [ -z "$WSREP_SST_OPT_DATA" ]; then
- MYSQLD_OPT_DATADIR="${value%/}"
+ MYSQLD_OPT_DATADIR=$(trim_dir "$value")
fi
- elif [ $option != 'u' -a \
- $option != 'P' ]
+ elif [ "$option" != 'u' -a \
+ "$option" != 'P' ]
then
+ if [ $cmd_tail -ne 0 ]; then
+ option="$option --"
+ fi
if [ -z "$value" ]; then
slist="$slist$option"
elif [ -z "$slist" ]; then
@@ -317,9 +384,16 @@ case "$1" in
else
slist="$slist -$option '$value'"
fi
+ break
+ fi
+ if [ $cmd_tail -ne 0 ]; then
+ if [ -n "$slist" ]; then
+ slist="$slist --"
+ else
+ slist='-'
+ fi
fi
break
-
else
slist="$slist$option"
fi
@@ -329,7 +403,7 @@ case "$1" in
original_cmd="$original_cmd -$slist"
fi
elif [ -z "$options" ]; then
- # We found an equal sign without any characters after it:
+ # We found an minus sign without any characters after it:
original_cmd="$original_cmd -"
else
# We found a value that does not start with a minus -
@@ -338,12 +412,25 @@ case "$1" in
original_cmd="$original_cmd '$1'"
fi
shift
- continue;
+ if [ $cmd_tail -ne 0 ]; then
+ # All other arguments must be copied unchanged:
+ while [ $# -gt 0 ]; do
+ original_cmd="$original_cmd '$1'"
+ shift
+ done
+ break
+ fi
+ continue
fi
# Now we are sure that we are working with an option
# that has a "long" name, so remove all characters after
# the first equal sign:
option="${1%%=*}"
+ # If the option name contains underscores, then replace
+ # them to minuses:
+ if [ "${option#*_}" != "$option" ]; then
+ option=$(to_minuses "$option")
+ fi
# The "--loose-" prefix should not affect the recognition
# of the option name:
if [ "${option#--loose-}" != "$option" ]; then
@@ -370,19 +457,19 @@ case "$1" in
case "$option" in
'--innodb-data-home-dir')
if [ -z "$INNODB_DATA_HOME_DIR" ]; then
- MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}"
+ MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value")
fi
skip_mysqld_arg=1
;;
'--innodb-log-group-home-dir')
if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
- MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}"
+ MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value")
fi
skip_mysqld_arg=1
;;
'--innodb-undo-directory')
if [ -z "$INNODB_UNDO_DIR" ]; then
- MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}"
+ MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value")
fi
skip_mysqld_arg=1
;;
@@ -412,7 +499,7 @@ case "$1" in
;;
'--datadir')
if [ -z "$WSREP_SST_OPT_DATA" ]; then
- MYSQLD_OPT_DATADIR="${value%/}"
+ MYSQLD_OPT_DATADIR=$(trim_dir "$value")
fi
skip_mysqld_arg=1
;;
@@ -471,8 +558,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then
if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then
# If the WSREP_SST_OPT_BINLOG variable is not set, but
# --log-basename is present among the arguments to mysqld,
- # then set WSREP_SST_OPT_BINLOG equal to the base name with
- # the "-bin" suffix:
+ # then set WSREP_SST_OPT_BINLOG equal to the base name
+ # with the "-bin" suffix:
readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin"
else
# Take the default name:
@@ -525,26 +612,23 @@ get_binlog()
WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index')
fi
# if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set,
- # try to get it from my.cnf:
+ # then try to get it from my.cnf:
if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then
WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename')
fi
if [ -z "$WSREP_SST_OPT_BINLOG" ]; then
- # If the --log-bin option is specified without a parameter,
+ # If the log-bin option is specified without a parameter,
# then we need to build the name of the index file according
# to the rules described in the server documentation:
- if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \
- $(in_config '--mysqld' 'log-bin') -eq 1 ]
- then
+ if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then
if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then
# If the WSREP_SST_OPT_BINLOG variable is not set, but
# --log-basename is present among the arguments of mysqld,
- # then set WSREP_SST_OPT_BINLOG equal to the base name with
- # the "-bin" suffix:
+ # then set WSREP_SST_OPT_BINLOG equal to the base name
+ # with the "-bin" suffix:
readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin"
else
- # If the --log-bin option is present without a value, then
- # we take the default name:
+ # Take the default name:
readonly WSREP_SST_OPT_BINLOG='mysql-bin'
fi
fi
@@ -554,13 +638,13 @@ get_binlog()
# it according to the specifications for the server:
if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then
if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then
- # If the WSREP_SST_OPT_BINLOG variable is not set, but
+ # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but
# --log-basename is present among the arguments of mysqld,
- # then set WSREP_SST_OPT_BINLOG equal to the base name with
- # the "-bin" suffix:
+ # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name
+ # with the "-bin" suffix:
readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index"
else
- # the default name (note that base of this name
+ # Use the default name (note that base of this name
# is already defined above):
readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index"
fi
@@ -632,7 +716,7 @@ commandex()
# try to use my_print_defaults, mysql and mysqldump that come
# with the sources (for MTR suite):
script_binary=$(dirname "$0")
-SCRIPTS_DIR=$(cd "$script_binary"; pwd -P)
+SCRIPTS_DIR=$(cd "$script_binary"; pwd)
EXTRA_DIR="$SCRIPTS_DIR/../extra"
CLIENT_DIR="$SCRIPTS_DIR/../client"
@@ -754,7 +838,11 @@ parse_cnf()
if [ -z "$reval" ]; then
[ -n "${3:-}" ] && reval="$3"
fi
- echo "$reval"
+ if [ -n "$BASH_VERSION" ]; then
+ printf '%s' "$reval"
+ else
+ echo "$reval"
+ fi
}
#
@@ -804,7 +892,11 @@ in_config()
break
fi
done
- echo $found
+ if [ -n "$BASH_VERSION" ]; then
+ printf '%s' $found
+ else
+ echo $found
+ fi
}
wsrep_auth_not_set()
@@ -937,11 +1029,22 @@ wsrep_gen_secret()
{
get_openssl
if [ -n "$OPENSSL_BINARY" ]; then
- echo $("$OPENSSL_BINARY" rand -hex 16)
- else
- printf "%04x%04x%04x%04x%04x%04x%04x%04x" \
+ "$OPENSSL_BINARY" rand -hex 16
+ elif [ -n "$BASH_VERSION" ]; then
+ printf '%04x%04x%04x%04x%04x%04x%04x%04x' \
$RANDOM $RANDOM $RANDOM $RANDOM \
$RANDOM $RANDOM $RANDOM $RANDOM
+ elif [ -n "$(commandex cksum)" -a \
+ -n "$(commandex printf)" ]
+ then
+ printf '%08x%08x%08x%08x' \
+ $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \
+ $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \
+ $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \
+ $(head -8 /dev/urandom | cksum | cut -d ' ' -f1)
+ else
+ wsrep_log_error "Unable to generate 16-byte secret"
+ exit 22
fi
}
@@ -979,14 +1082,14 @@ is_local_ip()
if [ -n "$ip_util" ]; then
# ip address show ouput format is " inet[6] <address>/<mask>":
"$ip_util" address show \
- | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \
+ | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \
| grep -F " $1/" >/dev/null && return 0
else
local ifconfig_util=$(commandex 'ifconfig')
if [ -n "$ifconfig_util" ]; then
# ifconfig output format is " inet[6] <address> ...":
"$ifconfig_util" \
- | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \
+ | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \
| grep -F " $1 " >/dev/null && return 0
fi
fi
@@ -1049,7 +1152,7 @@ check_port()
ss -nlpH "( sport = :$port )" 2>/dev/null | \
grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0
else
- wsrep_log_error "unknown sockets utility"
+ wsrep_log_error "Unknown sockets utility"
exit 2 # ENOENT
fi
@@ -1158,13 +1261,6 @@ verify_cert_matches_key()
exit 22
fi
- # If the diff utility is not installed, then
- # we will not do this certificate check:
- if [ -z "$(commandex diff)" ]; then
- wsrep_log_info "diff utility not found"
- return
- fi
-
# If the openssl utility is not installed, then
# we will not do this certificate check:
get_openssl
@@ -1175,9 +1271,9 @@ verify_cert_matches_key()
# Generate the public key from the cert and the key.
# They should match (otherwise we can't create an SSL connection).
- if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \
- <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1
- then
+ local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :)
+ local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :)
+ if [ "$pk1" != "$pk2" ]; then
wsrep_log_error "******************* FATAL ERROR *****************"
wsrep_log_error "* The certificate and private key do not match. *"
wsrep_log_error "* Please check your certificate and key files. *"
@@ -1220,28 +1316,6 @@ check_for_version()
return 0
}
-trim_string()
-{
- if [ -n "$BASH_VERSION" ]; then
- local pattern="[![:space:]${2:-}]"
- local x="${1#*$pattern}"
- local z=${#1}
- x=${#x}
- if [ $x -ne $z ]; then
- local y="${1%$pattern*}"
- y=${#y}
- x=$(( z-x-1 ))
- y=$(( y-x+1 ))
- printf '%s' "${1:$x:$y}"
- else
- printf ''
- fi
- else
- local pattern="[[:space:]${2:-}]"
- echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g"
- fi
-}
-
#
# Check whether process is still running.
# The first parameter contains the name of the PID file.
@@ -1272,6 +1346,10 @@ check_pid()
rm -f "$pid_file" || :
fi
fi
+ local config="${3:-}"
+ if [ -n "$config" -a -f "$config" ]; then
+ rm -f "$config" || :
+ fi
CHECK_PID=0
return 1
}
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 471b0e5f5b0..2eedff554e6 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -2,7 +2,7 @@
set -ue
-# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2017-2022 MariaDB
# Copyright (C) 2013 Percona Inc
#
# This program is free software; you can redistribute it and/or modify
@@ -40,7 +40,7 @@ tcert=""
tcap=""
tpem=""
tkey=""
-tmode="DISABLED"
+tmode='DISABLED'
sockopt=""
progress=""
ttime=0
@@ -85,13 +85,13 @@ backup_threads=""
encrypt_threads=""
encrypt_chunk=""
-readonly SECRET_TAG="secret"
+readonly SECRET_TAG='secret'
# Required for backup locks
# For backup locks it is 1 sent by joiner
sst_ver=1
-if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then
+if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then
pvopts="$pvopts $pvformat"
fi
pcmd="pv $pvopts"
@@ -104,17 +104,14 @@ if [ -z "$BACKUP_BIN" ]; then
fi
DATA="$WSREP_SST_OPT_DATA"
-INFO_FILE="xtrabackup_galera_info"
-IST_FILE="xtrabackup_ist"
+INFO_FILE='xtrabackup_galera_info'
+IST_FILE='xtrabackup_ist'
MAGIC_FILE="$DATA/$INFO_FILE"
INNOAPPLYLOG="$DATA/mariabackup.prepare.log"
INNOMOVELOG="$DATA/mariabackup.move.log"
INNOBACKUPLOG="$DATA/mariabackup.backup.log"
-# Setting the path for ss and ip
-export PATH="/usr/sbin:/sbin:$PATH"
-
timeit()
{
local stage="$1"
@@ -154,7 +151,7 @@ get_keys()
return
fi
- if [ $sfmt = 'tar' ]; then
+ if [ "$sfmt" = 'tar' ]; then
wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \
"cannot be enabled with tar format"
encrypt=-1
@@ -184,11 +181,11 @@ get_keys()
exit 2
fi
ecmd="'$OPENSSL_BINARY' enc -$ealgo"
- if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then
+ if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then
ecmd="$ecmd -pbkdf2"
- elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then
ecmd="$ecmd -iter 1"
- elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then
ecmd="$ecmd -md sha256"
fi
if [ -z "$ekey" ]; then
@@ -229,15 +226,15 @@ get_keys()
get_transfer()
{
- if [ $tfmt = 'nc' ]; then
+ if [ "$tfmt" = 'nc' ]; then
wsrep_log_info "Using netcat as streamer"
wsrep_check_programs nc
- tcmd="nc"
+ tcmd='nc'
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
- if nc -h 2>&1 | grep -q 'ncat'; then
+ if nc -h 2>&1 | grep -q -F 'ncat'; then
wsrep_log_info "Using Ncat as streamer"
tcmd="$tcmd -l"
- elif nc -h 2>&1 | grep -qw -- '-d'; then
+ elif nc -h 2>&1 | grep -qw -F -- '-d'; then
wsrep_log_info "Using Debian netcat as streamer"
tcmd="$tcmd -dl"
if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then
@@ -259,14 +256,14 @@ get_transfer()
# transfer and cause the command to timeout.
# Older versions of netcat did not need this flag and will
# return an error if the flag is used.
- if nc -h 2>&1 | grep -qw -- '-N'; then
+ if nc -h 2>&1 | grep -qw -F -- '-N'; then
tcmd="$tcmd -N"
wsrep_log_info "Using nc -N"
fi
# netcat doesn't understand [] around IPv6 address
- if nc -h 2>&1 | grep -q ncat; then
+ if nc -h 2>&1 | grep -q -F 'ncat'; then
wsrep_log_info "Using Ncat as streamer"
- elif nc -h 2>&1 | grep -qw -- '-d'; then
+ elif nc -h 2>&1 | grep -qw -F -- '-d'; then
wsrep_log_info "Using Debian netcat as streamer"
else
wsrep_log_info "Using traditional netcat as streamer"
@@ -456,7 +453,7 @@ adjust_progress()
fi
elif [ -z "$progress" -a -n "$rlimit" ]; then
# When rlimit is non-zero
- pcmd="pv -q"
+ pcmd='pv -q'
fi
if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
@@ -583,8 +580,14 @@ get_stream()
sig_joiner_cleanup()
{
+ local estatus=$?
+ if [ $estatus -ne 0 ]; then
+ wsrep_log_error "Cleanup after exit with status: $estatus"
+ fi
wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+ [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+ exit $estatus
}
cleanup_at_exit()
@@ -595,6 +598,8 @@ cleanup_at_exit()
wsrep_log_error "Cleanup after exit with status: $estatus"
fi
+ [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD"
+
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
wsrep_log_info "Removing the sst_in_progress file"
wsrep_cleanup_progress_file
@@ -624,7 +629,7 @@ cleanup_at_exit()
fi
# Final cleanup
- pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :)
+ pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :)
# This means no setsid done in mysqld.
# We don't want to kill mysqld here otherwise.
@@ -715,7 +720,7 @@ recv_joiner()
local ltcmd="$tcmd"
if [ $tmt -gt 0 ]; then
if [ -n "$(commandex timeout)" ]; then
- if timeout --help | grep -qw -- '-k'; then
+ if timeout --help | grep -qw -F -- '-k'; then
ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
else
ltcmd="timeout -s9 $tmt $tcmd"
@@ -815,7 +820,9 @@ monitor_process()
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
-if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
+if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \
+ "$WSREP_SST_OPT_ROLE" != 'donor' ]
+then
wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
exit 22
fi
@@ -823,25 +830,17 @@ fi
read_cnf
setup_ports
-if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then
disver=' --no-version-check'
fi
-# if no command line argument and INNODB_DATA_HOME_DIR environment variable
-# is not set, try to get it from my.cnf:
-if [ -z "$INNODB_DATA_HOME_DIR" ]; then
- INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
-fi
-
OLD_PWD="$(pwd)"
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_DATA_HOME_DIR" ]; then
- # handle both relative and absolute paths
- [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
- cd "$INNODB_DATA_HOME_DIR"
+if [ -n "$DATA" -a "$DATA" != '.' ]; then
+ [ ! -d "$DATA" ] && mkdir -p "$DATA"
+ cd "$DATA"
fi
-INNODB_DATA_HOME_DIR=$(pwd -P)
+DATA_DIR="$(pwd)"
cd "$OLD_PWD"
@@ -869,7 +868,7 @@ if [ $ssyslog -eq 1 ]; then
else
if [ $sstlogarchive -eq 1 ]
then
- ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
+ ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N')
if [ -n "$sstlogarchivedir" ]; then
if [ ! -d "$sstlogarchivedir" ]; then
@@ -929,7 +928,7 @@ setup_commands()
recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY"
fi
INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY"
- INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
+ INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP"
}
@@ -1052,6 +1051,11 @@ then
iopts="--parallel=$backup_threads${iopts:+ }$iopts"
fi
+ max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs')
+ if [ -n "$max_binlogs" ]; then
+ iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts"
+ fi
+
setup_commands
set +e
timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
@@ -1096,6 +1100,7 @@ then
echo "done $WSREP_SST_OPT_GTID"
wsrep_log_info "Total time on donor: $totime seconds"
+ wsrep_log_info "mariabackup SST/IST completed on donor"
elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
then
@@ -1103,22 +1108,53 @@ then
wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
[ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE"
- ib_home_dir="$INNODB_DATA_HOME_DIR"
+ # if no command line argument and INNODB_DATA_HOME_DIR environment
+ # variable is not set, try to get it from the my.cnf:
+ if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+ INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
+ INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR")
+ fi
+
+ if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+ cd "$INNODB_DATA_HOME_DIR"
+ ib_home_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
# if no command line argument and INNODB_LOG_GROUP_HOME is not set,
- # try to get it from my.cnf:
+ # then try to get it from the my.cnf:
if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir')
+ INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME")
fi
- ib_log_dir="$INNODB_LOG_GROUP_HOME"
+ if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME"
+ cd "$INNODB_LOG_GROUP_HOME"
+ ib_log_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
- # if no command line argument then try to get it from my.cnf:
+ # if no command line argument and INNODB_UNDO_DIR is not set,
+ # then try to get it from the my.cnf:
if [ -z "$INNODB_UNDO_DIR" ]; then
INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
+ INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR")
fi
- ib_undo_dir="$INNODB_UNDO_DIR"
+ if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
+ cd "$INNODB_UNDO_DIR"
+ ib_undo_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
if [ -n "$backup_threads" ]; then
impts="--parallel=$backup_threads${impts:+ }$impts"
@@ -1142,14 +1178,13 @@ then
stagemsg='Joiner-Recv'
- MODULE="xtrabackup_sst"
+ MODULE='xtrabackup_sst'
[ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
# May need xtrabackup_checkpoints later on
[ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary"
[ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info"
- [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0"
ADDR="$WSREP_SST_OPT_ADDR"
@@ -1165,7 +1200,7 @@ then
exit 42
fi
CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \
- tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
+ tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
sed s/\ %//)
fi
MY_SECRET="$(wsrep_gen_secret)"
@@ -1219,6 +1254,36 @@ then
jpid=$!
wsrep_log_info "Proceeding with SST"
+ get_binlog
+
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
+ binlog_base=$(basename "$WSREP_SST_OPT_BINLOG")
+ binlog_index="$WSREP_SST_OPT_BINLOG_INDEX"
+ cd "$DATA"
+ wsrep_log_info "Cleaning the old binary logs"
+ # If there is a file with binlogs state, delete it:
+ [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2
+ # Clean up the old binlog files and index:
+ if [ -f "$binlog_index" ]; then
+ while read bin_file || [ -n "$bin_file" ]; do
+ rm -fv "$bin_file" 1>&2 || :
+ done < "$binlog_index"
+ rm -fv "$binlog_index" 1>&2
+ fi
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
+ -d "$binlog_dir" ]
+ then
+ cd "$binlog_dir"
+ if [ "$(pwd)" != "$DATA_DIR" ]; then
+ wsrep_log_info \
+ "Cleaning the binlog directory '$binlog_dir' as well"
+ fi
+ fi
+ rm -fv "$binlog_base".[0-9]* 1>&2 || :
+ cd "$OLD_PWD"
+ fi
+
wsrep_log_info \
"Cleaning the existing datadir and innodb-data/log directories"
if [ "$OS" = 'FreeBSD' ]; then
@@ -1235,20 +1300,6 @@ then
-o -exec rm -rfv {} 1>&2 \+
fi
- get_binlog
-
- if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
- binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
- if [ -d "$binlog_dir" ]; then
- cd "$binlog_dir"
- wsrep_log_info "Cleaning the binlog directory $binlog_dir as well"
- rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || :
- [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \
- rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+
- cd "$OLD_PWD"
- fi
- fi
-
TDATA="$DATA"
DATA="$DATA/.sst"
@@ -1282,11 +1333,13 @@ then
dcmd="xargs -n 2 qpress -dT$nproc"
- if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
+ if [ -n "$progress" ] && \
+ pv --help | grep -qw -F -- '--line-mode'
+ then
count=$(find "$DATA" -type f -name '*.qp' | wc -l)
count=$(( count*2 ))
pvopts="-f -s $count -l -N Decompression"
- if pv --help | grep -qw -- '-F'; then
+ if pv --help | grep -qw -F -- '-F'; then
pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
fi
pcmd="pv $pvopts"
@@ -1296,8 +1349,9 @@ then
# Decompress the qpress files
wsrep_log_info "Decompression with $nproc threads"
- timeit "Joiner-Decompression" \
- "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+ timeit 'Joiner-Decompression' \
+ "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \
+ $dcmd"
extcode=$?
if [ $extcode -eq 0 ]; then
@@ -1314,25 +1368,9 @@ then
fi
fi
- if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
-
- BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG")
- BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG")
-
- # To avoid comparing data directory and BINLOG_DIRNAME
- mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || :
-
- cd "$BINLOG_DIRNAME"
- for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do
- echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX"
- done
- cd "$OLD_PWD"
-
- fi
-
wsrep_log_info "Preparing the backup at $DATA"
setup_commands
- timeit "mariabackup prepare stage" "$INNOAPPLY"
+ timeit 'mariabackup prepare stage' "$INNOAPPLY"
if [ $? -ne 0 ]; then
wsrep_log_error "mariabackup apply finished with errors." \
@@ -1340,10 +1378,43 @@ then
exit 22
fi
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ cd "$DATA"
+ binlogs=""
+ if [ -f 'xtrabackup_binlog_info' ]; then
+ NL=$'\n'
+ while read bin_string || [ -n "$bin_string" ]; do
+ bin_file=$(echo "$bin_string" | cut -f1)
+ if [ -f "$bin_file" ]; then
+ binlogs="$binlogs${binlogs:+$NL}$bin_file"
+ fi
+ done < 'xtrabackup_binlog_info'
+ else
+ binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :)
+ fi
+ cd "$DATA_DIR"
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then
+ [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir"
+ fi
+ index_dir=$(dirname "$binlog_index");
+ if [ -n "$index_dir" -a "$index_dir" != '.' ]; then
+ [ ! -d "$index_dir" ] && mkdir -p "$index_dir"
+ fi
+ if [ -n "$binlogs" ]; then
+ wsrep_log_info "Moving binary logs to $binlog_dir"
+ echo "$binlogs" | \
+ while read bin_file || [ -n "$bin_file" ]; do
+ mv "$DATA/$bin_file" "$binlog_dir"
+ echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index"
+ done
+ fi
+ cd "$OLD_PWD"
+ fi
+
MAGIC_FILE="$TDATA/$INFO_FILE"
wsrep_log_info "Moving the backup to $TDATA"
- timeit "mariabackup move stage" "$INNOMOVE"
+ timeit 'mariabackup move stage' "$INNOMOVE"
if [ $? -eq 0 ]; then
wsrep_log_info "Move successful, removing $DATA"
rm -rf "$DATA"
@@ -1370,6 +1441,7 @@ then
cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id
wsrep_log_info "Total time on joiner: $totime seconds"
+ wsrep_log_info "mariabackup SST/IST completed on joiner"
fi
exit 0
diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh
index 1c8fc181328..3c92f489cb5 100644
--- a/scripts/wsrep_sst_mysqldump.sh
+++ b/scripts/wsrep_sst_mysqldump.sh
@@ -3,7 +3,7 @@
set -ue
# Copyright (C) 2009-2015 Codership Oy
-# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2017-2022 MariaDB
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -40,8 +40,7 @@ then
fi
# Check client version
-if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null
-then
+if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then
$MYSQL_CLIENT --version >&2
wsrep_log_error "this operation requires MySQL client version 10.1 or newer"
exit $EINVAL
@@ -95,7 +94,7 @@ DROP PREPARE stmt;"
SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';"
SET_WSREP_GTID_DOMAIN_ID=""
-if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then
+if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then
SET_WSREP_GTID_DOMAIN_ID="
SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0);
SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0');
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index cc73db5da87..7845f143088 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -2,7 +2,7 @@
set -ue
-# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2017-2022 MariaDB
# Copyright (C) 2010-2014 Codership Oy
#
# This program is free software; you can redistribute it and/or modify
@@ -36,6 +36,8 @@ cleanup_joiner()
{
local failure=0
+ [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD"
+
wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \
"stunnel PID=$STUNNEL_REAL_PID"
@@ -58,6 +60,7 @@ cleanup_joiner()
if [ $failure -eq 0 ]; then
if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+ [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
else
wsrep_log_warning "rsync cleanup failed."
fi
@@ -140,66 +143,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
-BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar"
-BINLOG_N_FILES=1
-
get_binlog
if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
- BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG")
- BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG")
+ binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
+ binlog_base=$(basename "$WSREP_SST_OPT_BINLOG")
fi
-# if no command line argument and INNODB_LOG_GROUP_HOME is not set,
-# try to get it from my.cnf:
-if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
- INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir')
+OLD_PWD="$(pwd)"
+
+DATA="$WSREP_SST_OPT_DATA"
+if [ -n "$DATA" -a "$DATA" != '.' ]; then
+ [ ! -d "$DATA" ] && mkdir -p "$DATA"
+ cd "$DATA"
fi
+DATA_DIR="$(pwd)"
-OLD_PWD="$(pwd)"
+cd "$OLD_PWD"
+
+BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar"
-WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME"
+ib_log_dir="$DATA_DIR"
+ib_home_dir="$DATA_DIR"
+ib_undo_dir="$DATA_DIR"
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$WSREP_LOG_DIR" ]; then
- # handle both relative and absolute paths
- [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR"
- cd "$WSREP_LOG_DIR"
+# if no command line argument and INNODB_LOG_GROUP_HOME is not set,
+# then try to get it from the my.cnf:
+if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
+ INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir')
+ INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME")
fi
-WSREP_LOG_DIR=$(pwd -P)
-cd "$OLD_PWD"
+if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME"
+ cd "$INNODB_LOG_GROUP_HOME"
+ ib_log_dir="$(pwd)"
+ cd "$OLD_PWD"
+fi
-# if no command line argument and INNODB_DATA_HOME_DIR environment variable
-# is not set, try to get it from my.cnf:
+# if no command line argument and INNODB_DATA_HOME_DIR environment
+# variable is not set, try to get it from the my.cnf:
if [ -z "$INNODB_DATA_HOME_DIR" ]; then
INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
+ INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR")
fi
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_DATA_HOME_DIR" ]; then
- # handle both relative and absolute paths
+if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
[ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
cd "$INNODB_DATA_HOME_DIR"
+ ib_home_dir="$(pwd)"
+ cd "$OLD_PWD"
fi
-INNODB_DATA_HOME_DIR=$(pwd -P)
-
-cd "$OLD_PWD"
-# if no command line argument then try to get it from my.cnf:
+# if no command line argument and INNODB_UNDO_DIR is not set,
+# then try to get it from the my.cnf:
if [ -z "$INNODB_UNDO_DIR" ]; then
INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
+ INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR")
fi
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_UNDO_DIR" ]; then
- # handle both relative and absolute paths
+if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
[ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
cd "$INNODB_UNDO_DIR"
+ ib_undo_dir="$(pwd)"
+ cd "$OLD_PWD"
fi
-INNODB_UNDO_DIR=$(pwd -P)
-
-cd "$OLD_PWD"
encgroups='--mysqld|sst'
@@ -276,7 +290,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then
CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST"
fi
if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then
- CHECK_OPT_LOCAL="checkHost = localhost"
+ CHECK_OPT_LOCAL='checkHost = localhost'
fi
fi
fi
@@ -293,14 +307,58 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then
fi
fi
-readonly SECRET_TAG="secret"
+readonly SECRET_TAG='secret'
-if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
-then
+SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid"
+
+# give some time for previous SST to complete:
+check_round=0
+while check_pid "$SST_PID" 0; do
+ wsrep_log_info "Previous SST is not completed, waiting for it to exit"
+ check_round=$(( check_round + 1 ))
+ if [ $check_round -eq 20 ]; then
+ wsrep_log_error "previous SST script still running."
+ exit 114 # EALREADY
+ fi
+ sleep 1
+done
+
+echo $$ > "$SST_PID"
+
+# give some time for stunnel from the previous SST to complete:
+check_round=0
+while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do
+ wsrep_log_info "Lingering stunnel daemon found at startup," \
+ "waiting for it to exit"
+ check_round=$(( check_round + 1 ))
+ if [ $check_round -eq 10 ]; then
+ wsrep_log_error "stunnel daemon still running."
+ exit 114 # EALREADY
+ fi
+ sleep 1
+done
+
+MODULE='rsync_sst'
+RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
+RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+
+# give some time for rsync from the previous SST to complete:
+check_round=0
+while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do
+ wsrep_log_info "Lingering rsync daemon found at startup," \
+ "waiting for it to exit"
+ check_round=$(( check_round + 1 ))
+ if [ $check_round -eq 10 ]; then
+ wsrep_log_error "rsync daemon still running."
+ exit 114 # EALREADY
+ fi
+ sleep 1
+done
- [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
- [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
- [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
+[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
+
+if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
if [ -n "$STUNNEL" ]
then
@@ -319,8 +377,6 @@ ${VERIFY_OPT}
${CHECK_OPT}
${CHECK_OPT_LOCAL}
EOF
- else
- [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
fi
RC=0
@@ -333,7 +389,7 @@ EOF
[ -f "$FLUSHED" ] && rm -f "$FLUSHED"
[ -f "$ERROR" ] && rm -f "$ERROR"
- echo "flush tables"
+ echo 'flush tables'
# Wait for :
# (a) Tables to be flushed, AND
@@ -357,32 +413,78 @@ EOF
sync
- if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ]
- then
- # Prepare binlog files
- cd "$BINLOG_DIRNAME"
-
- binlog_files_full=$(tail -n $BINLOG_N_FILES \
- "$WSREP_SST_OPT_BINLOG_INDEX")
- binlog_files=""
- for file in $binlog_files_full; do
- binlog_file=$(basename "$file")
- binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'"
- done
-
- if [ -n "$binlog_files" ]; then
- wsrep_log_info "Preparing binlog files for transfer:"
- eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ # Change the directory to binlog base (if possible):
+ cd "$DATA"
+ # Let's check the existence of the file with the index:
+ if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then
+ # Let's read the binlog index:
+ max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs')
+ if [ -n "$max_binlogs" ]; then
+ binlog_files=""
+ if [ $max_binlogs -gt 0 ]; then
+ binlog_files=$(tail -n $max_binlogs \
+ "$WSREP_SST_OPT_BINLOG_INDEX")
+ fi
+ else
+ binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX")
+ fi
+ if [ -n "$binlog_files" ]; then
+ # Preparing binlog files for transfer:
+ wsrep_log_info "Preparing binlog files for transfer:"
+ tar_type=0
+ if tar --help | grep -qw -F -- '--transform'; then
+ tar_type=1
+ elif tar --help | grep -qw -E -- '-s[[:space:]]pattern'
+ then
+ tar_type=2
+ fi
+ if [ $tar_type -ne 0 ]; then
+ # Preparing list of the binlog file names:
+ echo "$binlog_files" | {
+ binlogs=""
+ while read bin_file || [ -n "$bin_file" ]; do
+ [ ! -f "$bin_file" ] && continue
+ binlogs="$binlogs${binlogs:+ }'$bin_file'"
+ done
+ if [ -n "$binlogs" ]; then
+ tar_options='/^.*\///g'
+ if [ $tar_type -eq 1 ]; then
+ tar_options="--transform='s$tar_options'"
+ else
+ tar_options="-s '$tar_options'"
+ fi
+ eval tar -P $tar_options \
+ -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2
+ fi
+ }
+ else
+ tar_options='-cvf'
+ echo "$binlog_files" | \
+ while read bin_file || [ -n "$bin_file" ]; do
+ [ ! -f "$bin_file" ] && continue
+ bin_dir=$(dirname "$bin_file")
+ bin_base=$(basename "$bin_file")
+ if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then
+ tar $tar_options "$BINLOG_TAR_FILE" \
+ -C "$bin_dir" "$bin_base" >&2
+ else
+ tar $tar_options "$BINLOG_TAR_FILE" \
+ "$bin_base" >&2
+ fi
+ tar_options='-rvf'
+ done
+ fi
+ fi
fi
-
cd "$OLD_PWD"
fi
- # Use deltaxfer only for WAN
+ # Use deltaxfer only for WAN:
inv=$(basename "$0")
WHOLE_FILE_OPT=""
if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then
- WHOLE_FILE_OPT="--whole-file"
+ WHOLE_FILE_OPT='--whole-file'
fi
# Old filter - include everything except selected
@@ -399,9 +501,9 @@ FILTER="-f '- /lost+found'
-f '- /.pid'
-f '- /.conf'
-f '+ /wsrep_sst_binlog.tar'
- -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
- -f '- $INNODB_DATA_HOME_DIR/ibdata*'
- -f '+ $INNODB_UNDO_DIR/undo*'
+ -f '- $ib_home_dir/ib_lru_dump'
+ -f '- $ib_home_dir/ibdata*'
+ -f '+ $ib_undo_dir/undo*'
-f '+ /*/'
-f '- /*'"
@@ -435,7 +537,7 @@ FILTER="-f '- /lost+found'
--owner --group --perms --links --specials \
--ignore-times --inplace --dirs --delete --quiet \
$WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \
- -f '- **' "$INNODB_DATA_HOME_DIR/" \
+ -f '- **' "$ib_home_dir/" \
"rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$?
if [ $RC -ne 0 ]; then
@@ -448,7 +550,7 @@ FILTER="-f '- /lost+found'
--owner --group --perms --links --specials \
--ignore-times --inplace --dirs --delete --quiet \
$WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \
- -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \
+ -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \
"rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$?
if [ $RC -ne 0 ]; then
@@ -459,7 +561,7 @@ FILTER="-f '- /lost+found'
# then, we parallelize the transfer of database directories,
# use '.' so that path concatenation works:
- cd "$WSREP_SST_OPT_DATA"
+ cd "$DATA"
backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads')
if [ -z "$backup_threads" ]; then
@@ -518,68 +620,21 @@ FILTER="-f '- /lost+found'
[ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
fi
+ [ -f "$SST_PID" ] && rm -f "$SST_PID"
+
+ wsrep_log_info "rsync SST/IST completed on donor"
+
elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
then
check_sockets_utils
- SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid"
-
- # give some time for previous SST to complete:
- check_round=0
- while check_pid "$SST_PID" 0 'wsrep_sst_'; do
- wsrep_log_info "previous SST is not completed, waiting for it to exit"
- check_round=$(( check_round + 1 ))
- if [ $check_round -eq 10 ]; then
- wsrep_log_error "previous SST script still running."
- exit 114 # EALREADY
- fi
- sleep 1
- done
-
- echo $$ > "$SST_PID"
-
- # give some time for stunnel from the previous SST to complete:
- check_round=0
- while check_pid "$STUNNEL_PID" 1; do
- wsrep_log_info "Lingering stunnel daemon found at startup," \
- "waiting for it to exit"
- check_round=$(( check_round + 1 ))
- if [ $check_round -eq 10 ]; then
- wsrep_log_error "stunnel daemon already running."
- exit 114 # EALREADY
- fi
- sleep 1
- done
-
- MODULE="rsync_sst"
- RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
- RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
-
- # give some time for rsync from the previous SST to complete:
- check_round=0
- while check_pid "$RSYNC_PID" 1; do
- wsrep_log_info "Lingering rsync daemon found at startup," \
- "waiting for it to exit"
- check_round=$(( check_round + 1 ))
- if [ $check_round -eq 10 ]; then
- wsrep_log_error "rsync daemon already running."
- exit 114 # EALREADY
- fi
- sleep 1
- done
-
- [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
- [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
-
- [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
-
ADDR="$WSREP_SST_OPT_ADDR"
RSYNC_PORT="$WSREP_SST_OPT_PORT"
RSYNC_ADDR="$WSREP_SST_OPT_HOST"
RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED"
- trap "exit 32" HUP PIPE
- trap "exit 3" INT TERM ABRT
+ trap 'exit 32' HUP PIPE
+ trap 'exit 3' INT TERM ABRT
trap cleanup_joiner EXIT
touch "$SST_PROGRESS_FILE"
@@ -600,13 +655,11 @@ $SILENT
path = $WSREP_SST_OPT_DATA
exclude = .zfs
[$MODULE-log_dir]
- path = $WSREP_LOG_DIR
+ path = $ib_log_dir
[$MODULE-data_dir]
- path = $INNODB_DATA_HOME_DIR
+ path = $ib_home_dir
EOF
-# rm -rf "$DATA/ib_logfile"* # we don't want old logs around
-
# If the IP is local, listen only on it:
if is_local_ip "$RSYNC_ADDR_UNESCAPED"
then
@@ -617,7 +670,7 @@ EOF
RSYNC_EXTRA_ARGS=""
STUNNEL_ACCEPT="$RSYNC_PORT"
# Overwrite address with all:
- RSYNC_ADDR="*"
+ RSYNC_ADDR='*'
fi
if [ -z "$STUNNEL" ]; then
@@ -675,11 +728,10 @@ EOF
TRANSFER_PID="$STUNNEL_PID"
fi
- if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]
- then # backward-incompatible behavior
+ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then
+ # backward-incompatible behavior
CN=""
- if [ -n "$SSTCERT" ]
- then
+ if [ -n "$SSTCERT" ]; then
# find out my Common Name
get_openssl
if [ -z "$OPENSSL_BINARY" ]; then
@@ -688,7 +740,7 @@ EOF
exit 42
fi
CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \
- tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
+ tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
sed s/\ %//)
fi
MY_SECRET="$(wsrep_gen_secret)"
@@ -725,16 +777,53 @@ EOF
exit 32
fi
- if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
- if [ -f "$BINLOG_TAR_FILE" ]; then
- cd "$BINLOG_DIRNAME"
+ if [ -r "$MAGIC_FILE" ]; then
+ if [ -n "$MY_SECRET" ]; then
+ # Check donor supplied secret:
+ SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
+ cut -d ' ' -f 2)
+ if [ "$SECRET" != "$MY_SECRET" ]; then
+ wsrep_log_error "Donor does not know my secret!"
+ wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
+ exit 32
+ fi
+ fi
+ else
+ # This message should cause joiner to abort:
+ wsrep_log_info "rsync process ended without creating magic file"
+ echo "rsync process ended without creating '$MAGIC_FILE'"
+ exit 32
+ fi
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ binlog_tar_present=0
+ [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1
+ # If it is SST (not an IST) or tar with binlogs is present
+ # among the transferred files, then we need to remove the
+ # old binlogs:
+ if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then
+ cd "$DATA"
+ # Clean up the old binlog files and index:
binlog_index="$WSREP_SST_OPT_BINLOG_INDEX"
-
- # Clean up old binlog files first
- rm -f "$BINLOG_FILENAME".[0-9]*
- [ -f "$binlog_index" ] && rm -f "$binlog_index"
-
+ if [ -f "$binlog_index" ]; then
+ while read bin_file || [ -n "$bin_file" ]; do
+ rm -f "$bin_file" || :
+ done < "$binlog_index"
+ rm -f "$binlog_index"
+ fi
+ binlog_cd=0
+ # Change the directory to binlog base (if possible):
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
+ -d "$binlog_dir" ]
+ then
+ binlog_cd=1
+ cd "$binlog_dir"
+ fi
+ # Clean up unindexed binlog files:
+ rm -f "$binlog_base".[0-9]* || :
+ [ $binlog_cd -ne 0 ] && cd "$DATA_DIR"
+ fi
+ if [ $binlog_tar_present -ne 0 ]; then
# Create a temporary file:
tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir')
if [ -z "$tmpdir" ]; then
@@ -744,46 +833,44 @@ EOF
else
tmpfile=$(TMPDIR="$tmpdir"; mktemp)
fi
-
+ index_dir=$(dirname "$binlog_index");
+ if [ -n "$index_dir" -a "$index_dir" != '.' ]; then
+ [ ! -d "$index_dir" ] && mkdir -p "$index_dir"
+ fi
+ binlog_cd=0
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then
+ [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir"
+ cd "$binlog_dir"
+ binlog_cd=1
+ fi
+ # Extracting binlog files:
wsrep_log_info "Extracting binlog files:"
if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then
- wsrep_log_error "Error unpacking tar file with binlog files"
rm -f "$tmpfile"
+ wsrep_log_error "Error unpacking tar file with binlog files"
exit 32
fi
-
# Rebuild binlog index:
- while read bin_file; do
- echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index"
+ [ $binlog_cd -ne 0 ] && cd "$DATA_DIR"
+ while read bin_file || [ -n "$bin_file" ]; do
+ echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index"
done < "$tmpfile"
rm -f "$tmpfile"
-
cd "$OLD_PWD"
fi
fi
- if [ -r "$MAGIC_FILE" ]; then
- if [ -n "$MY_SECRET" ]; then
- # check donor supplied secret
- SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \
- cut -d ' ' -f 2)
- if [ "$SECRET" != "$MY_SECRET" ]; then
- wsrep_log_error "Donor does not know my secret!"
- wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'"
- exit 32
- fi
- # remove secret from the magic file, and output
- # the UUID:seqno & wsrep_gtid_domain_id:
- grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE"
- else
- # Output the UUID:seqno and wsrep_gtid_domain_id:
- cat "$MAGIC_FILE"
- fi
+ if [ -n "$MY_SECRET" ]; then
+ # remove secret from the magic file, and output
+ # the UUID:seqno & wsrep_gtid_domain_id:
+ grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE"
else
- # this message should cause joiner to abort
- echo "rsync process ended without creating '$MAGIC_FILE'"
+ # Output the UUID:seqno and wsrep_gtid_domain_id:
+ cat "$MAGIC_FILE"
fi
+ wsrep_log_info "rsync SST/IST completed on joiner"
+
# wsrep_cleanup_progress_file
# cleanup_joiner
else
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index 569d5084c98..978c4805205 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -1,5 +1,8 @@
-#!/bin/bash -ue
-# Copyright (C) 2017-2021 MariaDB
+#!/usr/bin/env bash
+
+set -ue
+
+# Copyright (C) 2017-2022 MariaDB
# Copyright (C) 2013 Percona Inc
#
# This program is free software; you can redistribute it and/or modify
@@ -38,7 +41,7 @@ tcert=""
tcap=""
tpem=""
tkey=""
-tmode="DISABLED"
+tmode='DISABLED'
sockopt=""
progress=""
ttime=0
@@ -85,11 +88,11 @@ backup_threads=""
encrypt_threads=""
encrypt_chunk=""
-readonly SECRET_TAG="secret"
+readonly SECRET_TAG='secret'
sst_ver=-1
-if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then
+if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then
pvopts="$pvopts $pvformat"
fi
pcmd="pv $pvopts"
@@ -102,17 +105,14 @@ if [ -z "$BACKUP_BIN" ]; then
fi
DATA="$WSREP_SST_OPT_DATA"
-INFO_FILE="xtrabackup_galera_info"
-IST_FILE="xtrabackup_ist"
+INFO_FILE='xtrabackup_galera_info'
+IST_FILE='xtrabackup_ist'
MAGIC_FILE="$DATA/$INFO_FILE"
INNOAPPLYLOG="$DATA/innobackupex.prepare.log"
INNOMOVELOG="$DATA/innobackupex.move.log"
INNOBACKUPLOG="$DATA/innobackupex.backup.log"
-# Setting the path for ss and ip
-export PATH="/usr/sbin:/sbin:$PATH"
-
timeit()
{
local stage="$1"
@@ -152,7 +152,7 @@ get_keys()
return
fi
- if [ $sfmt = 'tar' ]; then
+ if [ "$sfmt" = 'tar' ]; then
wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \
"cannot be enabled with tar format"
encrypt=-1
@@ -182,11 +182,11 @@ get_keys()
exit 2
fi
ecmd="'$OPENSSL_BINARY' enc -$ealgo"
- if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then
+ if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then
ecmd="$ecmd -pbkdf2"
- elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then
ecmd="$ecmd -iter 1"
- elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then
ecmd="$ecmd -md sha256"
fi
if [ -z "$ekey" ]; then
@@ -231,15 +231,15 @@ get_keys()
get_transfer()
{
- if [ $tfmt = 'nc' ]; then
+ if [ "$tfmt" = 'nc' ]; then
wsrep_log_info "Using netcat as streamer"
wsrep_check_programs nc
- tcmd="nc"
+ tcmd='nc'
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
- if nc -h 2>&1 | grep -q 'ncat'; then
+ if nc -h 2>&1 | grep -q -F 'ncat'; then
wsrep_log_info "Using Ncat as streamer"
tcmd="$tcmd -l"
- elif nc -h 2>&1 | grep -qw -- '-d'; then
+ elif nc -h 2>&1 | grep -qw -F -- '-d'; then
wsrep_log_info "Using Debian netcat as streamer"
tcmd="$tcmd -dl"
if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then
@@ -261,14 +261,14 @@ get_transfer()
# transfer and cause the command to timeout.
# Older versions of netcat did not need this flag and will
# return an error if the flag is used.
- if nc -h 2>&1 | grep -qw -- '-N'; then
+ if nc -h 2>&1 | grep -qw -F -- '-N'; then
tcmd="$tcmd -N"
wsrep_log_info "Using nc -N"
fi
# netcat doesn't understand [] around IPv6 address
- if nc -h 2>&1 | grep -q ncat; then
+ if nc -h 2>&1 | grep -q -F 'ncat'; then
wsrep_log_info "Using Ncat as streamer"
- elif nc -h 2>&1 | grep -qw -- '-d'; then
+ elif nc -h 2>&1 | grep -qw -F -- '-d'; then
wsrep_log_info "Using Debian netcat as streamer"
else
wsrep_log_info "Using traditional netcat as streamer"
@@ -458,7 +458,7 @@ adjust_progress()
fi
elif [ -z "$progress" -a -n "$rlimit" ]; then
# When rlimit is non-zero
- pcmd="pv -q"
+ pcmd='pv -q'
fi
if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
@@ -592,8 +592,14 @@ get_stream()
sig_joiner_cleanup()
{
+ local estatus=$?
+ if [ $estatus -ne 0 ]; then
+ wsrep_log_error "Cleanup after exit with status: $estatus"
+ fi
wsrep_log_error "Removing $MAGIC_FILE file due to signal"
+ [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+ exit $estatus
}
cleanup_at_exit()
@@ -604,6 +610,8 @@ cleanup_at_exit()
wsrep_log_error "Cleanup after exit with status: $estatus"
fi
+ [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD"
+
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
wsrep_log_info "Removing the sst_in_progress file"
wsrep_cleanup_progress_file
@@ -633,7 +641,7 @@ cleanup_at_exit()
fi
# Final cleanup
- pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :)
+ pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :)
# This means no setsid done in mysqld.
# We don't want to kill mysqld here otherwise.
@@ -724,7 +732,7 @@ recv_joiner()
local ltcmd="$tcmd"
if [ $tmt -gt 0 ]; then
if [ -n "$(commandex timeout)" ]; then
- if timeout --help | grep -qw -- '-k'; then
+ if timeout --help | grep -qw -F -- '-k'; then
ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
else
ltcmd="timeout -s9 $tmt $tcmd"
@@ -824,9 +832,10 @@ monitor_process()
# check the version, we require XB-2.3.5 to ensure that we can pass the
# datadir via the command-line option
-XB_REQUIRED_VERSION="2.3.5"
+XB_REQUIRED_VERSION='2.3.5'
-XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
+XB_VERSION=$($BACKUP_BIN --version 2>&1 | \
+ grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1)
if [ -z "$XB_VERSION" ]; then
wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \
"Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \
@@ -843,7 +852,9 @@ fi
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
-if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
+if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \
+ "$WSREP_SST_OPT_ROLE" != 'donor' ]
+then
wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
exit 22
fi
@@ -851,25 +862,17 @@ fi
read_cnf
setup_ports
-if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then
disver=' --no-version-check'
fi
-# if no command line argument and INNODB_DATA_HOME_DIR environment variable
-# is not set, try to get it from my.cnf:
-if [ -z "$INNODB_DATA_HOME_DIR" ]; then
- INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
-fi
-
OLD_PWD="$(pwd)"
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_DATA_HOME_DIR" ]; then
- # handle both relative and absolute paths
- [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
- cd "$INNODB_DATA_HOME_DIR"
+if [ -n "$DATA" -a "$DATA" != '.' ]; then
+ [ ! -d "$DATA" ] && mkdir -p "$DATA"
+ cd "$DATA"
fi
-INNODB_DATA_HOME_DIR=$(pwd -P)
+DATA_DIR="$(pwd)"
cd "$OLD_PWD"
@@ -897,7 +900,7 @@ if [ $ssyslog -eq 1 ]; then
else
if [ $sstlogarchive -eq 1 ]
then
- ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
+ ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N')
if [ -n "$sstlogarchivedir" ]; then
if [ ! -d "$sstlogarchivedir" ]; then
@@ -953,7 +956,7 @@ setup_commands()
recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY"
fi
INNOAPPLY="$BACKUP_BIN$disver$recovery${iapts:+ }$iapts$INNOEXTRA --apply-log${rebuildcmd:+ }$rebuildcmd --datadir='$DATA' '$DATA' $INNOAPPLY"
- INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE"
+ INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --datadir='${TDATA:-$DATA}' '$DATA' $INNOMOVE"
local sfmt_work="$sfmt"
if [ "$sfmt" = 'mbstream' ]; then
sfmt_work='xbstream'
@@ -1124,6 +1127,7 @@ then
echo "done $WSREP_SST_OPT_GTID"
wsrep_log_info "Total time on donor: $totime seconds"
+ wsrep_log_info "xtrabackup SST/IST completed on donor"
elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
then
@@ -1131,22 +1135,53 @@ then
wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
[ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE"
- ib_home_dir="$INNODB_DATA_HOME_DIR"
+ # if no command line argument and INNODB_DATA_HOME_DIR environment
+ # variable is not set, try to get it from the my.cnf:
+ if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+ INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
+ INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR")
+ fi
+
+ if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+ cd "$INNODB_DATA_HOME_DIR"
+ ib_home_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
# if no command line argument and INNODB_LOG_GROUP_HOME is not set,
- # try to get it from my.cnf:
+ # then try to get it from the my.cnf:
if [ -z "$INNODB_LOG_GROUP_HOME" ]; then
INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir')
+ INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME")
fi
- ib_log_dir="$INNODB_LOG_GROUP_HOME"
+ if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME"
+ cd "$INNODB_LOG_GROUP_HOME"
+ ib_log_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
- # if no command line argument then try to get it from my.cnf:
+ # if no command line argument and INNODB_UNDO_DIR is not set,
+ # then try to get it from the my.cnf:
if [ -z "$INNODB_UNDO_DIR" ]; then
INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory')
+ INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR")
fi
- ib_undo_dir="$INNODB_UNDO_DIR"
+ if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then
+ # handle both relative and absolute paths:
+ cd "$DATA"
+ [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR"
+ cd "$INNODB_UNDO_DIR"
+ ib_undo_dir="$(pwd)"
+ cd "$OLD_PWD"
+ fi
if [ -n "$backup_threads" ]; then
impts="--parallel=$backup_threads${impts:+ }$impts"
@@ -1170,7 +1205,7 @@ then
stagemsg='Joiner-Recv'
- MODULE="xtrabackup_sst"
+ MODULE='xtrabackup_sst'
[ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
@@ -1193,7 +1228,7 @@ then
exit 42
fi
CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \
- tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
+ tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
sed s/\ %//)
fi
MY_SECRET="$(wsrep_gen_secret)"
@@ -1247,6 +1282,36 @@ then
jpid=$!
wsrep_log_info "Proceeding with SST"
+ get_binlog
+
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
+ binlog_base=$(basename "$WSREP_SST_OPT_BINLOG")
+ binlog_index="$WSREP_SST_OPT_BINLOG_INDEX"
+ cd "$DATA"
+ wsrep_log_info "Cleaning the old binary logs"
+ # If there is a file with binlogs state, delete it:
+ [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2
+ # Clean up the old binlog files and index:
+ if [ -f "$binlog_index" ]; then
+ while read bin_file || [ -n "$bin_file" ]; do
+ rm -fv "$bin_file" 1>&2 || :
+ done < "$binlog_index"
+ rm -fv "$binlog_index" 1>&2
+ fi
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \
+ -d "$binlog_dir" ]
+ then
+ cd "$binlog_dir"
+ if [ "$(pwd)" != "$DATA_DIR" ]; then
+ wsrep_log_info \
+ "Cleaning the binlog directory '$binlog_dir' as well"
+ fi
+ fi
+ rm -fv "$binlog_base".[0-9]* 1>&2 || :
+ cd "$OLD_PWD"
+ fi
+
wsrep_log_info \
"Cleaning the existing datadir and innodb-data/log directories"
if [ "$OS" = 'FreeBSD' ]; then
@@ -1263,20 +1328,6 @@ then
-o -exec rm -rfv {} 1>&2 \+
fi
- get_binlog
-
- if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
- binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG")
- if [ -d "$binlog_dir" ]; then
- cd "$binlog_dir"
- wsrep_log_info "Cleaning the binlog directory $binlog_dir as well"
- rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || :
- [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \
- rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+
- cd "$OLD_PWD"
- fi
- fi
-
TDATA="$DATA"
DATA="$DATA/.sst"
@@ -1312,11 +1363,13 @@ then
dcmd="xargs -n 2 qpress -dT$nproc"
- if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
+ if [ -n "$progress" ] && \
+ pv --help | grep -qw -F -- '--line-mode'
+ then
count=$(find "$DATA" -type f -name '*.qp' | wc -l)
count=$(( count*2 ))
pvopts="-f -s $count -l -N Decompression"
- if pv --help | grep -qw -- '-F'; then
+ if pv --help | grep -qw -F -- '-F'; then
pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
fi
pcmd="pv $pvopts"
@@ -1326,8 +1379,9 @@ then
# Decompress the qpress files
wsrep_log_info "Decompression with $nproc threads"
- timeit "Joiner-Decompression" \
- "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+ timeit 'Joiner-Decompression' \
+ "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \
+ $dcmd"
extcode=$?
if [ $extcode -eq 0 ]; then
@@ -1344,25 +1398,9 @@ then
fi
fi
- if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
-
- BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG")
- BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG")
-
- # To avoid comparing data directory and BINLOG_DIRNAME
- mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || :
-
- cd "$BINLOG_DIRNAME"
- for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do
- echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX"
- done
- cd "$OLD_PWD"
-
- fi
-
wsrep_log_info "Preparing the backup at $DATA"
setup_commands
- timeit "Xtrabackup prepare stage" "$INNOAPPLY"
+ timeit 'Xtrabackup prepare stage' "$INNOAPPLY"
if [ $? -ne 0 ]; then
wsrep_log_error "xtrabackup apply finished with errors." \
@@ -1370,10 +1408,43 @@ then
exit 22
fi
+ if [ -n "$WSREP_SST_OPT_BINLOG" ]; then
+ cd "$DATA"
+ binlogs=""
+ if [ -f 'xtrabackup_binlog_info' ]; then
+ NL=$'\n'
+ while read bin_string || [ -n "$bin_string" ]; do
+ bin_file=$(echo "$bin_string" | cut -f1)
+ if [ -f "$bin_file" ]; then
+ binlogs="$binlogs${binlogs:+$NL}$bin_file"
+ fi
+ done < 'xtrabackup_binlog_info'
+ else
+ binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :)
+ fi
+ cd "$DATA_DIR"
+ if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then
+ [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir"
+ fi
+ index_dir=$(dirname "$binlog_index");
+ if [ -n "$index_dir" -a "$index_dir" != '.' ]; then
+ [ ! -d "$index_dir" ] && mkdir -p "$index_dir"
+ fi
+ if [ -n "$binlogs" ]; then
+ wsrep_log_info "Moving binary logs to $binlog_dir"
+ echo "$binlogs" | \
+ while read bin_file || [ -n "$bin_file" ]; do
+ mv "$DATA/$bin_file" "$binlog_dir"
+ echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index"
+ done
+ fi
+ cd "$OLD_PWD"
+ fi
+
MAGIC_FILE="$TDATA/$INFO_FILE"
wsrep_log_info "Moving the backup to $TDATA"
- timeit "Xtrabackup move stage" "$INNOMOVE"
+ timeit 'Xtrabackup move stage' "$INNOMOVE"
if [ $? -eq 0 ]; then
wsrep_log_info "Move successful, removing $DATA"
rm -rf "$DATA"
@@ -1400,6 +1471,7 @@ then
cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id
wsrep_log_info "Total time on joiner: $totime seconds"
+ wsrep_log_info "xtrabackup SST/IST completed on joiner"
fi
exit 0