diff options
author | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2021-04-20 04:56:49 +0200 |
---|---|---|
committer | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2021-04-24 05:14:42 +0200 |
commit | 5be467bd219bce07a14a8814acd9524edadd35c8 (patch) | |
tree | 946454cf24c243acec4ba3bea520a1cc661e7735 | |
parent | d0baff638b7287e2838a4fa561f5c8b5601b97b0 (diff) | |
download | mariadb-git-bb-10.2-MDEV-24962-v3.tar.gz |
MDEV-24962: Galera SST innobackupex-move ignores Environment settingsbb-10.2-MDEV-24962-v3
After switching to the new mariabackup interface (instead of
the outdated innobackupex interface, which is supported for
compatibility), we need to explicitly pass a path to the datadir
directory as a parameter, since in the new interface the value
of this option is not automatically set in such a way that it
always matches the SST/IST logic. This commit adds passing this
option as an explicit parameter to mariabackup. This commit also
removed unnecessary options that are not used and not supported
by mariabackup.
-rw-r--r-- | mysql-test/suite/galera/r/galera_ist_innodb_flush_logs,debug.rdiff | 13 | ||||
-rw-r--r-- | mysql-test/suite/galera/r/galera_ist_xtrabackup-v2.result | 2 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf | 2 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test | 2 | ||||
-rw-r--r-- | scripts/wsrep_sst_common.sh | 207 | ||||
-rw-r--r-- | scripts/wsrep_sst_mariabackup.sh | 212 | ||||
-rw-r--r-- | scripts/wsrep_sst_rsync.sh | 59 | ||||
-rw-r--r-- | scripts/wsrep_sst_xtrabackup-v2.sh | 89 | ||||
-rw-r--r-- | scripts/wsrep_sst_xtrabackup.sh | 10 |
9 files changed, 327 insertions, 269 deletions
diff --git a/mysql-test/suite/galera/r/galera_ist_innodb_flush_logs,debug.rdiff b/mysql-test/suite/galera/r/galera_ist_innodb_flush_logs,debug.rdiff index fa1b67e7ef3..55f1150c930 100644 --- a/mysql-test/suite/galera/r/galera_ist_innodb_flush_logs,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_ist_innodb_flush_logs,debug.rdiff @@ -1,11 +1,12 @@ --- r/galera_ist_innodb_flush_logs.result 2018-09-05 10:34:36.192439933 +0300 +++ r/galera_ist_innodb_flush_logs.reject 2018-09-17 10:20:06.039150838 +0300 -@@ -86,3 +86,100 @@ +@@ -86,3 +86,111 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it ++connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; @@ -14,6 +15,7 @@ +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); ++connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); @@ -22,9 +24,12 @@ +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... ++connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); @@ -39,6 +44,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -46,7 +52,9 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); ++connection node_2; +Performing --wsrep-recover ... ++connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; @@ -57,6 +65,7 @@ +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; ++connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); @@ -71,6 +80,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -88,6 +98,7 @@ +1 +COMMIT; +SET AUTOCOMMIT=ON; ++connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 diff --git a/mysql-test/suite/galera/r/galera_ist_xtrabackup-v2.result b/mysql-test/suite/galera/r/galera_ist_xtrabackup-v2.result index 8a7c02ab1b6..80a28d349ba 100644 --- a/mysql-test/suite/galera/r/galera_ist_xtrabackup-v2.result +++ b/mysql-test/suite/galera/r/galera_ist_xtrabackup-v2.result @@ -21,7 +21,7 @@ INSERT INTO t1 VALUES ('node2_committed_before'); INSERT INTO t1 VALUES ('node2_committed_before'); COMMIT; Unloading wsrep provider ... -SET GLOBAL wsrep_provider = 'none'; +SET GLOBAL wsrep_cluster_address = ''; connection node_1; SET AUTOCOMMIT=OFF; START TRANSACTION; diff --git a/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf b/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf index b91c897e66e..a93c291b47c 100644 --- a/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf +++ b/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.cnf @@ -1,7 +1,7 @@ !include ../galera_2nodes.cnf [mysqld] -wsrep_sst_method=xtrabackup-v2 +wsrep_sst_method=mariabackup wsrep_sst_auth=root: innodb_safe_truncate=OFF diff --git a/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test b/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test index 839383ebfc1..fd362a26840 100644 --- a/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test +++ b/mysql-test/suite/galera/t/galera_ist_innodb_flush_logs.test @@ -7,7 +7,7 @@ --source include/big_test.inc --source include/galera_cluster.inc --source include/have_innodb.inc ---source include/have_xtrabackup.inc +--source include/have_mariabackup.inc --source suite/galera/include/galera_st_kill_slave.inc --source suite/galera/include/galera_st_kill_slave_ddl.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index 443cb26ab64..24806ac0184 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -30,15 +30,20 @@ WSREP_SST_OPT_EXTRA_DEFAULT="" WSREP_SST_OPT_SUFFIX_DEFAULT="" WSREP_SST_OPT_SUFFIX_VALUE="" WSREP_SST_OPT_MYSQLD="" +WSREP_SST_OPT_PORT="" +WSREP_SST_OPT_ADDR="" +WSREP_SST_OPT_ADDR_PORT="" +WSREP_SST_OPT_HOST="" +WSREP_SST_OPT_HOST_UNESCAPED="" +WSREP_SST_OPT_HOST_ESCAPED="" INNODB_DATA_HOME_DIR_ARG="" INNODB_LOG_GROUP_HOME_ARG="" INNODB_UNDO_DIR_ARG="" -LOG_BIN_ARG="" while [ $# -gt 0 ]; do case "$1" in '--address') - readonly WSREP_SST_OPT_ADDR="$2" + WSREP_SST_OPT_ADDR="$2" # # Break address string into host:port/path parts # @@ -47,12 +52,13 @@ case "$1" in # IPv6 # Remove the starting and ending square brackets, if present: addr_no_bracket=${WSREP_SST_OPT_ADDR#\[} + # Some utilities and subsequent code require an address + # without square brackets: readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} # Square brackets are needed in most cases: readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - # Some utilities and subsequent code require an address - # without square brackets: readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" + # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 ;; *) @@ -69,7 +75,7 @@ case "$1" in remain=${remain#:} # Extract the port number from the address - all characters # up to "/" (if present): - readonly WSREP_SST_OPT_ADDR_PORT=${remain%%/*} + WSREP_SST_OPT_ADDR_PORT=${remain%%/*} # If the "/" character is present, then the path is not empty: if [ "${remain#*/}" != "${remain}" ]; then # This operation removes everything up to the "/" character, @@ -136,10 +142,6 @@ case "$1" in readonly INNODB_UNDO_DIR_ARG="$2" shift ;; - '--log-bin') - readonly LOG_BIN_ARG="$2" - shift - ;; '--defaults-file') readonly WSREP_SST_OPT_DEFAULT="$1=$2" shift @@ -154,7 +156,28 @@ case "$1" in shift ;; '--host') - readonly WSREP_SST_OPT_HOST="$2" + case "$2" in + \[*) + # IPv6 + # Remove the starting and ending square brackets, if present: + addr_no_bracket=${2#\[} + # Some utilities and subsequent code require an address + # without square brackets: + readonly WSREP_SST_OPT_HOST_UNESCAPED=${addr_no_bracket%%\]*} + # Square brackets are needed in most cases: + readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" + readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" + # Mark this address as IPv6: + readonly WSREP_SST_OPT_HOST_IPv6=1 + ;; + *) + readonly WSREP_SST_OPT_HOST=$2 + readonly WSREP_SST_OPT_HOST_UNESCAPED=$2 + readonly WSREP_SST_OPT_HOST_ESCAPED=$2 + readonly WSREP_SST_OPT_HOST_IPv6=0 + ;; + esac + WSREP_SST_OPT_ADDR=$WSREP_SST_OPT_HOST shift ;; '--local-port') @@ -189,14 +212,14 @@ case "$1" in readonly WSREP_SST_OPT_GTID="$2" shift ;; - '--binlog') + '--binlog'|'--log-bin') WSREP_SST_OPT_BINLOG="$2" shift ;; - '--binlog-index') - WSREP_SST_OPT_BINLOG_INDEX="$2" - shift - ;; + '--binlog-index'|'--log-bin-index') + WSREP_SST_OPT_BINLOG_INDEX="$2" + shift + ;; '--gtid-domain-id') readonly WSREP_SST_OPT_GTID_DOMAIN_ID="$2" shift @@ -206,20 +229,22 @@ case "$1" in shift while [ $# -gt 0 ]; do option=${1%%=*} - if [[ "$option" != "--defaults-file" && \ - "$option" != "--defaults-extra-file" && \ - "$option" != "--defaults-group-suffix" && \ - "$option" != "--port" && \ - "$option" != "--socket" ]]; then + if [ "$option" != "--defaults-file" && \ + "$option" != "--defaults-extra-file" && \ + "$option" != "--defaults-group-suffix" && \ + "$option" != "--port" && \ + "$option" != "--socket" ]; then value=${1#*=} - if [ "$value" == "$1" ]; then + if [ "$value" = "$1" ]; then value="" fi + skip_mysqld_arg=0 case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR_ARG" ]; then readonly INNODB_DATA_HOME_DIR_ARG="$value" fi + skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME_ARG" ]; then @@ -232,15 +257,34 @@ case "$1" in fi ;; '--log-bin') - if [ -z "$LOG_BIN_ARG" ]; then - readonly LOG_BIN_ARG="$value" + if [ -n "$value" ]; then + if [ -z "$WSREP_SST_OPT_BINLOG" ]; then + readonly WSREP_SST_OPT_BINLOG="$value" + fi + skip_mysqld_arg=1 + fi + ;; + '--log-bin-index') + if [ -n "$value" ]; then + if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + readonly WSREP_SST_OPT_BINLOG_INDEX="$value" + fi + skip_mysqld_arg=1 fi ;; + '--datadir'|'-h') + if [ -z "$WSREP_SST_OPT_DATA" ]; then + readonly WSREP_SST_OPT_DATA="$value" + fi + skip_mysqld_arg=1 + ;; esac - if [ -z "$original_cmd" ]; then - original_cmd="$1" - else - original_cmd="$original_cmd $1" + if [ $skip_mysqld_arg -eq 0 ]; then + if [ -z "$original_cmd" ]; then + original_cmd="$1" + else + original_cmd="$original_cmd $1" + fi fi fi shift @@ -259,17 +303,39 @@ readonly WSREP_SST_OPT_BYPASS readonly WSREP_SST_OPT_BINLOG readonly WSREP_SST_OPT_BINLOG_INDEX -if [ -n "${WSREP_SST_OPT_ADDR_PORT:-}" ]; then - if [ -n "${WSREP_SST_OPT_PORT:-}" ]; then - if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then - echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 - exit 2 +if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then + if [ -n "$WSREP_SST_OPT_PORT" ]; then + if [ "$WSREP_SST_OPT_PORT" != "$WSREP_SST_OPT_ADDR_PORT" ]; then + echo "WSREP_SST: [ERROR] port in --port=$WSREP_SST_OPT_PORT differs from port in --address=$WSREP_SST_OPT_ADDR" >&2 + exit 2 + fi + else + readonly WSREP_SST_OPT_PORT=$WSREP_SST_OPT_ADDR_PORT + fi +elif [ -n "$WSREP_SST_OPT_ADDR" ]; then + # If the port is missing, take the default port: + if [ -z "$WSREP_SST_OPT_PORT" ]; then + readonly WSREP_SST_OPT_PORT=4444 + fi + WSREP_SST_OPT_ADDR_PORT=$WSREP_SST_OPT_PORT + # Let's remove the leading part that contains the host address: + remain=${WSREP_SST_OPT_ADDR#${WSREP_SST_OPT_HOST_ESCAPED}} + # Let's remove the ":" character that separates the port number + # from the hostname: + remain=${remain#:} + # Let's remove all characters upto first "/" character that + # separates the hostname with port number from the path: + remain=${remain#/} + # Let's construct a new value for the address with the port: + WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT" + if [ -n "$remain" ]; then + WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_ADDR/$remain" fi - else - readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" - fi fi +readonly WSREP_SST_OPT_ADDR +readonly WSREP_SST_OPT_ADDR_PORT + # try to use my_print_defaults, mysql and mysqldump that come with the sources # (for MTR suite) SCRIPTS_DIR="$(cd $(dirname "$0"); pwd -P)" @@ -279,13 +345,13 @@ CLIENT_DIR="$SCRIPTS_DIR/../client" if [ -x "$CLIENT_DIR/mysql" ]; then MYSQL_CLIENT="$CLIENT_DIR/mysql" else - MYSQL_CLIENT=$(which mysql) + MYSQL_CLIENT="$(command -v mysql)" fi if [ -x "$CLIENT_DIR/mysqldump" ]; then MYSQLDUMP="$CLIENT_DIR/mysqldump" else - MYSQLDUMP=$(which mysqldump) + MYSQLDUMP="$(command -v mysqldump)" fi if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then @@ -293,7 +359,7 @@ if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults" else - MY_PRINT_DEFAULTS=$(which my_print_defaults) + MY_PRINT_DEFAULTS="$(command -v my_print_defaults)" fi wsrep_defaults="$WSREP_SST_OPT_DEFAULT" @@ -307,16 +373,41 @@ fi readonly WSREP_SST_OPT_CONF="$wsrep_defaults$WSREP_SST_OPT_SUFFIX_DEFAULT" readonly MY_PRINT_DEFAULTS="$MY_PRINT_DEFAULTS $WSREP_SST_OPT_CONF" +# +# user can specify mariabackup specific settings that will be used during sst +# process like encryption, etc..... +# parse such configuration option. (group for xb settings is [sst] in my.cnf +# +# 1st param: group (config file section like sst) or my_print_defaults argument (like --mysqld) +# 2nd param: var : name of the variable in the section, e.g. server-id +# 3rd param: - : default value for the param +parse_cnf() +{ + local group=$1 + local var=$2 + local reval="" + + # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin) + # then search for needed variable + # finally get the variable value (if variables has been specified multiple time use the last value only) + + reval=$($MY_PRINT_DEFAULTS "$group" | awk 'BEGIN {OFS=FS="="} {gsub(/_/,"-",$1); if ($1=="--'"$var"'") lastval=substr($0,length($1)+2)} END {print lastval}') + + # use default if we haven't found a value + if [ -z "$reval" ]; then + [ -n "${3:-}" ] && reval=$3 + fi + echo $reval +} + wsrep_auth_not_set() { [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ] } # State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727. -if $MY_PRINT_DEFAULTS sst | grep -q "wsrep_sst_auth"; then - if wsrep_auth_not_set; then - WSREP_SST_OPT_AUTH=$($MY_PRINT_DEFAULTS sst | grep -- "--wsrep_sst_auth" | cut -d= -f2) - fi +if wsrep_auth_not_set; then + WSREP_SST_OPT_AUTH=$(parse_cnf sst "wsrep-sst-auth") fi readonly WSREP_SST_OPT_AUTH @@ -329,7 +420,7 @@ fi readonly WSREP_SST_OPT_USER readonly WSREP_SST_OPT_PSWD -if [ -n "${WSREP_SST_OPT_DATA:-}" ] +if [ -n "$WSREP_SST_OPT_DATA" ] then SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress" else @@ -368,7 +459,7 @@ wsrep_check_program() { local prog=$1 - if ! which $prog >/dev/null + if ! [ -x "$(command -v $prog)" ]; then echo "'$prog' not found in PATH" return 2 # no such file or directory @@ -388,29 +479,11 @@ wsrep_check_programs() return $ret } -# -# user can specify mariabackup specific settings that will be used during sst -# process like encryption, etc..... -# parse such configuration option. (group for xb settings is [sst] in my.cnf -# -# 1st param: group (config file section like sst) or my_print_defaults argument (like --mysqld) -# 2nd param: var : name of the variable in the section, e.g. server-id -# 3rd param: - : default value for the param -parse_cnf() +wsrep_check_datadir() { - local group=$1 - local var=$2 - local reval="" - - # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin) - # then search for needed variable - # finally get the variable value (if variables has been specified multiple time use the last value only) - - reval=$($MY_PRINT_DEFAULTS "${group}" | awk -v var="${var}" 'BEGIN { OFS=FS="=" } { gsub(/_/,"-",$1); if ( $1=="--"var) lastval=substr($0,length($1)+2) } END { print lastval}') - - # use default if we haven't found a value - if [ -z "$reval" ]; then - [ -n "$3" ] && reval=$3 + if [ -z "$WSREP_SST_OPT_DATA" ] + then + wsrep_log_error "The '--datadir' parameter must be passed to the SST script" + exit 2 fi - echo $reval } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 0499b6c0678..c9c71fc3cf1 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -21,6 +21,7 @@ # Make sure to read that before proceeding! . $(dirname $0)/wsrep_sst_common +wsrep_check_datadir OS=$(uname) ealgo="" @@ -56,8 +57,6 @@ sfmt="tar" strmcmd="" tfmt="" tcmd="" -rebuild=0 -rebuildcmd="" payload=0 pvformat="-F '%N => Rate:%r Avg:%a Elapsed:%t %e Bytes: %b %p' " pvopts="-f -i 10 -N $WSREP_SST_OPT_ROLE " @@ -84,8 +83,8 @@ pcmd="pv $pvopts" declare -a RC set +e -INNOBACKUPEX_BIN=$(which mariabackup) -if test -z $INNOBACKUPEX_BIN +MARIABACKUP_BIN="$(command -v mariabackup)" +if test -z $MARIABACKUP_BIN then wsrep_log_error 'mariabackup binary not found in $PATH' exit 42 @@ -136,7 +135,7 @@ get_keys() fi if [[ $encrypt -eq 0 ]];then - if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then + if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--encrypt";then wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html" fi return @@ -175,17 +174,13 @@ get_keys() get_transfer() { - if [[ -z $SST_PORT ]];then - TSST_PORT=4444 - else - TSST_PORT=$SST_PORT - fi + TSST_PORT=$SST_PORT if [[ $tfmt == 'nc' ]];then wsrep_check_programs nc wsrep_log_info "Using netcat as streamer" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then if nc -h 2>&1 | grep -q ncat;then # Ncat tcmd="nc -l ${TSST_PORT}" @@ -213,7 +208,7 @@ get_transfer() # return an error if the flag is used. # tcmd_extra="" - if nc -h 2>&1 | grep -qw -- -N; then + if nc -h 2>&1 | grep -qw -- -N;then tcmd_extra+="-N" wsrep_log_info "Using nc -N" fi @@ -249,7 +244,7 @@ get_transfer() exit 22 fi stagemsg+="-OpenSSL-Encrypted-2" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then wsrep_log_info "Decrypting with cert=${tpem}, cafile=${tcert}" tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},cafile=${tcert}${sockopt} stdio" else @@ -263,7 +258,7 @@ get_transfer() exit 22 fi stagemsg+="-OpenSSL-Encrypted-3" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then if [[ -z $tcert ]];then wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0" tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},verify=0${sockopt} stdio" @@ -281,7 +276,7 @@ get_transfer() fi fi else - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio" else tcmd="socat -u stdio TCP:${REMOTEIP}:${TSST_PORT}${sockopt}" @@ -290,21 +285,6 @@ get_transfer() fi } -parse_cnf() -{ - local group=$1 - local var=$2 - # print the default settings for given group using my_print_default. - # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin) - # then grep for needed variable - # finally get the variable value (if variables has been specified multiple time use the last value only) - reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1) - if [[ -z $reval ]];then - [[ -n $3 ]] && reval=$3 - fi - echo $reval -} - get_footprint() { pushd $WSREP_SST_OPT_DATA 1>/dev/null @@ -341,7 +321,7 @@ adjust_progress() pcmd="pv -q" fi - if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then + if [[ -n $rlimit && "$WSREP_SST_OPT_ROLE" == "donor" ]];then wsrep_log_info "Rate-limiting SST to $rlimit" pcmd+=" -L \$rlimit" fi @@ -351,29 +331,35 @@ read_cnf() { sfmt=$(parse_cnf sst streamfmt "xbstream") tfmt=$(parse_cnf sst transferfmt "socat") - tcert=$(parse_cnf sst tca "") - tpem=$(parse_cnf sst tcert "") - tkey=$(parse_cnf sst tkey "") + encrypt=$(parse_cnf sst encrypt 0) + if [ $encrypt -ge 2 ]; then + tcert=$(parse_cnf sst tca "") + tpem=$(parse_cnf sst tcert "") + if [ $encrypt -ge 3 ]; then + tkey=$(parse_cnf sst tkey "") + fi + elif [ $encrypt -ne -1 ]; then + # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html + ealgo=$(parse_cnf xtrabackup encrypt "") + if [ -z "$ealgo" ]; then + ealgo=$(parse_cnf sst encrypt-algo "") + ekey=$(parse_cnf sst encrypt-key "") + ekeyfile=$(parse_cnf sst encrypt-key-file "") + else + ekey=$(parse_cnf xtrabackup encrypt-key "") + ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") + fi + fi + sockopt=$(parse_cnf sst sockopt "") progress=$(parse_cnf sst progress "") - rebuild=$(parse_cnf sst rebuild 0) ttime=$(parse_cnf sst time 0) cpat=$(parse_cnf sst cpat '.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$') [[ $OS == "FreeBSD" ]] && cpat=$(parse_cnf sst cpat '.*galera\.cache$|.*sst_in_progress$|.*\.sst$|.*gvwstate\.dat$|.*grastate\.dat$|.*\.err$|.*\.log$|.*RPM_UPGRADE_MARKER$|.*RPM_UPGRADE_HISTORY$') - ealgo=$(parse_cnf xtrabackup encrypt "") - ekey=$(parse_cnf xtrabackup encrypt-key "") - ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "") scomp=$(parse_cnf sst compressor "") sdecomp=$(parse_cnf sst decompressor "") - # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html - if [[ -z $ealgo ]];then - ealgo=$(parse_cnf sst encrypt-algo "") - ekey=$(parse_cnf sst encrypt-key "") - ekeyfile=$(parse_cnf sst encrypt-key-file "") - fi - rlimit=$(parse_cnf sst rlimit "") uextra=$(parse_cnf sst use-extra 0) speciald=$(parse_cnf sst sst-special-dirs 1) @@ -393,7 +379,7 @@ read_cnf() fi if [[ $ssyslog -ne -1 ]];then - if $MY_PRINT_DEFAULTS mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then + if $MY_PRINT_DEFAULTS mysqld_safe | grep -q -- "--syslog";then ssyslog=1 fi fi @@ -409,7 +395,7 @@ get_stream() { if [[ $sfmt == 'mbstream' || $sfmt == 'xbstream' ]];then wsrep_log_info "Streaming with ${sfmt}" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then strmcmd="${XBSTREAM_BIN} -x" else strmcmd="${XBSTREAM_BIN} -c \${INFO_FILE}" @@ -417,7 +403,7 @@ get_stream() else sfmt="tar" wsrep_log_info "Streaming with tar" - if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then + if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then strmcmd="tar xfi - " else strmcmd="tar cf - \${INFO_FILE} " @@ -539,8 +525,8 @@ kill_xtrabackup() setup_ports() { - SST_PORT=${WSREP_SST_OPT_ADDR_PORT} - if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then + SST_PORT=$WSREP_SST_OPT_PORT + if [[ "$WSREP_SST_OPT_ROLE" == "donor" ]];then REMOTEIP=${WSREP_SST_OPT_HOST} lsn=${WSREP_SST_OPT_LSN} sst_ver=${WSREP_SST_OPT_SST_VER} @@ -570,8 +556,9 @@ check_extra() { local use_socket=1 if [[ $uextra -eq 1 ]];then - if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then - local eport=$($MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2) + local thread_handling=$(parse_cnf --mysqld thread-handling "") + if [ $thread_handling -eq "pool-of-threads" ];then + local eport=$(parse_cnf --mysqld extra-port "") if [[ -n $eport ]];then # Xtrabackup works only locally. # Hence, setting host to 127.0.0.1 unconditionally. @@ -680,7 +667,7 @@ monitor_process() done } -wsrep_check_programs "$INNOBACKUPEX_BIN" +wsrep_check_programs "$MARIABACKUP_BIN" rm -f "${MAGIC_FILE}" @@ -692,7 +679,7 @@ fi read_cnf setup_ports -if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -q -- '--version-check'; then +if ${MARIABACKUP_BIN} --help 2>/dev/null | grep -q -- '--version-check'; then disver="--no-version-check" fi @@ -703,11 +690,11 @@ if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then iopts+=" --no-backup-locks" fi -INNOEXTRA=$WSREP_SST_OPT_MYSQLD +INNOEXTRA= INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} # Try to set INNODB_DATA_HOME_DIR from the command line: -if [ ! -z "$INNODB_DATA_HOME_DIR_ARG" ]; then +if [ -n "$INNODB_DATA_HOME_DIR_ARG" ]; then INNODB_DATA_HOME_DIR=$INNODB_DATA_HOME_DIR_ARG fi # if no command line arg and INNODB_DATA_HOME_DIR environment variable @@ -718,8 +705,8 @@ fi if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '') fi -if [ ! -z "$INNODB_DATA_HOME_DIR" ]; then - INNOEXTRA+=" --innodb-data-home-dir=$INNODB_DATA_HOME_DIR" +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + INNOEXTRA+=" --innodb-data-home-dir=$INNODB_DATA_HOME_DIR" fi if [ -n "$INNODB_DATA_HOME_DIR" ]; then @@ -735,7 +722,7 @@ if [[ $ssyslog -eq 1 ]];then if ! command -v logger >/dev/null;then wsrep_log_error "logger not in path: $PATH. Ignoring" else - wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog" + wsrep_log_info "Logging all stderr of SST/mariabackup to syslog" exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE) @@ -748,12 +735,12 @@ if [[ $ssyslog -eq 1 ]];then { logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" } - - INNOAPPLY="${INNOBACKUPEX_BIN} --prepare $disver $iapts \$INNOEXTRA $rebuildcmd --target-dir=\${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply" - INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --move-back $disver $impts --force-non-empty-directories --target-dir=\${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move" - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --backup $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt --target-dir=\$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" fi + INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply" + INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move" + INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)" + else if [[ "$sstlogarchive" -eq 1 ]] @@ -790,7 +777,6 @@ then else newfile=${INNOMOVELOG}.${ARCHIVETIMESTAMP} fi - wsrep_log_info "Moving ${INNOMOVELOG} to ${newfile}" mv "${INNOMOVELOG}" "${newfile}" gzip "${newfile}" @@ -804,19 +790,22 @@ then else newfile=${INNOBACKUPLOG}.${ARCHIVETIMESTAMP} fi - wsrep_log_info "Moving ${INNOBACKUPLOG} to ${newfile}" mv "${INNOBACKUPLOG}" "${newfile}" gzip "${newfile}" fi fi - INNOAPPLY="${INNOBACKUPEX_BIN} --prepare $disver $iapts \$INNOEXTRA $rebuildcmd --target-dir=\${DATA} &> ${INNOAPPLYLOG}" - INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --move-back $disver $impts --force-non-empty-directories --target-dir=\${DATA} &> ${INNOMOVELOG}" - INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} --backup $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt --target-dir=\$itmpdir 2> ${INNOBACKUPLOG}" + INNOAPPLY="&> ${INNOAPPLYLOG}" + INNOMOVE="&> ${INNOMOVELOG}" + INNOBACKUP="2> ${INNOBACKUPLOG}" fi +INNOAPPLY="${MARIABACKUP_BIN} --prepare $disver $iapts \$INNOEXTRA --target-dir=\${DATA} --datadir=\${DATA} --mysqld-args \$WSREP_SST_OPT_MYSQLD $INNOAPPLY" +INNOMOVE="${MARIABACKUP_BIN} ${WSREP_SST_OPT_CONF} --move-back $disver $impts --force-non-empty-directories --target-dir=\${DATA} --datadir=\${TDATA} $INNOMOVE" +INNOBACKUP="${MARIABACKUP_BIN} ${WSREP_SST_OPT_CONF} --backup $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt --target-dir=\$itmpdir --datadir=\${DATA} --mysqld-args \$WSREP_SST_OPT_MYSQLD $INNOBACKUP" + get_stream get_transfer @@ -842,7 +831,7 @@ then fi itmpdir=$(mktemp -d) - wsrep_log_info "Using $itmpdir as innobackupex temporary directory" + wsrep_log_info "Using $itmpdir as mariabackup temporary directory" if [[ -n "${WSREP_SST_OPT_USER:-}" && "$WSREP_SST_OPT_USER" != "(null)" ]]; then INNOEXTRA+=" --user=$WSREP_SST_OPT_USER" @@ -856,15 +845,6 @@ then unset MYSQL_PWD fi - get_keys - if [[ $encrypt -eq 1 ]];then - if [[ -n $ekey ]];then - INNOEXTRA+=" --encrypt=$ealgo --encrypt-key=$ekey" - else - INNOEXTRA+=" --encrypt=$ealgo --encrypt-key-file=$ekeyfile" - fi - fi - check_extra wsrep_log_info "Streaming GTID file before SST" @@ -875,19 +855,19 @@ then ttcmd="$tcmd" + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + + get_keys if [[ $encrypt -eq 1 ]];then - if [[ -n $scomp ]];then - tcmd=" $ecmd | $scomp | $tcmd " - else - tcmd=" $ecmd | $tcmd " - fi - elif [[ -n $scomp ]];then - tcmd=" $scomp | $tcmd " + tcmd="$ecmd | $tcmd" fi send_donor $DATA "${stagemsg}-gtid" tcmd="$ttcmd" + if [[ -n $progress ]];then get_footprint tcmd="$pcmd | $tcmd" @@ -899,7 +879,7 @@ then wsrep_log_info "Sleeping before data transfer for SST" sleep 10 - wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP} ${SST_PORT:-4444}" + wsrep_log_info "Streaming the backup to joiner at ${REMOTEIP}:${SST_PORT}" if [[ -n $scomp ]];then tcmd="$scomp | $tcmd" @@ -910,7 +890,7 @@ then set -e if [ ${RC[0]} -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \ + wsrep_log_error "${MARIABACKUP_BIN} finished with error: ${RC[0]}. " \ "Check syslog or ${INNOBACKUPLOG} for details" exit 22 elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then @@ -918,7 +898,7 @@ then exit 22 fi - # innobackupex implicitly writes PID to fixed location in $xtmpdir + # mariabackup implicitly writes PID to fixed location in $xtmpdir XTRABACKUP_PID="$xtmpdir/xtrabackup_pid" else # BYPASS FOR IST @@ -930,16 +910,16 @@ then # (separated by a space). echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}" echo "1" > "${DATA}/${IST_FILE}" + + if [[ -n $scomp ]];then + tcmd="$scomp | $tcmd" + fi + get_keys if [[ $encrypt -eq 1 ]];then - if [[ -n $scomp ]];then - tcmd=" $ecmd | $scomp | $tcmd " - else - tcmd=" $ecmd | $tcmd " - fi - elif [[ -n $scomp ]];then - tcmd=" $scomp | $tcmd " + tcmd="$ecmd | $tcmd" fi + strmcmd+=" \${IST_FILE}" send_donor $DATA "${stagemsg}-IST" @@ -958,7 +938,7 @@ then WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} # Try to set WSREP_LOG_DIR from the command line: - if [ ! -z "$INNODB_LOG_GROUP_HOME_ARG" ]; then + if [ -n "$INNODB_LOG_GROUP_HOME_ARG" ]; then WSREP_LOG_DIR=$INNODB_LOG_GROUP_HOME_ARG fi # if no command line arg and WSREP_LOG_DIR is not set, @@ -973,7 +953,7 @@ then ib_log_dir=$WSREP_LOG_DIR # Try to set ib_undo_dir from the command line: - ib_undo_dir=${INNODB_UNDO_DIR_ARG:-""} + ib_undo_dir=$INNODB_UNDO_DIR_ARG # if no command line arg then try to get it from my.cnf: if [ -z "$ib_undo_dir" ]; then ib_undo_dir=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-undo-directory "") @@ -992,18 +972,9 @@ then rm -f "${DATA}/${IST_FILE}" # May need xtrabackup_checkpoints later on - rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/ib_logfile0 + rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/ib_logfile0 - ADDR=${WSREP_SST_OPT_ADDR} - if [ -z "${SST_PORT}" ] - then - SST_PORT=4444 - if [ "${ADDR#\[}" != "$ADDR" ]; then - ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F '\\]:' '{ print $1 }')]:${SST_PORT}" - else - ADDR="$(echo ${WSREP_SST_OPT_ADDR} | awk -F ':' '{ print $1 }'):${SST_PORT}" - fi - fi + ADDR=$WSREP_SST_OPT_ADDR wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} & @@ -1055,17 +1026,17 @@ then find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ fi - tempdir=$LOG_BIN_ARG + tempdir=$WSREP_SST_OPT_BINLOG if [ -z "$tempdir" ]; then tempdir=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE log-bin "") fi if [ -z "$tempdir" ]; then tempdir=$(parse_cnf --mysqld log-bin "") fi - if [[ -n ${tempdir:-} ]];then + if [ -n "$tempdir" ]; then binlog_dir=$(dirname $tempdir) binlog_file=$(basename $tempdir) - if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + if [[ -n "${binlog_dir:-}" && "$binlog_dir" != '.' && "$binlog_dir" != "$DATA" ]]; then pattern="$binlog_dir/$binlog_file\.[0-9]+$" wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true @@ -1083,20 +1054,15 @@ then get_proc if [[ ! -s ${DATA}/xtrabackup_checkpoints ]];then - wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor" + wsrep_log_error "xtrabackup_checkpoints missing, failed mariabackup/SST on donor" exit 2 fi - # Rebuild indexes for compact backups + # Compact backups are not supported by mariabackup if grep -q 'compact = 1' ${DATA}/xtrabackup_checkpoints;then wsrep_log_info "Index compaction detected" - rebuild=1 - fi - - if [[ $rebuild -eq 1 ]];then - nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc) - wsrep_log_info "Rebuilding during prepare with $nthreads threads" - rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads" + wsrel_log_error "Compact backups are not supported by mariabackup" + exit 2 fi if test -n "$(find ${DATA} -maxdepth 1 -type f -name '*.qp' -print -quit)";then @@ -1140,7 +1106,7 @@ then fi fi - if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) @@ -1161,13 +1127,11 @@ then if [ $? -ne 0 ]; then - wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check syslog or ${INNOAPPLYLOG} for details" + wsrep_log_error "${MARIABACKUP_BIN} apply finished with errors. Check syslog or ${INNOAPPLYLOG} for details" exit 22 fi MAGIC_FILE="${TDATA}/${INFO_FILE}" - set +e - set -e wsrep_log_info "Moving the backup to ${TDATA}" timeit "Xtrabackup move stage" "$INNOMOVE" if [[ $? -eq 0 ]];then diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 7e3f7b73301..b160e4623b6 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -29,6 +29,7 @@ OS=$(uname) export PATH="/usr/sbin:/sbin:$PATH" . $(dirname $0)/wsrep_sst_common +wsrep_check_datadir wsrep_check_programs rsync @@ -134,23 +135,21 @@ BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" BINLOG_N_FILES=1 rm -f "$BINLOG_TAR_FILE" || : -if ! [ -z $WSREP_SST_OPT_BINLOG ] -then +if [ -n "$WSREP_SST_OPT_BINLOG" ]; then BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) BINLOG_INDEX_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) BINLOG_INDEX_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) fi -if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] -then +if [ -n "$WSREP_SST_OPT_BINLOG_INDEX" ]; then BINLOG_INDEX_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG_INDEX) BINLOG_INDEX_FILENAME=$(basename $WSREP_SST_OPT_BINLOG_INDEX) fi WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} # Try to set WSREP_LOG_DIR from the command line: -if [ ! -z "$INNODB_LOG_GROUP_HOME_ARG" ]; then +if [ -n "$INNODB_LOG_GROUP_HOME_ARG" ]; then WSREP_LOG_DIR=$INNODB_LOG_GROUP_HOME_ARG fi # if no command line arg and WSREP_LOG_DIR is not set, @@ -172,7 +171,7 @@ fi INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} # Try to set INNODB_DATA_HOME_DIR from the command line: -if [ ! -z "$INNODB_DATA_HOME_DIR_ARG" ]; then +if [ -n "$INNODB_DATA_HOME_DIR_ARG" ]; then INNODB_DATA_HOME_DIR=$INNODB_DATA_HOME_DIR_ARG fi # if no command line arg and INNODB_DATA_HOME_DIR environment variable @@ -271,17 +270,17 @@ EOF sync - if ! [ -z $WSREP_SST_OPT_BINLOG ] - then + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then # Prepare binlog files OLD_PWD="$(pwd)" cd $BINLOG_DIRNAME - if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] - binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ] then - cd $BINLOG_INDEX_DIRNAME - binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_INDEX_FILENAME}.index) + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_FILENAME}.index) + else + cd $BINLOG_INDEX_DIRNAME + binlog_files_full=$(tail -n $BINLOG_N_FILES ${BINLOG_INDEX_FILENAME}.index) fi cd $BINLOG_DIRNAME @@ -291,11 +290,11 @@ EOF binlog_files="$binlog_files $(basename $ii)" done - if ! [ -z "$binlog_files" ] - then + if [ -n "$binlog_files" ]; then wsrep_log_info "Preparing binlog files for transfer:" tar -cvf $BINLOG_TAR_FILE $binlog_files >&2 fi + cd "$OLD_PWD" fi @@ -415,18 +414,8 @@ then rm -rf "$RSYNC_PID" ADDR=$WSREP_SST_OPT_ADDR - if [ "${ADDR#\[}" != "$ADDR" ]; then - RSYNC_PORT=$(echo $ADDR | awk -F '\\]:' '{ print $2 }') - RSYNC_ADDR=$(echo $ADDR | awk -F '\\]:' '{ print $1 }')"]" - else - RSYNC_PORT=$(echo $ADDR | awk -F ':' '{ print $2 }') - RSYNC_ADDR=$(echo $ADDR | awk -F ':' '{ print $1 }') - fi - if [ -z "$RSYNC_PORT" ] - then - RSYNC_PORT=4444 - ADDR="$RSYNC_ADDR:$RSYNC_PORT" - fi + RSYNC_PORT=$WSREP_SST_OPT_PORT + RSYNC_ADDR=$WSREP_SST_OPT_HOST trap "exit 32" HUP PIPE trap "exit 3" INT TERM ABRT @@ -457,7 +446,6 @@ EOF # rm -rf "$DATA"/ib_logfile* # we don't want old logs around - readonly RSYNC_PORT=${WSREP_SST_OPT_PORT:-4444} # If the IP is local listen only in it if is_local_ip "$RSYNC_ADDR" then @@ -516,30 +504,31 @@ EOF exit 32 fi - if ! [ -z $WSREP_SST_OPT_BINLOG ] - then + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then OLD_PWD="$(pwd)" cd $BINLOG_DIRNAME - if [ -f $BINLOG_TAR_FILE ] - then + if [ -f "$BINLOG_TAR_FILE" ]; then # Clean up old binlog files first rm -f ${BINLOG_FILENAME}.* wsrep_log_info "Extracting binlog files:" tar -xvf $BINLOG_TAR_FILE >&2 for ii in $(ls -1 ${BINLOG_FILENAME}.*) do - if ! [ -z $WSREP_SST_OPT_BINLOG_INDEX ] - echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_FILENAME}.index - then - echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_INDEX_DIRNAME}/${BINLOG_INDEX_FILENAME}.index + if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ] + then + echo ${BINLOG_DIRNAME}/${bin_file} >> ${BINLOG_FILENAME}.index + else + echo ${BINLOG_DIRNAME}/${ii} >> ${BINLOG_INDEX_DIRNAME}/${BINLOG_INDEX_FILENAME}.index fi done fi + cd "$OLD_PWD" fi + if [ -r "$MAGIC_FILE" ] then # UUID:seqno & wsrep_gtid_domain_id is received here. diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 8c8af9a0219..0f5dd1f61c7 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -18,10 +18,8 @@ # Documentation: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html # Make sure to read that before proceeding! - - - . $(dirname $0)/wsrep_sst_common +wsrep_check_datadir ealgo="" ekey="" @@ -249,7 +247,7 @@ verify_file_exists() get_transfer() { - TSST_PORT=${WSREP_SST_OPT_PORT:-4444} + TSST_PORT=$WSREP_SST_OPT_PORT if [[ $tfmt == 'nc' ]];then if [[ ! -x `which nc` ]];then @@ -267,26 +265,46 @@ get_transfer() wsrep_log_info "Using netcat as streamer" if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q ncat;then # Ncat - tcmd="nc $sockopt -l ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then + tcmd="nc -l ${TSST_PORT}" + elif nc -h 2>&1 | grep -qw -- '-d\>';then # Debian netcat - tcmd="nc $sockopt -dl ${TSST_PORT}" + if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ];then + # When host is not explicitly specified (when only the port + # is specified) netcat can only bind to an IPv4 address if + # the "-6" option is not explicitly specified: + tcmd="nc -dl -6 ${TSST_PORT}" + else + tcmd="nc -dl ${TSST_PORT}" + fi else # traditional netcat - tcmd="nc $sockopt -l -p ${TSST_PORT}" + tcmd="nc -l -p ${TSST_PORT}" fi else + # Check to see if netcat supports the '-N' flag. + # -N Shutdown the network socket after EOF on stdin + # If it supports the '-N' flag, then we need to use the '-N' + # flag, otherwise the transfer will stay open after the file + # transfer and cause the command to timeout. + # Older versions of netcat did not need this flag and will + # return an error if the flag is used. + # + tcmd_extra="" + if nc -h 2>&1 | grep -qw -- -N;then + tcmd_extra+="-N" + fi + # netcat doesn't understand [] around IPv6 address if nc -h 2>&1 | grep -q ncat;then # Ncat - tcmd="nc ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" - elif nc -h 2>&1 | grep -q -- '-d\>';then + tcmd="nc ${tcmd_extra} ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" + elif nc -h 2>&1 | grep -qw -- '-d\>';then # Debian netcat - tcmd="nc ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" + tcmd="nc ${tcmd_extra} ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" else # traditional netcat - tcmd="nc -q0 ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" + tcmd="nc -q0 ${tcmd_extra} ${WSREP_SST_OPT_HOST_UNESCAPED} ${TSST_PORT}" fi fi else @@ -863,7 +881,7 @@ if [[ ${FORCE_FTWRL:-0} -eq 1 ]];then fi -INNOEXTRA="" +INNOEXTRA= if [[ $ssyslog -eq 1 ]];then @@ -901,7 +919,7 @@ get_transfer INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""} # Try to set INNODB_DATA_HOME_DIR from the command line: -if [ ! -z "$INNODB_DATA_HOME_DIR_ARG" ]; then +if [ -n "$INNODB_DATA_HOME_DIR_ARG" ]; then INNODB_DATA_HOME_DIR=$INNODB_DATA_HOME_DIR_ARG fi # if no command line arg and INNODB_DATA_HOME_DIR environment variable @@ -912,8 +930,8 @@ fi if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '') fi -if [ ! -z "$INNODB_DATA_HOME_DIR" ]; then - INNOEXTRA+=" --innodb-data-home-dir=$INNODB_DATA_HOME_DIR" +if [ -n "$INNODB_DATA_HOME_DIR" ]; then + INNOEXTRA+=" --innodb-data-home-dir=$INNODB_DATA_HOME_DIR" fi if [ -n "$INNODB_DATA_HOME_DIR" ]; then @@ -925,7 +943,7 @@ else fi INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts \$INNOEXTRA --apply-log \$rebuildcmd \${DATA} ${INNOAPPLY}" -INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} ${INNOMOVE}" +INNOMOVE="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $impts --move-back --force-non-empty-directories \${DATA} ${INNOMOVE}" INNOBACKUP="${INNOBACKUPEX_BIN} ${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir ${INNOBACKUP}" if [ "$WSREP_SST_OPT_ROLE" = "donor" ] @@ -1000,7 +1018,7 @@ then wsrep_log_info "Sleeping before data transfer for SST" sleep 10 - wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT:-4444}" + wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT}" # Add compression to the head of the stream (if specified) if [[ -n $scomp ]]; then @@ -1064,17 +1082,25 @@ then ib_home_dir=$INNODB_DATA_HOME_DIR - # Try to set ib_log_dir from the command line: - ib_log_dir=$INNODB_LOG_GROUP_HOME_ARG - if [ -z "$ib_log_dir" ]; then - ib_log_dir=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-log-group-home-dir "") + WSREP_LOG_DIR=${WSREP_LOG_DIR:-""} + # Try to set WSREP_LOG_DIR from the command line: + if [ -n "$INNODB_LOG_GROUP_HOME_ARG" ]; then + WSREP_LOG_DIR=$INNODB_LOG_GROUP_HOME_ARG + fi + # if no command line arg and WSREP_LOG_DIR is not set, + # try to get it from my.cnf: + if [ -z "$WSREP_LOG_DIR" ]; then + WSREP_LOG_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-log-group-home-dir '') fi - if [ -z "$ib_log_dir" ]; then - ib_log_dir=$(parse_cnf --mysqld innodb-log-group-home-dir "") + if [ -z "$WSREP_LOG_DIR" ]; then + WSREP_LOG_DIR=$(parse_cnf --mysqld innodb-log-group-home-dir '') fi + ib_log_dir=$WSREP_LOG_DIR + # Try to set ib_undo_dir from the command line: ib_undo_dir=$INNODB_UNDO_DIR_ARG + # if no command line arg then try to get it from my.cnf: if [ -z "$ib_undo_dir" ]; then ib_undo_dir=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-undo-directory "") fi @@ -1094,7 +1120,7 @@ then # May need xtrabackup_checkpoints later on rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile - wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT:-4444} ${MODULE} & + wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT} ${MODULE} & trap sig_joiner_cleanup HUP PIPE INT TERM trap cleanup_joiner EXIT @@ -1146,17 +1172,17 @@ then find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+ fi - tempdir=$LOG_BIN_ARG + tempdir=$WSREP_SST_OPT_BINLOG if [ -z "$tempdir" ]; then tempdir=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE log-bin "") fi if [ -z "$tempdir" ]; then tempdir=$(parse_cnf --mysqld log-bin "") fi - if [[ -n ${tempdir:-} ]];then + if [ -n "$tempdir" ]; then binlog_dir=$(dirname $tempdir) binlog_file=$(basename $tempdir) - if [[ -n ${binlog_dir:-} && $binlog_dir != '.' && $binlog_dir != $DATA ]];then + if [[ -n "${binlog_dir:-}" && "$binlog_dir" != '.' && "$binlog_dir" != "$DATA" ]]; then pattern="$binlog_dir/$binlog_file\.[0-9]+$" wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" find $binlog_dir -maxdepth 1 -type f -regex $pattern -exec rm -fv {} 1>&2 \+ || true @@ -1164,12 +1190,9 @@ then fi fi - - TDATA=${DATA} DATA="${DATA}/.sst" - MAGIC_FILE="${DATA}/${INFO_FILE}" wsrep_log_info "Waiting for SST streaming to complete!" monitor_process $jpid @@ -1236,7 +1259,7 @@ then fi - if [[ ! -z $WSREP_SST_OPT_BINLOG ]];then + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then BINLOG_DIRNAME=$(dirname $WSREP_SST_OPT_BINLOG) BINLOG_FILENAME=$(basename $WSREP_SST_OPT_BINLOG) diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh index c4feac2a135..504e2fc039a 100644 --- a/scripts/wsrep_sst_xtrabackup.sh +++ b/scripts/wsrep_sst_xtrabackup.sh @@ -18,10 +18,8 @@ # Optional dependencies and options documented here: http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html # Make sure to read that before proceeding! - - - . $(dirname $0)/wsrep_sst_common +wsrep_check_datadir ealgo="" ekey="" @@ -134,7 +132,7 @@ get_keys() get_transfer() { - TSST_PORT=${WSREP_SST_OPT_PORT:-4444} + TSST_PORT=$WSREP_SST_OPT_PORT if [[ $tfmt == 'nc' ]];then if [[ ! -x `which nc` ]];then @@ -532,9 +530,9 @@ then # May need xtrabackup_checkpoints later on rm -f ${DATA}/xtrabackup_binary ${DATA}/xtrabackup_galera_info ${DATA}/xtrabackup_logfile - ADDR="${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT:-4444}" + ADDR="${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT}" - wait_for_listen ${WSREP_SST_OPT_PORT:-4444} ${ADDR} ${MODULE} & + wait_for_listen ${WSREP_SST_OPT_PORT} ${ADDR} ${MODULE} & trap sig_joiner_cleanup HUP PIPE INT TERM trap cleanup_joiner EXIT |