diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/wsrep_sst_common.sh | 321 | ||||
| -rw-r--r-- | scripts/wsrep_sst_mariabackup.sh | 263 | ||||
| -rw-r--r-- | scripts/wsrep_sst_mysqldump.sh | 7 | ||||
| -rw-r--r-- | scripts/wsrep_sst_rsync.sh | 475 |
4 files changed, 664 insertions, 402 deletions
diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..d4d9a58897d 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2012-2015 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -22,6 +22,62 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + +to_minuses() +{ + local x="$1" + local t="${1#*_}" + local r="" + while [ "$t" != "$x" ]; do + r="$r${x%%_*}-" + x="$t" + t="${t#*_}" + done + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$r$x" + else + echo "$r$x" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -43,10 +99,9 @@ WSREP_SST_OPT_ADDR="" WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" -WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -66,20 +121,22 @@ case "$1" in # without square brackets: readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: - readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" + readonly WSREP_SST_OPT_HOST="[$WSREP_SST_OPT_HOST_UNESCAPED]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*\]}" ;; *) readonly WSREP_SST_OPT_HOST="${WSREP_SST_OPT_ADDR%%[:/]*}" readonly WSREP_SST_OPT_HOST_UNESCAPED="$WSREP_SST_OPT_HOST" - readonly WSREP_SST_OPT_HOST_ESCAPED="$WSREP_SST_OPT_HOST" readonly WSREP_SST_OPT_HOST_IPv6=0 + # Let's remove the leading part that contains the host address: + remain="${WSREP_SST_OPT_ADDR#*[:/]}" ;; esac - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" + # If there is nothing but the address, then the remainder is empty: + [ "$remain" = "$WSREP_SST_OPT_ADDR" ] && remain="" # Let's remove the ":" character that separates the port number # from the hostname: remain="${remain#:}" @@ -87,39 +144,32 @@ case "$1" in # up to "/" (if present): WSREP_SST_OPT_ADDR_PORT="${remain%%/*}" # If the "/" character is present, then the path is not empty: - if [ "${remain#*/}" != "$remain" ]; then + if [ "$WSREP_SST_OPT_ADDR_PORT" != "$remain" ]; then # This operation removes everything up to the "/" character, # effectively removing the port number from the string: readonly WSREP_SST_OPT_PATH="${remain#*/}" else readonly WSREP_SST_OPT_PATH="" fi - # The rest of the string is the same as the path (for now): - remain="$WSREP_SST_OPT_PATH" - # If there is one more "/" in the string, then everything before - # it will be the module name, otherwise the module name is empty: - if [ "${remain%%/*}" != "$remain" ]; then - # This operation removes the tail after the very first - # occurrence of the "/" character (inclusively): - readonly WSREP_SST_OPT_MODULE="${remain%%/*}" - else - readonly WSREP_SST_OPT_MODULE="" - fi # Remove the module name part from the string, which ends with "/": remain="${WSREP_SST_OPT_PATH#*/}" - # If the rest of the string does not match the original, then there - # was something else besides the module name: + # This operation removes the tail after the very first occurrence + # of the "/" character, inclusively: + readonly WSREP_SST_OPT_MODULE="${WSREP_SST_OPT_PATH%%/*}" + # If there is one more "/" in the string, then everything before + # it will be the LSN, otherwise the LSN is empty: if [ "$remain" != "$WSREP_SST_OPT_PATH" ]; then # Extract the part that matches the LSN by removing all # characters starting from the very first "/": readonly WSREP_SST_OPT_LSN="${remain%%/*}" # Exctract everything after the first occurrence of # the "/" character in the string: + source="$remain" remain="${remain#*/}" # If the remainder does not match the original string, # then there is something else (the version number in # our case): - if [ "$remain" != "$WSREP_SST_OPT_LSN" ]; then + if [ "$remain" != "$source" ]; then # Let's extract the version number by removing the tail # after the very first occurence of the "/" character # (inclusively): @@ -138,22 +188,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -182,14 +232,12 @@ case "$1" in readonly WSREP_SST_OPT_HOST_UNESCAPED="${addr_no_bracket%%\]*}" # Square brackets are needed in most cases: readonly WSREP_SST_OPT_HOST="[${WSREP_SST_OPT_HOST_UNESCAPED}]" - readonly WSREP_SST_OPT_HOST_ESCAPED="\\[${WSREP_SST_OPT_HOST_UNESCAPED}\\]" # Mark this address as IPv6: readonly WSREP_SST_OPT_HOST_IPv6=1 ;; *) readonly WSREP_SST_OPT_HOST="$2" readonly WSREP_SST_OPT_HOST_UNESCAPED="$2" - readonly WSREP_SST_OPT_HOST_ESCAPED="$2" readonly WSREP_SST_OPT_HOST_IPv6=0 ;; esac @@ -247,6 +295,7 @@ case "$1" in '--mysqld-args') original_cmd="" shift + cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -261,7 +310,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -279,7 +328,7 @@ case "$1" in else # If it's not bash, then we need to use slow # external utilities: - option=$(echo "$options" | cut -c1-1) + option=$(echo "$options" | cut -c1) fi # And the subsequent characters consider option value: value="" @@ -301,15 +350,25 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" + elif [ "$2" = '--' ]; then + shift + if [ $# -gt 1 ]; then + cmd_tail=1 + shift + value="$1" + fi fi fi - if [ $option = 'h' ]; then + if [ "$option" = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi - elif [ $option != 'u' -a \ - $option != 'P' ] + elif [ "$option" != 'u' -a \ + "$option" != 'P' ] then + if [ $cmd_tail -ne 0 ]; then + option="$option --" + fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -317,9 +376,16 @@ case "$1" in else slist="$slist -$option '$value'" fi + break + fi + if [ $cmd_tail -ne 0 ]; then + if [ -n "$slist" ]; then + slist="$slist --" + else + slist='-' + fi fi break - else slist="$slist$option" fi @@ -329,7 +395,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an equal sign without any characters after it: + # We found an minus sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -338,12 +404,25 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + if [ $cmd_tail -ne 0 ]; then + # All other arguments must be copied unchanged: + while [ $# -gt 0 ]; do + original_cmd="$original_cmd '$1'" + shift + done + break + fi + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after # the first equal sign: option="${1%%=*}" + # If the option name contains underscores, then replace + # them to minuses: + if [ "${option#*_}" != "$option" ]; then + option=$(to_minuses "$option") + fi # The "--loose-" prefix should not affect the recognition # of the option name: if [ "${option#--loose-}" != "$option" ]; then @@ -370,49 +449,49 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-force-recovery') if [ -n "$value" -a "$value" != "0" ]; then - INNODB_FORCE_RECOVERY="$value" + INNODB_FORCE_RECOVERY=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin') if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - MYSQLD_OPT_LOG_BIN="$value" + MYSQLD_OPT_LOG_BIN=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-bin-index') if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then - MYSQLD_OPT_LOG_BIN_INDEX="$value" + MYSQLD_OPT_LOG_BIN_INDEX=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--log-basename') if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then - MYSQLD_OPT_LOG_BASENAME="$value" + MYSQLD_OPT_LOG_BASENAME=$(trim_string "$value") fi skip_mysqld_arg=1 ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -471,8 +550,8 @@ if [ -z "$WSREP_SST_OPT_BINLOG" -a -n "${MYSQLD_OPT_LOG_BIN+x}" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments to mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else # Take the default name: @@ -525,26 +604,23 @@ get_binlog() WSREP_SST_OPT_BINLOG_INDEX=$(parse_cnf '--mysqld' 'log-bin-index') fi # if no command line argument and WSREP_SST_OPT_LOG_BASENAME is not set, - # try to get it from my.cnf: + # then try to get it from my.cnf: if [ -z "$WSREP_SST_OPT_LOG_BASENAME" ]; then WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -ne 0 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,13 +630,13 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else - # the default name (note that base of this name + # Use the default name (note that base of this name # is already defined above): readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_BINLOG.index" fi @@ -594,27 +670,18 @@ if [ -n "$WSREP_SST_OPT_ADDR_PORT" ]; then # the corresponding variable: readonly WSREP_SST_OPT_PORT="$WSREP_SST_OPT_ADDR_PORT" fi -elif [ -n "$WSREP_SST_OPT_ADDR" ]; then +else # If the port is missing, take the default port: if [ -z "$WSREP_SST_OPT_PORT" ]; then readonly WSREP_SST_OPT_PORT=4444 fi WSREP_SST_OPT_ADDR_PORT="$WSREP_SST_OPT_PORT" - # Let's remove the leading part that contains the host address: - remain="${WSREP_SST_OPT_ADDR#$WSREP_SST_OPT_HOST_ESCAPED}" - # Let's remove the ":" character that separates the port number - # from the hostname: - remain="${remain#:}" - # Let's remove all characters upto first "/" character that - # separates the hostname with port number from the path: - remain="${remain#/}" - # Let's construct a new value for the address with the port: - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT" - if [ -n "$remain" ]; then - WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_ADDR/$remain" - fi fi +# Let's construct a new value for the address with the port: +sst_path="${WSREP_SST_OPT_PATH:+/}$WSREP_SST_OPT_PATH" +WSREP_SST_OPT_ADDR="$WSREP_SST_OPT_HOST:$WSREP_SST_OPT_PORT$sst_path" + readonly WSREP_SST_OPT_ADDR readonly WSREP_SST_OPT_ADDR_PORT @@ -632,7 +699,7 @@ commandex() # try to use my_print_defaults, mysql and mysqldump that come # with the sources (for MTR suite): script_binary=$(dirname "$0") -SCRIPTS_DIR=$(cd "$script_binary"; pwd -P) +SCRIPTS_DIR=$(cd "$script_binary"; pwd) EXTRA_DIR="$SCRIPTS_DIR/../extra" CLIENT_DIR="$SCRIPTS_DIR/../client" @@ -725,8 +792,11 @@ parse_cnf() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -751,10 +821,16 @@ parse_cnf() done # Use default if we haven't found a value: - if [ -z "$reval" ]; then - [ -n "${3:-}" ] && reval="$3" + [ -z "$reval" ] && reval="${3:-}" + + # Truncate spaces: + [ -n "$reval" ] && reval=$(trim_string "$reval") + + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" fi - echo "$reval" } # @@ -780,8 +856,11 @@ in_config() local group="${groups%%\|*}" # Remove the remainder (the group name) from the rest # of the groups list (as if it were a prefix): - groups="${groups#$group}" - groups="${groups#\|}" + if [ "$group" != "$groups" ]; then + groups="${groups#*\|}" + else + groups="" + fi # If the group name is the same as the "mysqld" without "--" prefix, # then try to use it together with the group suffix: if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then @@ -804,7 +883,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +1020,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n "$(commandex cksum)" -a \ + -n "$(commandex printf)" ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1073,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1049,7 +1143,7 @@ check_port() ss -nlpH "( sport = :$port )" 2>/dev/null | \ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0 else - wsrep_log_error "unknown sockets utility" + wsrep_log_error "Unknown sockets utility" exit 2 # ENOENT fi @@ -1158,13 +1252,6 @@ verify_cert_matches_key() exit 22 fi - # If the diff utility is not installed, then - # we will not do this certificate check: - if [ -z "$(commandex diff)" ]; then - wsrep_log_info "diff utility not found" - return - fi - # If the openssl utility is not installed, then # we will not do this certificate check: get_openssl @@ -1175,9 +1262,9 @@ verify_cert_matches_key() # Generate the public key from the cert and the key. # They should match (otherwise we can't create an SSL connection). - if ! diff <("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null) \ - <("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null) >/dev/null 2>&1 - then + local pk1=$("$OPENSSL_BINARY" x509 -in "$cert" -pubkey -noout 2>/dev/null || :) + local pk2=$("$OPENSSL_BINARY" pkey -in "$key" -pubout 2>/dev/null || :) + if [ "$pk1" != "$pk2" ]; then wsrep_log_error "******************* FATAL ERROR *****************" wsrep_log_error "* The certificate and private key do not match. *" wsrep_log_error "* Please check your certificate and key files. *" @@ -1220,28 +1307,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. @@ -1272,6 +1337,10 @@ check_pid() rm -f "$pid_file" || : fi fi + local config="${3:-}" + if [ -n "$config" -a -f "$config" ]; then + rm -f "$config" || : + fi CHECK_PID=0 return 1 } @@ -1377,13 +1446,9 @@ check_server_ssl_config() fi fi if [ -n "$tcert" ]; then - tcert=$(trim_string "$tcert") if [ "${tcert%/}" != "$tcert" -o -d "$tcert" ]; then tcap="$tcert" tcert="" fi fi - if [ -n "$tcap" ]; then - tcap=$(trim_string "$tcap") - fi } diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index 471b0e5f5b0..ce4001fdc56 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2013 Percona Inc # # This program is free software; you can redistribute it and/or modify @@ -40,7 +40,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode="" sockopt="" progress="" ttime=0 @@ -85,13 +85,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -104,17 +104,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -154,7 +151,7 @@ get_keys() return fi - if [ $sfmt = 'tar' ]; then + if [ "$sfmt" = 'tar' ]; then wsrep_log_info "NOTE: key-based encryption (encrypt=1)" \ "cannot be enabled with tar format" encrypt=-1 @@ -184,11 +181,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -229,15 +226,15 @@ get_keys() get_transfer() { - if [ $tfmt = 'nc' ]; then + if [ "$tfmt" = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -259,14 +256,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -326,7 +323,8 @@ get_transfer() if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then if [ -z "$ssl_dhparams" ]; then # Determine the socat version - SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*') + SOCAT_VERSION=$(socat -V 2>&1 | \ + grep -m1 -owE '[0-9]+(\.[0-9]+)+' | head -n1) if [ -z "$SOCAT_VERSION" ]; then wsrep_log_error "******** FATAL ERROR ******************" wsrep_log_error "* Cannot determine the socat version. *" @@ -456,7 +454,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -476,6 +474,15 @@ read_cnf() tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | \ tr [:lower:] [:upper:]) + case "$tmode" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$tmode'" + exit 22 # EINVAL + ;; + esac + if [ $encrypt -eq 0 -o $encrypt -ge 2 ]; then if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]; then check_server_ssl_config @@ -583,8 +590,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -595,6 +608,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -624,7 +639,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -661,16 +676,13 @@ setup_ports() # wait_for_listen() { - local PORT="$1" - local ADDR="$2" - local MODULE="$3" for i in {1..150}; do - if check_port "" "$PORT" 'socat|nc'; then + if check_port "" "$SST_PORT" 'socat|nc'; then break fi sleep 0.2 done - echo "ready $ADDR/$MODULE//$sst_ver" + echo "ready $ADDR:$SST_PORT/$MODULE/$lsn/$sst_ver" } check_extra() @@ -715,7 +727,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -727,7 +739,7 @@ recv_joiner() set +e if [ $wait -ne 0 ]; then - wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" & + wait_for_listen & fi timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -761,7 +773,7 @@ recv_joiner() # check donor supplied secret SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) + cut -d ' ' -f2) if [ "$SECRET" != "$MY_SECRET" ]; then wsrep_log_error "Donor does not know my secret!" wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" @@ -815,7 +827,9 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -823,25 +837,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd)" cd "$OLD_PWD" @@ -869,7 +875,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -929,7 +935,7 @@ setup_commands() recovery=" --innodb-force-recovery=$INNODB_FORCE_RECOVERY" fi INNOAPPLY="$BACKUP_BIN --prepare$disver$recovery${iapts:+ }$iapts$INNOEXTRA --target-dir='$DATA' --datadir='$DATA'$mysqld_args $INNOAPPLY" - INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" + INNOMOVE="$BACKUP_BIN$WSREP_SST_OPT_CONF --move-back$disver${impts:+ }$impts$INNOEXTRA --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE" INNOBACKUP="$BACKUP_BIN$WSREP_SST_OPT_CONF --backup$disver${iopts:+ }$iopts$tmpopts$INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA'$mysqld_args $INNOBACKUP" } @@ -1052,6 +1058,11 @@ then iopts="--parallel=$backup_threads${iopts:+ }$iopts" fi + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + iopts="--sst-max-binlogs=$max_binlogs${iopts:+ }$iopts" + fi + setup_commands set +e timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )" @@ -1096,6 +1107,7 @@ then echo "done $WSREP_SST_OPT_GTID" wsrep_log_info "Total time on donor: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on donor" elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then @@ -1103,22 +1115,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1142,16 +1185,15 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE="${WSREP_SST_OPT_MODULE:-xtrabackup_sst}" [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" # May need xtrabackup_checkpoints later on [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary" [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info" - [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0" - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" if [ "${tmode#VERIFY}" != "$tmode" ]; then # backward-incompatible behavior: @@ -1165,7 +1207,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1219,6 +1261,36 @@ then jpid=$! wsrep_log_info "Proceeding with SST" + get_binlog + + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + cd "$DATA" + wsrep_log_info "Cleaning the old binary logs" + # If there is a file with binlogs state, delete it: + [ -f "$binlog_base.state" ] && rm -fv "$binlog_base.state" 1>&2 + # Clean up the old binlog files and index: + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done < "$binlog_index" + rm -fv "$binlog_index" 1>&2 + fi + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + if [ "$(pwd)" != "$DATA_DIR" ]; then + wsrep_log_info \ + "Cleaning the binlog directory '$binlog_dir' as well" + fi + fi + rm -fv "$binlog_base".[0-9]* 1>&2 || : + cd "$OLD_PWD" + fi + wsrep_log_info \ "Cleaning the existing datadir and innodb-data/log directories" if [ "$OS" = 'FreeBSD' ]; then @@ -1235,20 +1307,6 @@ then -o -exec rm -rfv {} 1>&2 \+ fi - get_binlog - - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" - fi - fi - TDATA="$DATA" DATA="$DATA/.sst" @@ -1282,11 +1340,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1296,8 +1356,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1314,25 +1375,9 @@ then fi fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done - cd "$OLD_PWD" - - fi - wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,10 +1385,43 @@ then exit 22 fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + cd "$DATA_DIR" + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + fi + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + if [ -n "$binlogs" ]; then + wsrep_log_info "Moving binary logs to $binlog_dir" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + mv "$DATA/$bin_file" "$binlog_dir" + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi + cd "$OLD_PWD" + fi + MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" @@ -1370,6 +1448,7 @@ then cat "$MAGIC_FILE" # Output : UUID:seqno wsrep_gtid_domain_id wsrep_log_info "Total time on joiner: $totime seconds" + wsrep_log_info "mariabackup SST/IST completed on joiner" fi exit 0 diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index 1c8fc181328..3c92f489cb5 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -3,7 +3,7 @@ set -ue # Copyright (C) 2009-2015 Codership Oy -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,8 +40,7 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null -then +if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]'; then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" exit $EINVAL @@ -95,7 +94,7 @@ DROP PREPARE stmt;" SET_START_POSITION="SET GLOBAL wsrep_start_position='$WSREP_SST_OPT_GTID';" SET_WSREP_GTID_DOMAIN_ID="" -if [ -n $WSREP_SST_OPT_GTID_DOMAIN_ID ]; then +if [ -n "$WSREP_SST_OPT_GTID_DOMAIN_ID" ]; then SET_WSREP_GTID_DOMAIN_ID=" SET @val = (SELECT GLOBAL_VALUE FROM INFORMATION_SCHEMA.SYSTEM_VARIABLES WHERE VARIABLE_NAME = 'WSREP_GTID_STRICT_MODE' AND GLOBAL_VALUE > 0); SET @stmt = IF (@val IS NOT NULL, 'SET GLOBAL WSREP_GTID_DOMAIN_ID=$WSREP_SST_OPT_GTID_DOMAIN_ID', 'SET @dummy = 0'); diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index cc73db5da87..67a7afc638f 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -2,7 +2,7 @@ set -ue -# Copyright (C) 2017-2021 MariaDB +# Copyright (C) 2017-2022 MariaDB # Copyright (C) 2010-2014 Codership Oy # # This program is free software; you can redistribute it and/or modify @@ -36,6 +36,8 @@ cleanup_joiner() { local failure=0 + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -58,6 +60,7 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -140,66 +143,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') +OLD_PWD="$(pwd)" + +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi +DATA_DIR="$(pwd)" -OLD_PWD="$(pwd)" +cd "$OLD_PWD" + +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi -WSREP_LOG_DIR=$(pwd -P) -cd "$OLD_PWD" +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd)" + cd "$OLD_PWD" +fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) -cd "$OLD_PWD" - -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -224,6 +238,15 @@ if [ -z "$SSLMODE" ]; then elif [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then SSLMODE='REQUIRED' fi +else + case "$SSLMODE" in + 'VERIFY_IDENTITY'|'VERIFY_CA'|'REQUIRED'|'DISABLED') + ;; + *) + wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" + exit 22 # EINVAL + ;; + esac fi if [ -n "$SSTKEY" -a -n "$SSTCERT" ]; then @@ -248,18 +271,11 @@ VERIFY_OPT="" CHECK_OPT="" CHECK_OPT_LOCAL="" if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then - case "$SSLMODE" in - 'VERIFY_IDENTITY') + if [ "$SSLMODE" = 'VERIFY_IDENTITY' ]; then VERIFY_OPT='verifyPeer = yes' - ;; - 'VERIFY_CA') + else VERIFY_OPT='verifyChain = yes' - ;; - *) - wsrep_log_error "Unrecognized ssl-mode option: '$SSLMODE'" - exit 22 # EINVAL - ;; - esac + fi if [ -z "$SSTCA$SSTCAP" ]; then wsrep_log_error "Can't have ssl-mode='$SSLMODE' without CA file or path" exit 22 # EINVAL @@ -276,7 +292,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -293,14 +309,59 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' -if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] -then +SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" + +# give some time for previous SST to complete: +check_round=0 +while check_pid "$SST_PID" 0; do + wsrep_log_info "Previous SST is not completed, waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 20 ]; then + wsrep_log_error "previous SST script still running." + exit 114 # EALREADY + fi + sleep 1 +done + +echo $$ > "$SST_PID" + +# give some time for stunnel from the previous SST to complete: +check_round=0 +while check_pid "$STUNNEL_PID" 1 "$STUNNEL_CONF"; do + wsrep_log_info "Lingering stunnel daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "stunnel daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +MODULE="${WSREP_SST_OPT_MODULE:-rsync_sst}" + +RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" +RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" + +# give some time for rsync from the previous SST to complete: +check_round=0 +while check_pid "$RSYNC_PID" 1 "$RSYNC_CONF"; do + wsrep_log_info "Lingering rsync daemon found at startup," \ + "waiting for it to exit" + check_round=$(( check_round + 1 )) + if [ $check_round -eq 10 ]; then + wsrep_log_error "rsync daemon still running." + exit 114 # EALREADY + fi + sleep 1 +done + +[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" +[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" +if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then if [ -n "$STUNNEL" ] then @@ -319,8 +380,6 @@ ${VERIFY_OPT} ${CHECK_OPT} ${CHECK_OPT_LOCAL} EOF - else - [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" fi RC=0 @@ -333,7 +392,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -357,32 +416,100 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + max_binlogs=$(parse_cnf "$encgroups" 'sst-max-binlogs') + if [ -n "$max_binlogs" ]; then + binlog_files="" + if [ $max_binlogs -gt 0 ]; then + binlog_files=$(tail -n $max_binlogs \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qw -F -- '--transform'; then + tar_type=1 + elif tar --version | grep -q -E '^bsdtar\>'; then + tar_type=2 + fi + if [ $tar_type -ne 2 ]; then + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$binlog_files" >&2 + else + echo "$binlog_files" >&2 + fi + fi + if [ $tar_type -ne 0 ]; then + # Preparing list of the binlog file names: + echo "$binlog_files" | { + binlogs="" + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + if [ -n "$BASH_VERSION" ]; then + first="${bin_file:0:1}" + else + first=$(echo "$bin_file" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_file="./$bin_file" + fi + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + if [ -n "$binlogs" ]; then + if [ $tar_type -eq 1 ]; then + tar_options="--transform='s/^.*\///g'" + else + # bsdtar handles backslash incorrectly: + tar_options="-s '?^.*/??g'" + fi + eval tar -P $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + fi + } + else + tar_options='-cvf' + echo "$binlog_files" | \ + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + if [ -n "$BASH_VERSION" ]; then + first="${bin_base:0:1}" + else + first=$(echo "$bin_base" | cut -c1) + fi + if [ "$first" = '-' -o "$first" = '@' ]; then + bin_base="./$bin_base" + fi + if [ -n "$bin_dir" -a "$bin_dir" != '.' ]; then + tar $tar_options "$BINLOG_TAR_FILE" \ + -C "$bin_dir" "$bin_base" >&2 + else + tar $tar_options "$BINLOG_TAR_FILE" \ + "$bin_base" >&2 + fi + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -399,9 +526,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -435,7 +562,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -448,7 +575,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -459,7 +586,7 @@ FILTER="-f '- /lost+found' # then, we parallelize the transfer of database directories, # use '.' so that path concatenation works: - cd "$WSREP_SST_OPT_DATA" + cd "$DATA" backup_threads=$(parse_cnf '--mysqld|sst' 'backup-threads') if [ -z "$backup_threads" ]; then @@ -518,68 +645,21 @@ FILTER="-f '- /lost+found' [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID" fi + [ -f "$SST_PID" ] && rm -f "$SST_PID" + + wsrep_log_info "rsync SST/IST completed on donor" + elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ] then check_sockets_utils - SST_PID="$WSREP_SST_OPT_DATA/wsrep_sst.pid" - - # give some time for previous SST to complete: - check_round=0 - while check_pid "$SST_PID" 0 'wsrep_sst_'; do - wsrep_log_info "previous SST is not completed, waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "previous SST script still running." - exit 114 # EALREADY - fi - sleep 1 - done - - echo $$ > "$SST_PID" - - # give some time for stunnel from the previous SST to complete: - check_round=0 - while check_pid "$STUNNEL_PID" 1; do - wsrep_log_info "Lingering stunnel daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "stunnel daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - MODULE="rsync_sst" - RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" - RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" - - # give some time for rsync from the previous SST to complete: - check_round=0 - while check_pid "$RSYNC_PID" 1; do - wsrep_log_info "Lingering rsync daemon found at startup," \ - "waiting for it to exit" - check_round=$(( check_round + 1 )) - if [ $check_round -eq 10 ]; then - wsrep_log_error "rsync daemon already running." - exit 114 # EALREADY - fi - sleep 1 - done - - [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" - [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" - - [ -z "$STUNNEL" -a -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF" - - ADDR="$WSREP_SST_OPT_ADDR" + ADDR="$WSREP_SST_OPT_HOST" RSYNC_PORT="$WSREP_SST_OPT_PORT" RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -600,13 +680,11 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF -# rm -rf "$DATA/ib_logfile"* # we don't want old logs around - # If the IP is local, listen only on it: if is_local_ip "$RSYNC_ADDR_UNESCAPED" then @@ -617,7 +695,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -675,11 +753,10 @@ EOF TRANSFER_PID="$STUNNEL_PID" fi - if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ] - then # backward-incompatible behavior + if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then + # backward-incompatible behavior: CN="" - if [ -n "$SSTCERT" ] - then + if [ -n "$SSTCERT" ]; then # find out my Common Name get_openssl if [ -z "$OPENSSL_BINARY" ]; then @@ -688,7 +765,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d '=' -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -696,7 +773,6 @@ EOF ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST" else MY_SECRET="" # for check down in recv_joiner() - ADDR="$WSREP_SST_OPT_HOST" fi until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID \ @@ -725,16 +801,53 @@ EOF exit 32 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + # This message should cause joiner to abort: + wsrep_log_info "rsync process ended without creating magic file" + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + cd "$DATA" + # Clean up the old binlog files and index: binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + if [ -f "$binlog_index" ]; then + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done < "$binlog_index" + rm -f "$binlog_index" + fi + binlog_cd=0 + # Change the directory to binlog base (if possible): + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi + # Clean up unindexed binlog files: + rm -f "$binlog_base".[0-9]* || : + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + fi + if [ $binlog_tar_present -ne 0 ]; then # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -744,46 +857,52 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp) fi - + index_dir=$(dirname "$binlog_index"); + if [ -n "$index_dir" -a "$index_dir" != '.' ]; then + [ ! -d "$index_dir" ] && mkdir -p "$index_dir" + fi + binlog_cd=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + binlog_cd=1 + cd "$binlog_dir" + fi + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" - if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" + RC=0 + if tar --version | grep -q -E '^bsdtar\>'; then + tar -tf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + tar -xvf "$BINLOG_TAR_FILE" > /dev/null || RC=$? + else + tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile" && \ + cat "$tmpfile" >&2 || RC=$? + fi + if [ $RC -ne 0 ]; then rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" + [ $binlog_cd -ne 0 ] && cd "$DATA_DIR" + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" done < "$tmpfile" rm -f "$tmpfile" - cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi + wsrep_log_info "rsync SST/IST completed on joiner" + # wsrep_cleanup_progress_file # cleanup_joiner else |