Backport 4bb49d84a9df, correct handling on defaults[-extra]-file is SST scripts

(from 10.1 to 10.0-galera)

This conflicted signficantly with 7d550c76be13c58551e203f4eeb4f87ebd58ba4b
which added --defaults-group-suffix support.

Took the approach of 4bb49d84a9df and adapted the
--defaults-group-suffix handling to be consistent.

The following changes as follows:

SST scripts now use $MY_PRINT_DEFAULTS rather than the lowercase for
consistency and this include all required --default arguements.

Backport/merge by Daniel Black <daniel@linux.vnet.ibm.com>

5 files changed, 58 insertions, 70 deletions

diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index 15632bf3683..eca1277f17e 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -20,11 +20,13 @@ set -u
 
 WSREP_SST_OPT_BYPASS=0
 WSREP_SST_OPT_BINLOG=""
-WSREP_SST_OPT_CONF_SUFFIX=""
 WSREP_SST_OPT_DATA=""
 WSREP_SST_OPT_AUTH=${WSREP_SST_OPT_AUTH:-}
 WSREP_SST_OPT_USER=${WSREP_SST_OPT_USER:-}
 WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
+WSREP_SST_OPT_DEFAULT=""
+WSREP_SST_OPT_EXTRA_DEFAULT=""
+WSREP_SST_OPT_SUFFIX_DEFAULT=""
 
 while [ $# -gt 0 ]; do
 case "$1" in
@@ -56,11 +58,15 @@ case "$1" in
         shift
         ;;
     '--defaults-file')
-        readonly WSREP_SST_OPT_CONF="$2"
+        readonly WSREP_SST_OPT_DEFAULT="$1=$2"
+        shift
+        ;;
+    '--defaults-extra-file')
+        readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$2"
         shift
         ;;
     '--defaults-group-suffix')
-        WSREP_SST_OPT_CONF_SUFFIX="$2"
+        readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2"
         shift
         ;;
     '--host')
@@ -112,7 +118,6 @@ shift
 done
 readonly WSREP_SST_OPT_BYPASS
 readonly WSREP_SST_OPT_BINLOG
-readonly WSREP_SST_OPT_CONF_SUFFIX
 
 # try to use my_print_defaults, mysql and mysqldump that come with the sources
 # (for MTR suite)
@@ -140,17 +145,18 @@ else
     MY_PRINT_DEFAULTS=$(which my_print_defaults)
 fi
 
+readonly WSREP_SST_OPT_CONF="$WSREP_SST_OPT_DEFAULT $WSREP_SST_OPT_EXTRA_DEFAULT $WSREP_SST_OPT_SUFFIX_DEFAULT"
+readonly MY_PRINT_DEFAULTS="${MY_PRINT_DEFAULTS} $WSREP_SST_OPT_CONF"
+
 wsrep_auth_not_set()
 {
     [ -z "$WSREP_SST_OPT_AUTH" -o "$WSREP_SST_OPT_AUTH" = "(null)" ]
 }
 
-# For Bug:1200727
-if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF sst | grep -q "wsrep_sst_auth"
-then
-    if wsrep_auth_not_set
-    then
-        WSREP_SST_OPT_AUTH=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF sst | grep -- "--wsrep_sst_auth" | cut -d= -f2)
+# State Snapshot Transfer authentication password was displayed in the ps output. Bug fixed #1200727.
+if $MY_PRINT_DEFAULTS sst | grep -q "wsrep_sst_auth"; then
+    if wsrep_auth_not_set; then
+            WSREP_SST_OPT_AUTH=$($MY_PRINT_DEFAULTS sst | grep -- "--wsrep_sst_auth" | cut -d= -f2)
     fi
 fi
 readonly WSREP_SST_OPT_AUTH
@@ -229,7 +235,7 @@ wsrep_check_programs()
 # process like encryption, etc.....
 # parse such configuration option. (group for xb settings is [sst] in my.cnf
 #
-# 1st param: group : name of the config file section, e.g. mysqld
+# 1st param: group (config file section like sst) or my_print_defaults argument (like --mysqld)
 # 2nd param: var : name of the variable in the section, e.g. server-id
 # 3rd param: - : default value for the param
 parse_cnf()
@@ -238,20 +244,12 @@ parse_cnf()
     local var=$2
     local reval=""
 
-    # print the default settings for given group using my_print_default.
     # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
     # then grep for needed variable
     # finally get the variable value (if variables has been specified multiple time use the last value only)
 
-    # look in group+suffix
-    if [ -n $WSREP_SST_OPT_CONF_SUFFIX ]; then
-        reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF "${group}${WSREP_SST_OPT_CONF_SUFFIX}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1)
-    fi
-
-    # look in group
-    if [ -z $reval ]; then
-        reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1)
-    fi
+    # TODO: get awk to do the grep/cut bits as well
+    reval=$($MY_PRINT_DEFAULTS "${group}" | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1)
 
     # use default if we haven't found a value
     if [ -z $reval ]; then
diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh
index f6f2187006d..2143176e6ba 100644
--- a/scripts/wsrep_sst_mysqldump.sh
+++ b/scripts/wsrep_sst_mysqldump.sh
@@ -17,10 +17,6 @@
 
 # This is a reference script for mysqldump-based state snapshot tansfer
 
-# This variable is not used in mysqldump sst, so better initialize it
-# to avoid shell's "parameter not set" message.
-WSREP_SST_OPT_CONF=""
-
 . $(dirname $0)/wsrep_sst_common
 PATH=$PATH:/usr/sbin:/usr/bin:/sbin:/bin
 
@@ -106,7 +102,7 @@ GTID_BINLOG_STATE=$(echo "SHOW GLOBAL VARIABLES LIKE 'gtid_binlog_state'" |\
 $MYSQL_CLIENT $AUTH -S$WSREP_SST_OPT_SOCKET --disable-reconnect --connect_timeout=10 |\
 tail -1 | awk -F ' ' '{ print $2 }')
 
-MYSQL="$MYSQL_CLIENT --defaults-extra-file=$WSREP_SST_OPT_CONF "\
+MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\
 "$AUTH -h${WSREP_SST_OPT_HOST_UNESCAPED:-$WSREP_SST_OPT_HOST} "\
 "-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10"
 
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index 1c1b09ce68f..5617142ea99 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -102,19 +102,7 @@ fi
 WSREP_LOG_DIR=${WSREP_LOG_DIR:-""}
 # if WSREP_LOG_DIR env. variable is not set, try to get it from my.cnf
 if [ -z "$WSREP_LOG_DIR" ]; then
-    WSREP_LOG_DIR=$(parse_cnf mariadb-10.0 innodb_log_group_home_dir "")
-fi
-if [ -z "$WSREP_LOG_DIR" ]; then
-    WSREP_LOG_DIR=$(parse_cnf mysqld innodb_log_group_home_dir "")
-fi
-if [ -z "$WSREP_LOG_DIR" ]; then
-    WSREP_LOG_DIR=$(parse_cnf server innodb_log_group_home_dir "")
-fi
-if [ -z "$WSREP_LOG_DIR" ]; then
-    WSREP_LOG_DIR=$(parse_cnf mariadb innodb_log_group_home_dir "")
-fi
-if [ -z "$WSREP_LOG_DIR" ]; then
-    WSREP_LOG_DIR=$(parse_cnf mysqld-10.0 innodb_log_group_home_dir "")
+    WSREP_LOG_DIR=$(parse_cnf --mysqld innodb-log-group-home-dir '')
 fi
 
 if [ -n "$WSREP_LOG_DIR" ]; then
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index a702dc641e0..0de5cb344d9 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -127,7 +127,7 @@ get_keys()
     fi
 
     if [[ $encrypt -eq 0 ]];then 
-        if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+        if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
             wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
         fi
         return
@@ -400,7 +400,7 @@ get_footprint()
 {
     pushd $WSREP_SST_OPT_DATA 1>/dev/null
     payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }')
-    if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+    if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then 
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
         payload=$(( payload*1/2 ))
@@ -491,7 +491,7 @@ read_cnf()
     ssystag+="-"
 
     if [[ $ssyslog -ne -1 ]];then 
-        if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld_safe | tr '_' '-' | grep -q -- "--syslog";then
+        if $MY_PRINT_DEFAULTS mysqld_safe | grep -q -- "--syslog";then
             ssyslog=1
         fi
     fi
@@ -667,8 +667,8 @@ check_extra()
 {
     local use_socket=1
     if [[ $uextra -eq 1 ]];then 
-        if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
-            local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+        if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$($MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
             if [[ -n $eport ]];then 
                 # Xtrabackup works only locally.
                 # Hence, setting host to 127.0.0.1 unconditionally. 
@@ -863,20 +863,31 @@ if [[ $ssyslog -eq 1 ]];then
             logger  -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@" 
         }
 
-        INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} 2>&1  | logger -p daemon.err -t ${ssystag}innobackupex-apply "
-        INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} 2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move "
-        INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
+        INNOAPPLY="2>&1  | logger -p daemon.err -t ${ssystag}innobackupex-apply "
+        INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move "
+        INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
     fi
 
 else 
-    INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
-    INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF}  --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $impts --datadir=${DATA} --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log"
-    INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF}  --defaults-group=mysqld${WSREP_SST_OPT_CONF_SUFFIX} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log"
+    INNOAPPLY="&>\${DATA}/innobackup.prepare.log"
+    INNOMOVE="&>\${DATA}/innobackup.move.log"
+    INNOBACKUP="2>\${DATA}/innobackup.backup.log"
 fi
 
 get_stream
 get_transfer
 
+# This is as close as innobackupex gets to mysqld defaults. Important note --defaults-extra-file passed to
+# mysqld is not passed to innobackupex.
+# WSREP_SST_OPT_SUFFIX_DEFAULT goes from --defaults-group-suffix=.X to --defaults-group=mysqld.X
+# Ref: https://bugs.launchpad.net/codership-mysql/+bug/1378355
+
+readonly INNOBACKUP_OPT_DEFAULT="${$WSREP_SST_OPT_DEFAULT} ${WSREP_SST_OPT_SUFFIX_DEFAULT/-suffix=/=mysqld}"
+
+INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} ${INNOAPPLY}"
+INNOMOVE="${INNOBACKUPEX_BIN} ${INNOBACKUP_OPT_DEFAULT} $disver $impts  --move-back --force-non-empty-directories \${DATA} ${INNOMOVE}"
+INNOBACKUP="${INNOBACKUPEX_BIN} ${INNOBACKUP_OPT_DEFAULT} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir ${INNOBACKUP}"
+
 if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
 then
     trap cleanup_donor EXIT
@@ -1168,7 +1179,6 @@ then
 
         fi
 
-
         wsrep_log_info "Preparing the backup at ${DATA}"
         timeit "Xtrabackup prepare stage" "$INNOAPPLY"
 
diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh
index 287faf6b9de..22ee7b7658b 100644
--- a/scripts/wsrep_sst_xtrabackup.sh
+++ b/scripts/wsrep_sst_xtrabackup.sh
@@ -99,7 +99,7 @@ get_keys()
     fi
 
     if [[ $encrypt -eq 0 ]];then 
-        if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q encrypt;then
+        if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
             wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
         fi
         return
@@ -194,22 +194,11 @@ get_transfer()
 
 }
 
-parse_cnf()
-{
-    local group=$1
-    local var=$2
-    reval=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2-)
-    if [[ -z $reval ]];then 
-        [[ -n $3 ]] && reval=$3
-    fi
-    echo $reval
-}
-
 get_footprint()
 {
     pushd $WSREP_SST_OPT_DATA 1>/dev/null
     payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c | awk 'END { print $1 }')
-    if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
+    if $MY_PRINT_DEFAULTS xtrabackup | grep -q -- "--compress";then 
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
         payload=$(( payload*1/2 ))
@@ -388,8 +377,8 @@ check_extra()
 {
     local use_socket=1
     if [[ $uextra -eq 1 ]];then 
-        if $MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
-            local eport=$($MY_PRINT_DEFAULTS -c $WSREP_SST_OPT_CONF mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
+        if $MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--thread-handling=" | grep -q 'pool-of-threads';then 
+            local eport=$($MY_PRINT_DEFAULTS --mysqld | tr '_' '-' | grep -- "--extra-port=" | cut -d= -f2)
             if [[ -n $eport ]];then 
                 # Xtrabackup works only locally.
                 # Hence, setting host to 127.0.0.1 unconditionally. 
@@ -426,9 +415,16 @@ setup_ports
 get_stream
 get_transfer
 
+# This is as close as innobackupex gets to mysqld defaults. Important note --defaults-extra-file passed to
+# mysqld is not passed to innobackupex.
+# WSREP_SST_OPT_SUFFIX_DEFAULT goes from --defaults-group-suffix=.X to --defaults-group=mysqld.X
+# Ref: https://bugs.launchpad.net/codership-mysql/+bug/1378355
+
+readonly INNOBACKUP_OPT_DEFAULT="${$WSREP_SST_OPT_DEFAULT} ${WSREP_SST_OPT_SUFFIX_DEFAULT/-suffix=/=mysqld}"
+
 INNOEXTRA=""
-INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
-INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log"
+INNOAPPLY="${INNOBACKUPEX_BIN} ${INNOBACKUP_OPT_DEFAULT} --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} ${INNOBACKUP_OPT_DEFAULT} \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log"
 
 if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
 then
@@ -533,7 +529,7 @@ then
 
     if [[ $incremental -eq 1 ]];then 
         wsrep_log_info "Incremental SST enabled"
-        #lsn=$(/pxc/bin/mysqld --defaults-file=$WSREP_SST_OPT_CONF  --basedir=/pxc  --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1)
+        #lsn=$(/pxc/bin/mysqld $WSREP_SST_OPT_CONF  --basedir=/pxc  --wsrep-recover 2>&1 | grep -o 'log sequence number .*' | cut -d " " -f 4 | head -1)
         lsn=$(grep to_lsn xtrabackup_checkpoints | cut -d= -f2 | tr -d ' ')
         wsrep_log_info "Recovered LSN: $lsn"
     fi
@@ -686,7 +682,7 @@ then
 
         if [[ $incremental -eq 1 ]];then 
             # Added --ibbackup=xtrabackup_55 because it fails otherwise citing connection issues.
-            INNOAPPLY="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} \
+            INNOAPPLY="${INNOBACKUPEX_BIN} ${INNOBACKUP_OPT_DEFAULT} \
                 --ibbackup=xtrabackup_55 --apply-log $rebuildcmd --redo-only $BDATA --incremental-dir=${DATA} &>>${BDATA}/innobackup.prepare.log"
         fi