MDEV-26971: Implement progress reporting by mariabackup SST script

Currently covers network transfer stage on donor and joiner.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

2 files changed, 113 insertions, 52 deletions

diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index ce4001fdc56..bfb42d53b1c 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -40,7 +40,7 @@ tcert=""
 tcap=""
 tpem=""
 tkey=""
-tmode=""
+tmode="DISABLED"
 sockopt=""
 progress=""
 ttime=0
@@ -85,16 +85,14 @@ backup_threads=""
 encrypt_threads=""
 encrypt_chunk=""
 
-readonly SECRET_TAG='secret'
+readonly SECRET_TAG="secret"
+readonly TOTAL_TAG="total"
+readonly COMPLETE_TAG="complete"
 
 # Required for backup locks
 # For backup locks it is 1 sent by joiner
 sst_ver=1
 
-if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then
-    pvopts="$pvopts $pvformat"
-fi
-pcmd="pv $pvopts"
 declare -a RC
 
 BACKUP_BIN=$(commandex 'mariabackup')
@@ -104,14 +102,17 @@ if [ -z "$BACKUP_BIN" ]; then
 fi
 
 DATA="$WSREP_SST_OPT_DATA"
-INFO_FILE='xtrabackup_galera_info'
-IST_FILE='xtrabackup_ist'
+INFO_FILE="xtrabackup_galera_info"
+IST_FILE="xtrabackup_ist"
 MAGIC_FILE="$DATA/$INFO_FILE"
 
 INNOAPPLYLOG="$DATA/mariabackup.prepare.log"
 INNOMOVELOG="$DATA/mariabackup.move.log"
 INNOBACKUPLOG="$DATA/mariabackup.backup.log"
 
+# Setting the path for ss and ip
+export PATH="/usr/sbin:/sbin:$PATH"
+
 timeit()
 {
     local stage="$1"
@@ -121,18 +122,19 @@ timeit()
 
     if [ $ttime -eq 1 ]; then
         x1=$(date +%s)
-        wsrep_log_info "Evaluating $cmd"
-        eval "$cmd"
-        extcode=$?
+    fi
+
+    wsrep_log_info "Evaluating $cmd"
+    eval $cmd
+    extcode=$?
+
+    if [ $ttime -eq 1 ]; then
         x2=$(date +%s)
         took=$(( x2-x1 ))
         wsrep_log_info "NOTE: $stage took $took seconds"
         totime=$(( totime+took ))
-    else
-        wsrep_log_info "Evaluating $cmd"
-        eval "$cmd"
-        extcode=$?
     fi
+
     return $extcode
 }
 
@@ -422,16 +424,33 @@ get_transfer()
 get_footprint()
 {
     pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
-    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \
-              -type f -print0 | du --files0-from=- --block-size=1 -c -s | \
-              awk 'END { print $1 }')
+    local payload_data=$(find . \
+        -regex '.*undo[0-9]+$\|.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' \
+        -type f -print0 | du --files0-from=- --block-size=1 -c -s | \
+        awk 'END { print $1 }')
+    popd 1>/dev/null
+
+    ib_undo_dir=$(parse_cnf 'mysqld' 'innodb-undo-directory' "")
+    if [ -n "$ib_undo_dir" -a -d "$ib_undo_dir" ]; then
+        pushd "$ib_undo_dir" 1>/dev/null
+        local payload_undo=$(find . -regex '.*undo[0-9]+$' -type f -print0 | \
+            du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
+        popd 1>/dev/null
+    else
+        local payload_undo=0
+    fi
+    wsrep_log_info "SST footprint estimate: data: $payload_data, undo: $payload_undo"
+
+    payload=$(( $payload_data + $payload_undo ))
+
     if [ "$compress" != 'none' ]; then
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
         payload=$(( payload*1/2 ))
     fi
-    popd 1>/dev/null
-    pcmd="$pcmd -s $payload"
+
+    # report to parent the total footprint of the SST
+    echo "$TOTAL_TAG $payload"
     adjust_progress
 }
 
@@ -446,20 +465,42 @@ adjust_progress()
         return
     fi
 
-    if [ -n "$progress" -a "$progress" != '1' ]; then
-        if [ -e "$progress" ]; then
-            pcmd="$pcmd 2>>'$progress'"
-        else
-            pcmd="$pcmd 2>'$progress'"
-        fi
-    elif [ -z "$progress" -a -n "$rlimit" ]; then
-            # When rlimit is non-zero
-            pcmd='pv -q'
-    fi
-
     if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
         wsrep_log_info "Rate-limiting SST to $rlimit"
-        pcmd="$pcmd -L \$rlimit"
+        rlimitopts=" -L $rlimit"
+    else
+        rlimitopts=""
+    fi
+
+    if [ -n "$progress" ]; then
+        # Backward compatibility: user configured progress output
+        if pv --help | grep -qw -- '-F'; then
+            pvopts="$pvopts $pvformat"
+        fi
+
+        if [ $payload -ne 0 ]; then
+            pvopts="$pvopts -s $payload"
+        fi
+
+        pcmd="pv $pvopts$rlimitopts"
+
+        if [ "$progress" != '1' ]; then
+            if [ -e "$progress" ]; then
+                pcmd="$pcmd 2>>'$progress'"
+            else
+                pcmd="$pcmd 2>'$progress'"
+            fi
+        fi
+
+        rcmd=":"
+    else
+        # Default progress output parseable by parent
+        pvopts="-f -i 1 -n -b$rlimitopts"
+        pcmd="pv $pvopts"
+
+        # read progress data, add tag and post to stdout
+        # for the parent
+        rcmd="stdbuf -oL tr '\r' '\n' | xargs -n1 echo complete"
     fi
 }
 
@@ -780,8 +821,16 @@ recv_joiner()
             exit 32
         fi
 
-        # remove secret from the magic file
-        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+        # check total SST footprint
+        total=$(grep -F -- "$TOTAL_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+        if [ $total -ge 0 ]; then
+            # report to parent
+            echo "$TOTAL_TAG $total"
+        fi
+
+        # remove secret and total from the magic file
+        grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" | \
+        grep -v -F -- "$TOTAL_TAG " > "$MAGIC_FILE.new"
         mv "$MAGIC_FILE.new" "$MAGIC_FILE"
     fi
 }
@@ -984,12 +1033,19 @@ then
 
         check_extra
 
+        wsrep_log_info "Estimating total transfer size"
+        get_footprint
+        wsrep_log_info "To transfer: $payload"
+
         wsrep_log_info "Streaming GTID file before SST"
 
         # Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
         # (separated by a space).
         echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
 
+        # Tell joiner what to expect:
+        echo "$TOTAL_TAG $payload" >> "$MAGIC_FILE"
+
         if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
             # Let joiner know that we know its secret
             echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
@@ -1011,12 +1067,14 @@ then
         # Restore the transport commmand to its original state
         tcmd="$ttcmd"
 
-        if [ -n "$progress" ]; then
-            get_footprint
-            tcmd="$pcmd | $tcmd"
-        elif [ -n "$rlimit" ]; then
-            adjust_progress
-            tcmd="$pcmd | $tcmd"
+        if [ -n "$pcmd" ]; then
+            if [ "$rcmd" != ":" ]; then
+                # redirect pv stderr to rcmd for tagging and output to parent
+                tcmd="{ $pcmd 2>&3 | $tcmd; } 3>&1 | $rcmd"
+            else
+                # use user-configured pv output
+                tcmd="$pcmd | $tcmd"
+            fi
         fi
 
         wsrep_log_info "Sleeping before data transfer for SST"
@@ -1220,11 +1278,6 @@ then
     trap sig_joiner_cleanup HUP PIPE INT TERM
     trap cleanup_at_exit EXIT
 
-    if [ -n "$progress" ]; then
-        adjust_progress
-        tcmd="$tcmd | $pcmd"
-    fi
-
     get_keys
     if [ $encrypt -eq 1 ]; then
         strmcmd="$ecmd | $strmcmd"
@@ -1234,6 +1287,17 @@ then
         strmcmd="$sdecomp | $strmcmd"
     fi
 
+    adjust_progress
+    if [ -n "$pcmd" ]; then
+        if [ "$rcmd" != ":" ]; then
+            # redirect pv stderr to rcmd for tagging and output to parent
+            strmcmd="{ $pcmd 2>&3 | $strmcmd; } 3>&1 | $rcmd"
+        else
+            # use user-configured pv output
+            strmcmd="$pcmd | $strmcmd"
+        fi
+    fi
+
     check_sockets_utils
 
     STATDIR="$(mktemp -d)"
@@ -1340,13 +1404,11 @@ then
 
             dcmd="xargs -n 2 qpress -dT$nproc"
 
-            if [ -n "$progress" ] && \
-               pv --help | grep -qw -F -- '--line-mode'
-            then
+            if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
                 count=$(find "$DATA" -type f -name '*.qp' | wc -l)
                 count=$(( count*2 ))
                 pvopts="-f -s $count -l -N Decompression"
-                if pv --help | grep -qw -F -- '-F'; then
+                if pv --help | grep -qw -- '-F'; then
                     pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
                 fi
                 pcmd="pv $pvopts"
diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc
index 68a4dd2bb57..3062890a8e4 100644
--- a/sql/wsrep_sst.cc
+++ b/sql/wsrep_sst.cc
@@ -630,14 +630,13 @@ static void* sst_joiner_thread (void* a)
     wsrep_uuid_t  ret_uuid = WSREP_UUID_UNDEFINED;
     wsrep_seqno_t ret_seqno= WSREP_SEQNO_UNDEFINED;
 
-    // current state progress
+    // current stage progress
     long long total= 0;
     long long complete= 0;
     // previous stages cumulative progress
     long long total_prev= 0;
 
-    // in case of successfull receiver start, wait for SST
-    // completion/end
+    // in case of successful receiver start, wait for SST completion/end
     const char* tmp= NULL;
     err= EINVAL;