MDEV-26391 BF abortable mariabackup execution

This commit changes backup execution (namely the block ddl phase),
so that node is not paused from cluster. Instead, the following
backup execution is declared as vulnerable for possible cluster
level conflicts, especially with DDL statement applying.
With this, the mariabackup execution may be aborted, if DDL
statements happen during backup execution. This abortable
backup execution is optional feature and may be
enabled/disabled by wsrep_mode: BF_ABORT_MARIABACKUP.
Note that old style node desync and pause, despite of
WSREP_MODE_BF_MARIABACKUP is needed if node is operating as
SST donor.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

10 files changed, 53 insertions, 15 deletions

diff --git a/mysql-test/suite/galera/r/galera_var_wsrep_mode.result b/mysql-test/suite/galera/r/galera_var_wsrep_mode.result
index 8682ad685a5..b5b33fe8ae5 100644
--- a/mysql-test/suite/galera/r/galera_var_wsrep_mode.result
+++ b/mysql-test/suite/galera/r/galera_var_wsrep_mode.result
@@ -20,8 +20,8 @@ SET GLOBAL wsrep_mode='A';
 ERROR 42000: Variable 'wsrep_mode' can't be set to the value of 'A'
 SET GLOBAL wsrep_mode=NULL;
 ERROR 42000: Variable 'wsrep_mode' can't be set to the value of 'NULL'
-SET GLOBAL wsrep_mode=64;
-ERROR 42000: Variable 'wsrep_mode' can't be set to the value of '64'
+SET GLOBAL wsrep_mode=128;
+ERROR 42000: Variable 'wsrep_mode' can't be set to the value of '128'
 SET GLOBAL wsrep_mode=REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM;
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1
 SET GLOBAL wsrep_mode=1;
diff --git a/mysql-test/suite/galera/t/galera_var_wsrep_mode.test b/mysql-test/suite/galera/t/galera_var_wsrep_mode.test
index 56953b1e71b..002dce0a05a 100644
--- a/mysql-test/suite/galera/t/galera_var_wsrep_mode.test
+++ b/mysql-test/suite/galera/t/galera_var_wsrep_mode.test
@@ -22,7 +22,7 @@ SET GLOBAL wsrep_mode='A';
 --error ER_WRONG_VALUE_FOR_VAR
 SET GLOBAL wsrep_mode=NULL;
 --error ER_WRONG_VALUE_FOR_VAR
-SET GLOBAL wsrep_mode=64;
+SET GLOBAL wsrep_mode=128;
 --error ER_PARSE_ERROR
 SET GLOBAL wsrep_mode=REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM;
 #
diff --git a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
index 6bea1a88caa..6c584e190f1 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_wsrep.result
@@ -342,7 +342,7 @@ VARIABLE_COMMENT	Set of WSREP features that are enabled.
 NUMERIC_MIN_VALUE	NULL
 NUMERIC_MAX_VALUE	NULL
 NUMERIC_BLOCK_SIZE	NULL
-ENUM_VALUE_LIST	STRICT_REPLICATION,BINLOG_ROW_FORMAT_ONLY,REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM,REPLICATE_ARIA,DISALLOW_LOCAL_GTID
+ENUM_VALUE_LIST	STRICT_REPLICATION,BINLOG_ROW_FORMAT_ONLY,REQUIRED_PRIMARY_KEY,REPLICATE_MYISAM,REPLICATE_ARIA,DISALLOW_LOCAL_GTID,BF_ABORT_MARIABACKUP
 READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 GLOBAL_VALUE_PATH	NULL
diff --git a/sql/backup.cc b/sql/backup.cc
index 0a5cc97f431..5f74c67add7 100644
--- a/sql/backup.cc
+++ b/sql/backup.cc
@@ -35,8 +35,10 @@
 #include "sql_handler.h"                        // mysql_ha_cleanup_no_free
 #include <my_sys.h>
 #include <strfunc.h>                           // strconvert()
+#include "debug_sync.h"
 #ifdef WITH_WSREP
 #include "wsrep_server_state.h"
+#include "wsrep_mysqld.h"
 #endif /* WITH_WSREP */
 
 static const char *stage_names[]=
@@ -291,16 +293,26 @@ static bool backup_block_ddl(THD *thd)
 
 #ifdef WITH_WSREP
   /*
-    We desync the node for BACKUP STAGE because applier threads
+    if user is specifically choosing to allow BF aborting for BACKUP STAGE BLOCK_DDL lock
+    holder, then do not desync and pause the node from cluster replication.
+    e.g. mariabackup uses BACKUP STATE BLOCK_DDL; and will be abortable by this.
+    But, If node is processing as SST donor or WSREP_MODE_BF_MARIABACKUP mode is not set,
+    we desync the node for BACKUP STAGE because applier threads
     bypass backup MDL locks (see MDL_lock::can_grant_lock)
   */
   if (WSREP_NNULL(thd))
   {
     Wsrep_server_state &server_state= Wsrep_server_state::instance();
-    if (server_state.desync_and_pause().is_undefined()) {
-      DBUG_RETURN(1);
+    if (!wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP) ||
+        server_state.state() == Wsrep_server_state::s_donor)
+    {
+      if (server_state.desync_and_pause().is_undefined()) {
+        DBUG_RETURN(1);
+      }
+      thd->wsrep_desynced_backup_stage= true;
     }
-    thd->wsrep_desynced_backup_stage= true;
+    else
+      WSREP_INFO("Server not desynched from group because WSREP_MODE_BF_MARIABACKUP used.");
   }
 #endif /* WITH_WSREP */
 
@@ -340,6 +352,18 @@ static bool backup_block_ddl(THD *thd)
   /* There can't be anything more that needs to be logged to ddl log */
   THD_STAGE_INFO(thd, org_stage);
   stop_ddl_logging();
+#ifdef WITH_WSREP
+  // Allow tests to block the applier thread using the DBUG facilities
+  DBUG_EXECUTE_IF("sync.wsrep_after_mdl_block_ddl",
+                  {
+                   const char act[]=
+                     "now "
+                     "signal signal.wsrep_apply_toi";
+                   DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                      STRING_WITH_LEN(act)));
+                  };);
+#endif /* WITH_WSREP */
+
   DBUG_RETURN(0);
 err:
   THD_STAGE_INFO(thd, org_stage);
@@ -399,7 +423,8 @@ bool backup_end(THD *thd)
     thd->current_backup_stage= BACKUP_FINISHED;
     thd->mdl_context.release_lock(old_ticket);
 #ifdef WITH_WSREP
-    if (WSREP_NNULL(thd) && thd->wsrep_desynced_backup_stage)
+    if (WSREP_NNULL(thd) && thd->wsrep_desynced_backup_stage &&
+	!wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
     {
       Wsrep_server_state &server_state= Wsrep_server_state::instance();
       server_state.resume_and_resync();
diff --git a/sql/mdl.cc b/sql/mdl.cc
index f1c9c6e0444..4795d0bb674 100644
--- a/sql/mdl.cc
+++ b/sql/mdl.cc
@@ -1747,10 +1747,9 @@ MDL_lock::can_grant_lock(enum_mdl_type type_arg,
 #ifdef WITH_WSREP
   /*
     Approve lock request in BACKUP namespace for BF threads.
-    We should get rid of this code and forbid FTWRL/BACKUP statements
-    when wsrep is active.
   */
-  if ((wsrep_thd_is_toi(requestor_ctx->get_thd()) ||
+  if (!wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP) &&
+      (wsrep_thd_is_toi(requestor_ctx->get_thd()) ||
        wsrep_thd_is_applying(requestor_ctx->get_thd())) &&
       key.mdl_namespace() == MDL_key::BACKUP)
   {
diff --git a/sql/service_wsrep.cc b/sql/service_wsrep.cc
index 43183ff7595..f0f9530c80d 100644
--- a/sql/service_wsrep.cc
+++ b/sql/service_wsrep.cc
@@ -241,7 +241,7 @@ extern "C" my_bool wsrep_thd_bf_abort(THD *bf_thd, THD *victim_thd,
     victim_thd->awake_no_mutex(KILL_QUERY);
     mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
   } else {
-    WSREP_DEBUG("wsrep_thd_bf_abort skipped awake");
+    WSREP_DEBUG("wsrep_thd_bf_abort skipped awake, signal %d", signal);
   }
   return ret;
 }
@@ -277,7 +277,6 @@ extern "C" my_bool wsrep_thd_is_aborting(const MYSQL_THD thd)
       return (cs.state() == wsrep::client_state::s_exec ||
               cs.state() == wsrep::client_state::s_result);
     case wsrep::transaction::s_aborting:
-    case wsrep::transaction::s_aborted:
       return true;
     default:
       return false;
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index f844c8c6912..ffa301df12b 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -6013,6 +6013,7 @@ static const char *wsrep_mode_names[]=
   "REPLICATE_MYISAM",
   "REPLICATE_ARIA",
   "DISALLOW_LOCAL_GTID",
+  "BF_ABORT_MARIABACKUP",
   NullS
 };
 static Sys_var_set Sys_wsrep_mode(
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index 7d684cef35d..b5c5e98d0d8 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -3053,6 +3053,11 @@ void wsrep_handle_mdl_conflict(MDL_context *requestor_ctx,
       WSREP_DEBUG("BF thread waiting for FLUSH");
       ticket->wsrep_report(wsrep_debug);
       mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
+      if (granted_thd->current_backup_stage != BACKUP_FINISHED &&
+	  wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
+      {
+	wsrep_abort_thd(request_thd, granted_thd, 1);
+      }
     }
     else if (request_thd->lex->sql_command == SQLCOM_DROP_TABLE)
     {
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
index 6f9a2d127b4..c9e08d77b6c 100644
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@ -130,7 +130,8 @@ enum enum_wsrep_mode {
   WSREP_MODE_REQUIRED_PRIMARY_KEY= (1ULL << 2),
   WSREP_MODE_REPLICATE_MYISAM= (1ULL << 3),
   WSREP_MODE_REPLICATE_ARIA= (1ULL << 4),
-  WSREP_MODE_DISALLOW_LOCAL_GTID= (1ULL << 5)
+  WSREP_MODE_DISALLOW_LOCAL_GTID= (1ULL << 5),
+  WSREP_MODE_BF_MARIABACKUP= (1ULL << 6)
 };
 
 // Streaming Replication
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index e610d3a6c2b..65b537d8b84 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -375,6 +375,14 @@ bool wsrep_bf_abort(THD* bf_thd, THD* victim_thd)
        have acquired MDL locks (due to DDL execution), and this has caused BF conflict.
        such case does not require aborting in wsrep or replication provider state.
     */
+    if (victim_thd->current_backup_stage != BACKUP_FINISHED &&
+        wsrep_check_mode(WSREP_MODE_BF_MARIABACKUP))
+    {
+      WSREP_DEBUG("killing connection for non wsrep session");
+      mysql_mutex_lock(&victim_thd->LOCK_thd_data);
+      victim_thd->awake_no_mutex(KILL_CONNECTION);
+      mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
+    }
     return false;
   }