Merge mariadb-10.2.41 into 10.2

23 files changed, 477 insertions, 277 deletions

diff --git a/mysql-test/suite/galera/r/galera_UK_conflict.result b/mysql-test/suite/galera/r/galera_UK_conflict.result
index 76649f1b268..2795a86d6a6 100644
--- a/mysql-test/suite/galera/r/galera_UK_conflict.result
+++ b/mysql-test/suite/galera/r/galera_UK_conflict.result
@@ -68,6 +68,9 @@ f1	f2	f3
 10	10	0
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -78,6 +81,9 @@ f1	f2	f3
 8	8	8
 10	10	0
 connection node_1;
+SELECT COUNT(*) FROM t1;
+COUNT(*)
+7
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
@@ -85,5 +91,6 @@ f1	f2	f3
 4	4	2
 5	5	2
 7	7	7
+8	8	8
 10	10	0
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/r/galera_ssl_upgrade.result b/mysql-test/suite/galera/r/galera_ssl_upgrade.result
index 1443e34d041..32b5cf10757 100644
--- a/mysql-test/suite/galera/r/galera_ssl_upgrade.result
+++ b/mysql-test/suite/galera/r/galera_ssl_upgrade.result
@@ -1,7 +1,7 @@
 connection node_1;
-call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown.*");
+call mtr.add_suppression("WSREP: write_handler().*");
 connection node_2;
-call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown.*");
+call mtr.add_suppression("WSREP: write_handler():.*");
 connection node_1;
 connection node_2;
 connection node_1;
@@ -24,5 +24,7 @@ connection node_1;
 SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 VARIABLE_VALUE = 2
 1
-disconnect node_2;
-disconnect node_1;
+connection node_1;
+call mtr.add_suppression("WSREP: write_handler().*");
+connection node_2;
+call mtr.add_suppression("WSREP: write_handler():.*");
diff --git a/mysql-test/suite/galera/r/galera_to_error.result b/mysql-test/suite/galera/r/galera_to_error.result
new file mode 100644
index 00000000000..545029bf069
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_to_error.result
@@ -0,0 +1,30 @@
+connection node_1;
+connection node_2;
+connection node_1;
+call mtr.add_suppression("WSREP: TO isolation failed for: ");
+CREATE TABLE t2(a int not null auto_increment primary key, b int, key(b)) engine=innodb;
+INSERT INTO t2 values (NULL,1),(NULL,2),(NULL,3),(NULL,4),(NULL,5),(NULL,6);
+connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1a;
+BEGIN;
+UPDATE t2 set b = b + 20 where b BETWEEN 2 and 5;;
+connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1b;
+connection node_2;
+Killing server ...
+connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1;
+connection node_1c;
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=InnoDB;
+Got one of the listed errors
+Got one of the listed errors
+CREATE UNIQUE INDEX b2 ON t2(b);
+Got one of the listed errors
+connection node_1;
+disconnect node_1a;
+disconnect node_1b;
+disconnect node_1c;
+connection node_2;
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+DROP TABLE t2;
+disconnect node_2;
+disconnect node_1;
diff --git a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
index 94752ed7c76..a972394f87c 100644
--- a/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
+++ b/mysql-test/suite/galera/r/galera_toi_ddl_fk_insert.result
@@ -21,22 +21,6 @@ connection node_1a;
 connection node_1b;
 connection node_2;
 connection node_2a;
-connection node_1;
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-COUNT(*)
-20001
-SELECT COUNT(*) FROM child;
-COUNT(*)
-10000
-connection node_2;
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-COUNT(*)
-20001
-SELECT COUNT(*) FROM child;
-COUNT(*)
-10000
 DROP TABLE child;
 DROP TABLE parent;
 DROP TABLE ten;
diff --git a/mysql-test/suite/galera/t/galera_UK_conflict.test b/mysql-test/suite/galera/t/galera_UK_conflict.test
index 57bafbf8ae0..83d0e47dc3d 100644
--- a/mysql-test/suite/galera/t/galera_UK_conflict.test
+++ b/mysql-test/suite/galera/t/galera_UK_conflict.test
@@ -140,9 +140,13 @@ SELECT * FROM t1;
 # original state in node 1
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
+SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 --connection node_1
+--let $wait_condition = SELECT COUNT(*) = 7 FROM t1
+--source include/wait_condition.inc
+SELECT COUNT(*) FROM t1;
 SELECT * FROM t1;
 
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_ssl_upgrade.test b/mysql-test/suite/galera/t/galera_ssl_upgrade.test
index 4d6b9159c78..1098e84488f 100644
--- a/mysql-test/suite/galera/t/galera_ssl_upgrade.test
+++ b/mysql-test/suite/galera/t/galera_ssl_upgrade.test
@@ -9,9 +9,9 @@
 --source include/have_ssl_communication.inc
 
 --connection node_1
-call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown.*");
+call mtr.add_suppression("WSREP: write_handler().*");
 --connection node_2
-call mtr.add_suppression("WSREP: write_handler(): protocol is shutdown.*");
+call mtr.add_suppression("WSREP: write_handler():.*");
 
 # Save original auto_increment_offset values.
 --let $node_1=node_1
@@ -72,4 +72,8 @@ SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_N
 # Restore original auto_increment_offset values.
 --source include/auto_increment_offset_restore.inc
 
---source include/galera_end.inc
+--connection node_1
+call mtr.add_suppression("WSREP: write_handler().*");
+--connection node_2
+call mtr.add_suppression("WSREP: write_handler():.*");
+
diff --git a/mysql-test/suite/galera/t/galera_to_error.test b/mysql-test/suite/galera/t/galera_to_error.test
new file mode 100644
index 00000000000..576fa01e64d
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_to_error.test
@@ -0,0 +1,71 @@
+#
+# Confirm that with two nodes, killing one causes the other to stop accepting connections
+#
+
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--connection node_1
+call mtr.add_suppression("WSREP: TO isolation failed for: ");
+--let $wsrep_cluster_address_orig = `SELECT @@wsrep_cluster_address`
+
+CREATE TABLE t2(a int not null auto_increment primary key, b int, key(b)) engine=innodb;
+INSERT INTO t2 values (NULL,1),(NULL,2),(NULL,3),(NULL,4),(NULL,5),(NULL,6);
+
+--connect node_1a, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connection node_1a
+BEGIN;
+--send UPDATE t2 set b = b + 20 where b BETWEEN 2 and 5;
+
+#
+# Take thread id for above query
+#
+--connect node_1b, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connection node_1b
+--let $k_thread = `SELECT ID FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'root' AND COMMAND = 'Sleep' LIMIT 1`
+
+--connection node_2
+--source include/kill_galera.inc
+
+--connect node_1c, 127.0.0.1, root, , test, $NODE_MYPORT_1
+--connection node_1c
+--error ER_LOCK_DEADLOCK,ER_UNKNOWN_COM_ERROR
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) ENGINE=InnoDB;
+
+--disable_query_log
+--error ER_LOCK_DEADLOCK, ER_CANNOT_USER,ER_KILL_QUERY_DENIED_ERROR
+--eval KILL QUERY $k_thread;
+--enable_query_log
+
+# Reset the master and restart the slave so that post-test checks can run
+--error ER_LOCK_DEADLOCK,ER_UNKNOWN_COM_ERROR
+CREATE UNIQUE INDEX b2 ON t2(b);
+
+--connection node_1
+--disconnect node_1a
+--disconnect node_1b
+--disconnect node_1c
+
+--connection node_2
+--source include/start_mysqld.inc
+--source include/wait_until_connected_again.inc
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--source include/wait_until_connected_again.inc
+
+DROP TABLE t2;
+
+# Restore original auto_increment_offset values.
+--let $node_2=node_2a
+--source include/auto_increment_offset_restore.inc
+
+--source include/galera_end.inc
+
diff --git a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
index fadc94d78ff..3b4b427f551 100644
--- a/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
+++ b/mysql-test/suite/galera/t/galera_toi_ddl_fk_insert.test
@@ -54,15 +54,11 @@ INSERT INTO parent VALUES (1, 0);
 --connection node_2a
 --reap
 
---connection node_1
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-SELECT COUNT(*) FROM child;
-
---connection node_2
-SET SESSION wsrep_sync_wait=15;
-SELECT COUNT(*) FROM parent;
-SELECT COUNT(*) FROM child;
+#
+# ALTER TABLE could bf kill one or more of INSERTs to parent, so
+# the actual number of rows in PARENT depends on whether
+# the INSERT is committed before ALTER TABLE is executed
+#
 
 DROP TABLE child;
 DROP TABLE parent;
diff --git a/mysql-test/suite/galera/t/galera_var_auto_inc_control_off.test b/mysql-test/suite/galera/t/galera_var_auto_inc_control_off.test
index c0bbe5af8cf..241b62dbf8c 100644
--- a/mysql-test/suite/galera/t/galera_var_auto_inc_control_off.test
+++ b/mysql-test/suite/galera/t/galera_var_auto_inc_control_off.test
@@ -94,11 +94,13 @@ SELECT * FROM t1;
 --eval SET GLOBAL wsrep_auto_increment_control = $auto_increment_control_orig
 --eval SET GLOBAL auto_increment_increment = $auto_increment_increment_node1
 --eval SET GLOBAL auto_increment_offset = $auto_increment_offset_node1
+--disconnect node_1a
 
 --connection node_2
 --eval SET GLOBAL wsrep_auto_increment_control = $auto_increment_control_orig
 --eval SET GLOBAL auto_increment_increment = $auto_increment_increment_node2
 --eval SET GLOBAL auto_increment_offset = $auto_increment_offset_node2
+--disconnect node_2a
 
 --enable_query_log
 
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
index 0cf13650ce0..875315c0e7c 100644
--- a/mysql-test/suite/wsrep/t/variables.test
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -66,7 +66,7 @@ call mtr.add_suppression("WSREP: Failed to get provider options");
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_slave_threads;
 SELECT @@global.wsrep_cluster_address;
@@ -77,7 +77,7 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 
 #evalp SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
@@ -101,7 +101,7 @@ SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VA
 SELECT VARIABLE_VALUE AS EXPECT_1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_rollbacker_thread_count';
 SELECT VARIABLE_VALUE AS EXPECT_2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
 
---replace_regex /.*libgalera_smm.*/libgalera_smm.so/
+--replace_regex /.*libgalera.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_cluster_address;
 SELECT @@global.wsrep_on;
diff --git a/sql/sql_class.cc b/sql/sql_class.cc
index 5ada018e540..2eec056ec9d 100644
--- a/sql/sql_class.cc
+++ b/sql/sql_class.cc
@@ -1863,6 +1863,7 @@ bool THD::notify_shared_lock(MDL_context_owner *ctx_in_use,
 
   if (needs_thr_lock_abort)
   {
+    bool mutex_released= false;
     mysql_mutex_lock(&in_use->LOCK_thd_data);
     /* If not already dying */
     if (in_use->killed != KILL_CONNECTION_HARD)
@@ -1879,18 +1880,21 @@ bool THD::notify_shared_lock(MDL_context_owner *ctx_in_use,
           thread can see those instances (e.g. see partitioning code).
         */
         if (!thd_table->needs_reopen())
-        {
           signalled|= mysql_lock_abort_for_thread(this, thd_table);
-          if (WSREP(this) && wsrep_thd_is_BF(this, FALSE))
-          {
-            WSREP_DEBUG("remove_table_from_cache: %llu",
-                        (unsigned long long) this->real_id);
-            wsrep_abort_thd((void *)this, (void *)in_use, FALSE);
-          }
-        }
       }
+#ifdef WITH_WSREP
+      if (WSREP(this) && wsrep_thd_is_BF(this, false))
+      {
+        WSREP_DEBUG("notify_shared_lock: BF thread %llu query %s"
+                    " victim %llu query %s",
+                    this->real_id, wsrep_thd_query(this),
+                    in_use->real_id, wsrep_thd_query(in_use));
+        wsrep_abort_thd((void *)this, (void *)in_use, false);
+        mutex_released= true;
+      }
+#endif /* WITH_WSREP */
     }
-    mysql_mutex_unlock(&in_use->LOCK_thd_data);
+    if (!mutex_released) mysql_mutex_unlock(&in_use->LOCK_thd_data);
   }
   DBUG_RETURN(signalled);
 }
diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc
index 3e1f248b082..67e2fcd6764 100644
--- a/sql/sql_parse.cc
+++ b/sql/sql_parse.cc
@@ -1,5 +1,5 @@
 /* Copyright (c) 2000, 2017, Oracle and/or its affiliates.
-   Copyright (c) 2008, 2020, MariaDB
+   Copyright (c) 2008, 2021, MariaDB
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -9069,6 +9069,18 @@ static
 void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
 {
   uint error;
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    WSREP_DEBUG("sql_kill called");
+    if (thd->wsrep_applier)
+    {
+      WSREP_DEBUG("KILL in applying, bailing out here");
+      return;
+    }
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
+  }
+#endif /* WITH_WSREP */
   if (!(error= kill_one_thread(thd, id, state, type)))
   {
     if (!thd->killed)
@@ -9078,6 +9090,13 @@ void sql_kill(THD *thd, longlong id, killed_state state, killed_type type)
   }
   else
     my_error(error, MYF(0), id);
+#ifdef WITH_WSREP
+  return;
+ wsrep_error_label:
+  error= (type == KILL_TYPE_QUERY ? ER_KILL_QUERY_DENIED_ERROR :
+                                    ER_KILL_DENIED_ERROR);
+  my_error(error, MYF(0), id);
+#endif /* WITH_WSREP */
 }
 
 
@@ -9086,6 +9105,18 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
 {
   uint error;
   ha_rows rows;
+#ifdef WITH_WSREP
+  if (WSREP(thd))
+  {
+    WSREP_DEBUG("sql_kill_user called");
+    if (thd->wsrep_applier)
+    {
+      WSREP_DEBUG("KILL in applying, bailing out here");
+      return;
+    }
+    WSREP_TO_ISOLATION_BEGIN(WSREP_MYSQL_DB, NULL, NULL)
+  }
+#endif /* WITH_WSREP */
   if (!(error= kill_threads_for_user(thd, user, state, &rows)))
     my_ok(thd, rows);
   else
@@ -9096,6 +9127,11 @@ void sql_kill_user(THD *thd, LEX_USER *user, killed_state state)
     */
     my_error(error, MYF(0), user->host.str, user->user.str);
   }
+#ifdef WITH_WSREP
+  return;
+ wsrep_error_label:
+  my_error(ER_CANNOT_USER, MYF(0), user ? user->user.str : "NULL");
+#endif /* WITH_WSREP */
 }
 
 
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index f22d8bf0f5a..4a99f781fdd 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -1,4 +1,4 @@
-/* Copyright 2008-2015 Codership Oy <http://www.codership.com>
+/* Copyright 2008-2021 Codership Oy <http://www.codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -835,13 +835,25 @@ void wsrep_thr_init()
   DBUG_VOID_RETURN;
 }
 
+/* This is wrapper for wsrep_break_lock in thr_lock.c */
+static int wsrep_thr_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
+{
+  THD* victim_thd= (THD *) victim_thd_ptr;
+  /* We need to lock THD::LOCK_thd_data to protect victim
+  from concurrent usage or disconnect or delete. */
+  mysql_mutex_lock(&victim_thd->LOCK_thd_data);
+  int res= wsrep_abort_thd(bf_thd_ptr, victim_thd_ptr, signal);
+  return res;
+}
+
+
 void wsrep_init_startup (bool first)
 {
   if (wsrep_init()) unireg_abort(1);
 
   wsrep_thr_lock_init(
      (wsrep_thd_is_brute_force_fun)wsrep_thd_is_BF,
-     (wsrep_abort_thd_fun)wsrep_abort_thd,
+     (wsrep_abort_thd_fun)wsrep_thr_abort_thd,
      wsrep_debug, wsrep_convert_LOCK_to_trx,
      (wsrep_on_fun)wsrep_on);
 
@@ -1694,6 +1706,11 @@ static int wsrep_TOI_begin(THD *thd, char *db_, char *table_,
   case SQLCOM_DROP_TABLE:
     buf_err= wsrep_drop_table_query(thd, &buf, &buf_len);
     break;
+  case SQLCOM_KILL:
+    WSREP_DEBUG("KILL as TOI: %s", thd->query());
+    buf_err= wsrep_to_buf_helper(thd, thd->query(), thd->query_length(),
+                                 &buf, &buf_len);
+    break;
   case SQLCOM_CREATE_ROLE:
     if (sp_process_definer(thd))
     {
@@ -2058,8 +2075,13 @@ bool wsrep_grant_mdl_exception(MDL_context *requestor_ctx,
         ticket->wsrep_report(true);
       }
 
-      mysql_mutex_unlock(&granted_thd->LOCK_thd_data);
-      wsrep_abort_thd((void *) request_thd, (void *) granted_thd, 1);
+      /* This will call wsrep_abort_transaction so we should hold
+      THD::LOCK_thd_data to protect victim from concurrent usage
+      or disconnect or delete. */
+      if (request_thd->wsrep_exec_mode == REPL_RECV)
+	DEBUG_SYNC(request_thd, "wsrep_after_granted_lock");
+
+      wsrep_abort_thd((void *) request_thd, (void *) granted_thd, true);
       ret= false;
     }
   }
@@ -2241,6 +2263,7 @@ error:
 static bool abort_replicated(THD *thd)
 {
   bool ret_code= false;
+  mysql_mutex_lock(&thd->LOCK_thd_data);
   if (thd->wsrep_query_state== QUERY_COMMITTING)
   {
     WSREP_DEBUG("aborting replicated trx: %llu", (ulonglong)(thd->real_id));
@@ -2248,6 +2271,8 @@ static bool abort_replicated(THD *thd)
     (void)wsrep_abort_thd(thd, thd, TRUE);
     ret_code= true;
   }
+  else
+    mysql_mutex_unlock(&thd->LOCK_thd_data);
   return ret_code;
 }
 
@@ -2294,6 +2319,8 @@ static bool have_client_connections()
                        (longlong) tmp->thread_id));
     if (is_client_connection(tmp) && tmp->killed == KILL_CONNECTION)
     {
+      WSREP_DEBUG("Informing thread %lld that it's time to die",
+                  (longlong)tmp->thread_id);
       (void)abort_replicated(tmp);
       return true;
     }
@@ -2378,6 +2405,8 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   {
     DBUG_PRINT("quit",("Informing thread %lld that it's time to die",
                        (longlong) tmp->thread_id));
+    WSREP_DEBUG("Informing thread %lld that it's time to die",
+                (longlong)tmp->thread_id);
     /* We skip slave threads & scheduler on this first loop through. */
     if (!is_client_connection(tmp))
       continue;
@@ -2394,15 +2423,18 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
       continue;
     }
 
-    /* replicated transactions must be skipped */
+    /* replicated transactions must be skipped and aborted
+    with wsrep_abort_thd. */
     if (abort_replicated(tmp))
       continue;
 
     WSREP_DEBUG("closing connection %lld", (longlong) tmp->thread_id);
 
     /*
-      instead of wsrep_close_thread() we do now  soft kill by THD::awake
-     */
+      instead of wsrep_close_thread() we do now  soft kill by
+      THD::awake(). Here also victim needs to be protected from
+      concurrent usage or disconnect or delete.
+    */
     mysql_mutex_lock(&tmp->LOCK_thd_data);
 
     tmp->awake(KILL_CONNECTION);
@@ -2423,7 +2455,6 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
   I_List_iterator<THD> it2(threads);
   while ((tmp=it2++))
   {
-#ifndef __bsdi__				// Bug in BSDI kernel
     if (is_client_connection(tmp) &&
         !abort_replicated(tmp)    &&
         !is_replaying_connection(tmp) &&
@@ -2432,7 +2463,6 @@ void wsrep_close_client_connections(my_bool wait_to_end, THD *except_caller_thd)
       WSREP_INFO("killing local connection: %lld", (longlong) tmp->thread_id);
       close_connection(tmp,0);
     }
-#endif
   }
 
   DBUG_PRINT("quit",("Waiting for threads to die (count=%u)",thread_count));
@@ -2621,7 +2651,8 @@ extern "C" void wsrep_thd_set_query_state(
 
 void wsrep_thd_set_conflict_state(THD *thd, enum wsrep_conflict_state state)
 {
-  if (WSREP(thd)) thd->wsrep_conflict_state= state;
+  mysql_mutex_assert_owner(&thd->LOCK_thd_data);
+  thd->wsrep_conflict_state= state;
 }
 
 
@@ -2762,6 +2793,9 @@ extern "C" void wsrep_thd_awake(THD *thd, my_bool signal)
 {
   if (signal)
   {
+    /* Here we should hold THD::LOCK_thd_data to
+    protect from concurrent usage. */
+    mysql_mutex_assert_owner(&thd->LOCK_thd_data);
     thd->awake(KILL_QUERY);
   }
   else
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index ef8c0e132f7..ae797c4c712 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -1,4 +1,4 @@
-/* Copyright (C) 2013 Codership Oy <info@codership.com>
+/* Copyright (C) 2013-2021 Codership Oy <info@codership.com>
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -804,10 +804,12 @@ my_bool wsrep_thd_is_local(void *thd_ptr, my_bool sync)
 
 int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
 {
-  THD *victim_thd = (THD *) victim_thd_ptr;
-  THD *bf_thd     = (THD *) bf_thd_ptr;
+  THD *victim_thd= (THD *) victim_thd_ptr;
+  THD *bf_thd= (THD *) bf_thd_ptr;
   DBUG_ENTER("wsrep_abort_thd");
 
+  mysql_mutex_assert_owner(&victim_thd->LOCK_thd_data);
+
   if ( (WSREP(bf_thd) ||
          ( (WSREP_ON || bf_thd->variables.wsrep_OSU_method == WSREP_OSU_RSU) &&
            bf_thd->wsrep_exec_mode == TOTAL_ORDER) )                         &&
@@ -821,6 +823,7 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
                   "aborted. Ignoring.",
                   (bf_thd) ? (long long)bf_thd->real_id : 0,
                   (long long)victim_thd->real_id);
+      mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
       DBUG_RETURN(1);
     }
 
@@ -831,6 +834,7 @@ int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, my_bool signal)
   else
   {
     WSREP_DEBUG("wsrep_abort_thd not effective: %p %p", bf_thd, victim_thd);
+    mysql_mutex_unlock(&victim_thd->LOCK_thd_data);
   }
 
   DBUG_RETURN(1);
diff --git a/storage/connect/colblk.h b/storage/connect/colblk.h
index f303528f982..03a467e9230 100644
--- a/storage/connect/colblk.h
+++ b/storage/connect/colblk.h
@@ -1,7 +1,7 @@
 /*************** Colblk H Declares Source Code File (.H) ***************/
 /*  Name: COLBLK.H    Version 1.7                                      */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          2005-2015    */
+/*  (C) Copyright to the author Olivier BERTRAND          2005-2019    */
 /*                                                                     */
 /*  This file contains the COLBLK and derived classes declares.        */
 /***********************************************************************/
diff --git a/storage/connect/mysql-test/connect/t/mongo.inc b/storage/connect/mysql-test/connect/t/mongo.inc
index fab2ca84139..25c3f207696 100644
--- a/storage/connect/mysql-test/connect/t/mongo.inc
+++ b/storage/connect/mysql-test/connect/t/mongo.inc
@@ -1,3 +1,11 @@
-let $MONGO= C:/Applic/MongoDB/Server/3.6/bin/mongo;
-let $MONGOIMPORT= C:/Applic/MongoDB/Server/3.6/bin/mongoimport;
+#let $MONGO= C:/Applic/MongoDB/Server/3.6/bin/mongo;
+#let $MONGOIMPORT= C:/Applic/MongoDB/Server/3.6/bin/mongoimport;
+let $MONGO= mongo;
+let $MONGOIMPORT= mongoimport;
+
+
+
+
+
+
 
diff --git a/storage/connect/odbconn.cpp b/storage/connect/odbconn.cpp
index 5b20b33e240..8ff8d3a84e1 100644
--- a/storage/connect/odbconn.cpp
+++ b/storage/connect/odbconn.cpp
@@ -1,7 +1,7 @@
 /***********************************************************************/
-/*  Name: ODBCONN.CPP  Version 2.3                                     */
+/*  Name: ODBCONN.CPP  Version 2.4                                     */
 /*                                                                     */
-/*  (C) Copyright to the author Olivier BERTRAND          1998-2017    */
+/*  (C) Copyright to the author Olivier BERTRAND          1998-2021    */
 /*                                                                     */
 /*  This file contains the ODBC connection classes functions.          */
 /***********************************************************************/
@@ -1509,7 +1509,7 @@ int ODBConn::ExecDirectSQL(char *sql, ODBCCOL *tocols)
       ThrowDBX(MSG(COL_NUM_MISM));
 
     // Now bind the column buffers
-    for (n = 1, colp = tocols; colp; colp = (PODBCCOL)colp->GetNext())
+    for (colp = tocols; colp; colp = (PODBCCOL)colp->GetNext())
       if (!colp->IsSpecial()) {
         buffer = colp->GetBuffer(m_RowsetSize);
         len = colp->GetBuflen();
@@ -1525,12 +1525,11 @@ int ODBConn::ExecDirectSQL(char *sql, ODBCCOL *tocols)
           htrc("Binding col=%u type=%d buf=%p len=%d slen=%p\n",
                   n, tp, buffer, len, colp->GetStrLen());
 
-        rc = SQLBindCol(hstmt, n, tp, buffer, len, colp->GetStrLen());
+        rc = SQLBindCol(hstmt, colp->GetIndex(), tp, buffer, len, colp->GetStrLen());
 
         if (!Check(rc))
           ThrowDBX(rc, "SQLBindCol", hstmt);
 
-        n++;
         } // endif pcol
 
   } catch(DBX *x) {
diff --git a/storage/connect/tabbson.cpp b/storage/connect/tabbson.cpp
index a7c561318c7..8569e39f678 100644
--- a/storage/connect/tabbson.cpp
+++ b/storage/connect/tabbson.cpp
@@ -376,7 +376,7 @@ int BSONDISC::GetColumns(PGLOBAL g, PCSZ db, PCSZ dsn, PTOS topt)
       } // endswitch ReadDB
 
     } else
-      jsp = bp->GetArrayValue(bdp, i);
+      jsp = bp->GetNext(jsp);
 
     if (!(row = (jsp) ? bp->GetObject(jsp) : NULL))
       break;
@@ -2185,7 +2185,9 @@ void BSONCOL::WriteColumn(PGLOBAL g)
 TDBBSON::TDBBSON(PGLOBAL g, PBDEF tdp, PTXF txfp) : TDBBSN(g, tdp, txfp)
 {
   Docp = NULL;
+  Docrow = NULL;
   Multiple = tdp->Multiple;
+  Docsize = 0;
   Done = Changed = false;
   Bp->SetPretty(2);
 } // end of TDBBSON standard constructor
@@ -2193,7 +2195,9 @@ TDBBSON::TDBBSON(PGLOBAL g, PBDEF tdp, PTXF txfp) : TDBBSN(g, tdp, txfp)
 TDBBSON::TDBBSON(PBTDB tdbp) : TDBBSN(tdbp)
 {
   Docp = tdbp->Docp;
+  Docrow = tdbp->Docrow;
   Multiple = tdbp->Multiple;
+  Docsize = tdbp->Docsize;
   Done = tdbp->Done;
   Changed = tdbp->Changed;
 } // end of TDBBSON copy constructor
@@ -2374,6 +2378,7 @@ int TDBBSON::MakeDocument(PGLOBAL g)
 
   } // endif jsp
 
+  Docsize = Bp->GetSize(Docp);
   Done = true;
   return RC_OK;
 } // end of MakeDocument
@@ -2388,7 +2393,7 @@ int TDBBSON::Cardinality(PGLOBAL g)
   else if (Cardinal < 0) {
     if (!Multiple) {
       if (MakeDocument(g) == RC_OK)
-        Cardinal = Bp->GetSize(Docp);
+        Cardinal = Docsize;
 
     } else
       return 10;
@@ -2417,6 +2422,7 @@ void TDBBSON::ResetSize(void)
   MaxSize = Cardinal = -1;
   Fpos = -1;
   N = 0;
+  Docrow = NULL;
   Done = false;
 } // end of ResetSize
 
@@ -2475,6 +2481,7 @@ bool TDBBSON::SetRecpos(PGLOBAL, int recpos)
 #endif // 0
 
   Fpos = recpos - 1;
+  Docrow = NULL;
   return false;
 } // end of SetRecpos
 
@@ -2490,6 +2497,7 @@ bool TDBBSON::OpenDB(PGLOBAL g)
     Fpos = -1;
     NextSame = false;
     SameRow = 0;
+    Docrow = NULL;
     return false;
   } // endif use
 
@@ -2530,12 +2538,9 @@ int TDBBSON::ReadDB(PGLOBAL)
     NextSame = false;
     M++;
     rc = RC_OK;
-  } else if (++Fpos < (signed)Bp->GetSize(Docp)) {
-    Row = Bp->GetArrayValue(Docp, Fpos);
-
-  if (Row->Type == TYPE_JVAL)
-      Row = Bp->GetBson(Row);
-
+  } else if (++Fpos < Docsize) {
+    Docrow = (Docrow) ? Bp->GetNext(Docrow) : Bp->GetArrayValue(Docp, Fpos);
+    Row = (Docrow->Type == TYPE_JVAL) ? Bp->GetBson(Docrow) : Docrow;
     SameRow = 0;
     M = 1;
     rc = RC_OK;
diff --git a/storage/connect/tabbson.h b/storage/connect/tabbson.h
index 7f41bba6bd9..1696f4dfdbc 100644
--- a/storage/connect/tabbson.h
+++ b/storage/connect/tabbson.h
@@ -318,7 +318,9 @@ protected:
 
   // Members
   PBVAL Docp;                      // The document array
+  PBVAL Docrow;                    // Document row
   int   Multiple;                  // 0: No 1: DIR 2: Section 3: filelist
+  int   Docsize;                   // The document size
   bool  Done;                      // True when document parsing is done
   bool  Changed;                   // After Update, Insert or Delete
 }; // end of class TDBBSON
diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc
index 37748cb497a..8fb52b211c6 100644
--- a/storage/innobase/handler/ha_innodb.cc
+++ b/storage/innobase/handler/ha_innodb.cc
@@ -60,7 +60,6 @@ this program; if not, write to the Free Software Foundation, Inc.,
 
 #include <my_service_manager.h>
 #include <key.h>
-#include <sql_manager.h>
 
 /* Include necessary InnoDB headers */
 #include "btr0btr.h"
@@ -5234,17 +5233,18 @@ UNIV_INTERN void lock_cancel_waiting_and_release(lock_t* lock);
 @sa THD::awake() @sa ha_kill_query() */
 static void innobase_kill_query(handlerton*, THD* thd, enum thd_kill_levels)
 {
-	DBUG_ENTER("innobase_kill_query");
+  DBUG_ENTER("innobase_kill_query");
 #ifdef WITH_WSREP
-	if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT) {
-		/* if victim has been signaled by BF thread and/or aborting
-		   is already progressing, following query aborting is not necessary
-		   any more.
-		   Also, BF thread should own trx mutex for the victim, which would
-		   conflict with trx_mutex_enter() below
-		*/
-		DBUG_VOID_RETURN;
-	}
+  if (wsrep_thd_get_conflict_state(thd) != NO_CONFLICT)
+  {
+    /* if victim has been signaled by BF thread and/or aborting
+    is already progressing, following query aborting is not necessary
+    any more. */
+    WSREP_DEBUG("Victim thread %ld bail out conflict_state %s query %s",
+		thd_get_thread_id(thd),
+		wsrep_thd_conflict_state_str(thd), wsrep_thd_query(thd));
+    DBUG_VOID_RETURN;
+  }
 #endif /* WITH_WSREP */
 
   if (trx_t* trx= thd_to_trx(thd))
@@ -19498,109 +19498,109 @@ static struct st_mysql_storage_engine innobase_storage_engine=
 { MYSQL_HANDLERTON_INTERFACE_VERSION };
 
 #ifdef WITH_WSREP
+static
 void
 wsrep_abort_slave_trx(
-/*==================*/
-	wsrep_seqno_t bf_seqno,
-	wsrep_seqno_t victim_seqno)
-{
-	WSREP_ERROR("Trx %lld tries to abort slave trx %lld. This could be "
-		"caused by:\n\t"
-		"1) unsupported configuration options combination, please check documentation.\n\t"
-		"2) a bug in the code.\n\t"
-		"3) a database corruption.\n Node consistency compromized, "
-		"need to abort. Restart the node to resync with cluster.",
-		(long long)bf_seqno, (long long)victim_seqno);
-	abort();
-}
-
-struct bg_wsrep_kill_trx_arg {
-	my_thread_id	thd_id;
-	trx_id_t	trx_id;
-	int64_t		bf_seqno;
-	ibool		signal;
-};
-
-static void bg_wsrep_kill_trx(
-	void *void_arg)
+	THD* bf_thd,
+	THD* victim_thd)
+{
+  wsrep_seqno_t bf_seqno= wsrep_thd_trx_seqno(bf_thd);
+  wsrep_seqno_t victim_seqno= wsrep_thd_trx_seqno(victim_thd);
+
+  WSREP_ERROR("wsrep_abort_slave_trx: BF Aborter %s thread: %ld "
+	      "seqno: %lld query_state: %s conflict_state: %s "
+	      "exec mode %s query: %s",
+	      wsrep_thd_is_BF(bf_thd, false) ? "BF" : "normal",
+	      thd_get_thread_id(bf_thd),
+	      bf_seqno,
+	      wsrep_thd_query_state_str(bf_thd),
+	      wsrep_thd_conflict_state_str(bf_thd),
+	      wsrep_thd_exec_mode_str(bf_thd),
+	      wsrep_thd_query(bf_thd));
+
+  WSREP_ERROR("wsrep_abort_slave_trx: Victim %s thread: %ld "
+	      "seqno: %lld query_state: %s  conflict_state: %s "
+	      "exec mode %s query: %s",
+	      wsrep_thd_is_BF(victim_thd, false) ? "BF" : "normal",
+	      thd_get_thread_id(victim_thd),
+	      wsrep_thd_trx_seqno(victim_thd),
+	      wsrep_thd_query_state_str(victim_thd),
+	      wsrep_thd_conflict_state_str(victim_thd),
+	      wsrep_thd_exec_mode_str(victim_thd),
+	      wsrep_thd_query(victim_thd));
+
+  WSREP_ERROR("Trx %lld tries to abort slave trx %lld. This could be "
+	      "caused by:\n\t"
+	      "1) unsupported configuration options combination, please check documentation.\n\t"
+	      "2) a bug in the code.\n\t"
+	      "3) a database corruption.\n Node consistency compromized, "
+	      "need to abort. Restart the node to resync with cluster.",
+	      (long long)bf_seqno, (long long)victim_seqno);
+  abort();
+}
+
+/** This function is used to kill one transaction in BF. */
+static
+void
+wsrep_kill_victim(
+	MYSQL_THD const bf_thd,
+	const trx_t* const bf_trx,
+	MYSQL_THD thd,
+	trx_t* victim_trx,
+	my_bool signal)
 {
-	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)void_arg;
-	THD *thd 		   = find_thread_by_id(arg->thd_id, false);
-	trx_t *victim_trx 	   = NULL;
-	bool awake 		   = false;
-	DBUG_ENTER("bg_wsrep_kill_trx");
-
-	if (thd) {
-		victim_trx= thd_to_trx(thd);
-		/* Victim trx might not exist e.g. on MDL-conflict. */
-		if (victim_trx) {
-			lock_mutex_enter();
-			trx_mutex_enter(victim_trx);
-			if (victim_trx->id != arg->trx_id ||
-			    victim_trx->state == TRX_STATE_COMMITTED_IN_MEMORY)
-			{
-				/* Victim was meanwhile rolled back or
-				committed */
-				trx_mutex_exit(victim_trx);
-				lock_mutex_exit();
-				wsrep_thd_UNLOCK(thd);
-				victim_trx= NULL;
-			}
-		} else {
-			/* find_thread_by_id locked
-			THD::LOCK_thd_data */
-			wsrep_thd_UNLOCK(thd);
-		}
-	}
-
-	if (!victim_trx) {
-		/* Victim trx might not exist (MDL-conflict) or victim
-		was meanwhile rolled back or committed because of
-		a KILL statement or a disconnect. */
-		goto ret;
-	}
+	ut_ad(bf_thd);
+	ut_ad(thd);
+	ut_ad(victim_trx);
+	ut_ad(lock_mutex_own());
+	ut_ad(trx_mutex_own(victim_trx));
 
-	WSREP_DEBUG("BF kill (" ULINTPF ", seqno: " INT64PF
-		    "), victim: (%lu) trx: " TRX_ID_FMT,
-		    arg->signal, arg->bf_seqno,
-		    thd_get_thread_id(thd),
-		    victim_trx->id);
+	DBUG_ENTER("wsrep_kill_victim");
 
-	WSREP_DEBUG("Aborting query: %s conf %d trx: %" PRId64,
-		    (wsrep_thd_query(thd)) ? wsrep_thd_query(thd) : "void",
-		    wsrep_thd_conflict_state(thd, FALSE),
-		    wsrep_thd_ws_handle(thd)->trx_id);
+	const int64_t bf_seqno= wsrep_thd_trx_seqno(bf_thd);
 
 	if (wsrep_thd_query_state(thd) == QUERY_EXITING) {
 		WSREP_DEBUG("kill trx EXITING for " TRX_ID_FMT,
 			    victim_trx->id);
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
 	}
 
 	if (wsrep_thd_exec_mode(thd) != LOCAL_STATE) {
-		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT ", state: %d",
+		WSREP_DEBUG("withdraw for BF trx: " TRX_ID_FMT
+			    ", state: %s exec %s",
 			    victim_trx->id,
-		wsrep_thd_get_conflict_state(thd));
+			    wsrep_thd_conflict_state_str(thd),
+			    wsrep_thd_exec_mode_str(thd));
 	}
 
 	switch (wsrep_thd_get_conflict_state(thd)) {
 	case NO_CONFLICT:
+		/* This will cause any call to innobase_kill_query()
+		for this thd to bail out. */
 		wsrep_thd_set_conflict_state(thd, MUST_ABORT);
 		break;
         case MUST_ABORT:
 		WSREP_DEBUG("victim " TRX_ID_FMT " in MUST ABORT state",
 			    victim_trx->id);
-		goto ret_awake;
+		wsrep_thd_awake(thd, signal);
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
+		break;
 	case ABORTED:
 	case ABORTING: // fall through
 	default:
-		WSREP_DEBUG("victim " TRX_ID_FMT " in state %d",
-			    victim_trx->id, wsrep_thd_get_conflict_state(thd));
-		goto ret_unlock;
+		WSREP_DEBUG("victim " TRX_ID_FMT " in state %s",
+			    victim_trx->id,
+			    wsrep_thd_conflict_state_str(thd));
+		wsrep_thd_UNLOCK(thd);
+		DBUG_VOID_RETURN;
+		break;
 	}
 
 	switch (wsrep_thd_query_state(thd)) {
 	case QUERY_COMMITTING:
+	{
 		enum wsrep_status rcode;
 
 		WSREP_DEBUG("kill query for: %ld",
@@ -19609,12 +19609,11 @@ static void bg_wsrep_kill_trx(
 			    victim_trx->id);
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-			wsrep_abort_slave_trx(arg->bf_seqno,
-					      wsrep_thd_trx_seqno(thd));
+			wsrep_abort_slave_trx(bf_thd, thd);
 		} else {
 			wsrep_t *wsrep= get_wsrep();
 			rcode = wsrep->abort_pre_commit(
-				wsrep, arg->bf_seqno,
+				wsrep, bf_seqno,
 				(wsrep_trx_id_t)wsrep_thd_ws_handle(thd)->trx_id
 			);
 
@@ -19623,7 +19622,10 @@ static void bg_wsrep_kill_trx(
 				WSREP_DEBUG("cancel commit warning: "
 					    TRX_ID_FMT,
 					    victim_trx->id);
-				goto ret_awake;
+				wsrep_thd_awake(thd, signal);
+				wsrep_thd_UNLOCK(thd);
+				DBUG_VOID_RETURN;
+				break;
 			case WSREP_OK:
 				break;
 			default:
@@ -19636,21 +19638,24 @@ static void bg_wsrep_kill_trx(
 				 * kill the lock holder first.
 				 */
 				abort();
+				break;
 			}
 		}
-		goto ret_awake;
+		wsrep_thd_awake(thd, signal);
+		wsrep_thd_UNLOCK(thd);
+		break;
+	}
 	case QUERY_EXEC:
+	{
 		/* it is possible that victim trx is itself waiting for some
 		 * other lock. We need to cancel this waiting
 		 */
 		WSREP_DEBUG("kill trx QUERY_EXEC for " TRX_ID_FMT,
 			    victim_trx->id);
 
-		victim_trx->lock.was_chosen_as_deadlock_victim= TRUE;
-
 		if (victim_trx->lock.wait_lock) {
 			WSREP_DEBUG("victim has wait flag: %ld",
-				thd_get_thread_id(thd));
+				    thd_get_thread_id(thd));
 			lock_t*  wait_lock = victim_trx->lock.wait_lock;
 
 			if (wait_lock) {
@@ -19659,150 +19664,166 @@ static void bg_wsrep_kill_trx(
 				lock_cancel_waiting_and_release(wait_lock);
 			}
 
+			wsrep_thd_awake(thd, signal);
+			wsrep_thd_UNLOCK(thd);
 		} else {
 			/* abort currently executing query */
-			DBUG_PRINT("wsrep",("sending KILL_QUERY to: %lu",
-                                            thd_get_thread_id(thd)));
 			WSREP_DEBUG("kill query for: %ld",
-				thd_get_thread_id(thd));
+				    thd_get_thread_id(thd));
 
 			/* for BF thd, we need to prevent him from committing */
 			if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
-				wsrep_abort_slave_trx(arg->bf_seqno,
-						    wsrep_thd_trx_seqno(thd));
+				wsrep_abort_slave_trx(bf_thd, thd);
 			}
+
+			/* Note that innobase_kill_query will take lock_mutex
+			and trx_mutex */
+			wsrep_thd_awake(thd, signal);
+			wsrep_thd_UNLOCK(thd);
 		}
-		goto ret_awake;
+		break;
+	}
 	case QUERY_IDLE:
 	{
 		WSREP_DEBUG("kill IDLE for " TRX_ID_FMT, victim_trx->id);
 
 		if (wsrep_thd_exec_mode(thd) == REPL_RECV) {
 			WSREP_DEBUG("kill BF IDLE, seqno: %lld",
-				    (long long)wsrep_thd_trx_seqno(thd));
-			wsrep_abort_slave_trx(arg->bf_seqno,
-					      wsrep_thd_trx_seqno(thd));
-			goto ret_unlock;
+				    wsrep_thd_trx_seqno(thd));
+			wsrep_abort_slave_trx(bf_thd, thd);
 		}
-                /* This will lock thd from proceeding after net_read() */
-		wsrep_thd_set_conflict_state(thd, ABORTING);
 
+                /* This will lock thd from proceeding after net_read() and
+		will cause any call to innobase_kill_query() for this
+		thd to bail out. */
+		wsrep_thd_set_conflict_state(thd, ABORTING);
 		wsrep_lock_rollback();
 
 		if (wsrep_aborting_thd_contains(thd)) {
 			WSREP_WARN("duplicate thd aborter %lu",
-			           (ulong) thd_get_thread_id(thd));
+			           thd_get_thread_id(thd));
 		} else {
 			wsrep_aborting_thd_enqueue(thd);
-			DBUG_PRINT("wsrep",("enqueuing trx abort for %lu",
-			                    thd_get_thread_id(thd)));
 			WSREP_DEBUG("enqueuing trx abort for (%lu)",
-			            thd_get_thread_id(thd));
+				    thd_get_thread_id(thd));
 		}
 
-		DBUG_PRINT("wsrep",("signalling wsrep rollbacker"));
 		WSREP_DEBUG("signaling aborter");
 		wsrep_unlock_rollback();
-		goto ret_unlock;
+		wsrep_thd_UNLOCK(thd);
+		break;
 	}
 	default:
 		WSREP_WARN("bad wsrep query state: %d",
 			  wsrep_thd_query_state(thd));
-		goto ret_unlock;
+		ut_error;
 	}
-
-ret_awake:
-	awake= true;
-
-ret_unlock:
-	trx_mutex_exit(victim_trx);
-	lock_mutex_exit();
-	if (awake)
-		wsrep_thd_awake(thd, arg->signal);
-	wsrep_thd_UNLOCK(thd);
-
-ret:
-	free(arg);
 	DBUG_VOID_RETURN;
-
 }
 
-/*******************************************************************//**
+/*******************************************************************
 This function is used to kill one transaction in BF. */
-UNIV_INTERN
 void
 wsrep_innobase_kill_one_trx(
-/*========================*/
 	MYSQL_THD const bf_thd,
 	const trx_t * const bf_trx,
 	trx_t *victim_trx,
-	ibool signal)
+	my_bool signal)
 {
-	ut_ad(bf_thd);
-	ut_ad(victim_trx);
-	ut_ad(lock_mutex_own());
-	ut_ad(trx_mutex_own(victim_trx));
-
-	bg_wsrep_kill_trx_arg *arg = (bg_wsrep_kill_trx_arg*)malloc(sizeof(*arg));
-	arg->thd_id	= thd_get_thread_id(victim_trx->mysql_thd);
-	arg->trx_id	= victim_trx->id;
-	arg->bf_seqno	= wsrep_thd_trx_seqno((THD*)bf_thd);
-	arg->signal	= signal;
-
-	DBUG_ENTER("wsrep_innobase_kill_one_trx");
-
-	WSREP_LOG_CONFLICT(bf_thd, victim_trx->mysql_thd, TRUE);
-
-	DBUG_EXECUTE_IF("sync.wsrep_after_BF_victim_lock",
-		{
-		  const char act[]=
-		    "now "
-		    "wait_for signal.wsrep_after_BF_victim_lock";
-		  DBUG_ASSERT(!debug_sync_set_action(bf_thd,
-						     STRING_WITH_LEN(act)));
-		};);
-
-
-	mysql_manager_submit(bg_wsrep_kill_trx, arg);
-	DBUG_VOID_RETURN;
+  ut_ad(bf_thd);
+  ut_ad(victim_trx);
+  ut_ad(lock_mutex_own());
+  ut_ad(trx_mutex_own(victim_trx));
+
+  DBUG_ENTER("wsrep_innobase_kill_one_trx");
+  THD *thd= (THD *) victim_trx->mysql_thd;
+
+  /* Here we need to lock THD::LOCK_thd_data to protect from
+  concurrent usage or disconnect or delete. */
+  DEBUG_SYNC(bf_thd, "wsrep_before_BF_victim_lock");
+  wsrep_thd_LOCK(thd);
+  DEBUG_SYNC(bf_thd, "wsrep_after_BF_victim_lock");
+
+  WSREP_LOG_CONFLICT(bf_thd, thd, TRUE);
+
+  WSREP_DEBUG("wsrep_innobase_kill_one_trx: Aborter %s "
+	      "trx_id: " TRX_ID_FMT " thread: %ld "
+	      "seqno: %lld query_state: %s conflict_state: %s "
+	      "exec mode %s query: %s",
+	      wsrep_thd_is_BF(bf_thd, false) ? "BF" : "normal",
+	      bf_trx ? bf_trx->id : TRX_ID_MAX,
+	      thd_get_thread_id(bf_thd),
+	      wsrep_thd_trx_seqno(bf_thd),
+	      wsrep_thd_query_state_str(bf_thd),
+	      wsrep_thd_conflict_state_str(bf_thd),
+	      wsrep_thd_exec_mode_str(bf_thd),
+	      wsrep_thd_query(bf_thd));
+
+  WSREP_DEBUG("wsrep_innobase_kill_one_trx: Victim %s "
+	      "trx_id: " TRX_ID_FMT " thread: %ld "
+	      "seqno: %lld query_state: %s  conflict_state: %s "
+	      "exec mode %s query: %s",
+	      wsrep_thd_is_BF(thd, false) ? "BF" : "normal",
+	      victim_trx->id,
+	      thd_get_thread_id(thd),
+	      wsrep_thd_trx_seqno(thd),
+	      wsrep_thd_query_state_str(thd),
+	      wsrep_thd_conflict_state_str(thd),
+	      wsrep_thd_exec_mode_str(thd),
+	      wsrep_thd_query(thd));
+
+  wsrep_kill_victim(bf_thd, bf_trx, thd, victim_trx, signal);
+  DBUG_VOID_RETURN;
 }
 
 static
 void
 wsrep_abort_transaction(
-/*====================*/
 	handlerton* hton,
 	THD *bf_thd,
 	THD *victim_thd,
 	my_bool signal)
 {
-	DBUG_ENTER("wsrep_abort_transaction");
-
-	trx_t* victim_trx	= thd_to_trx(victim_thd);
-	trx_t* bf_trx		= (bf_thd) ? thd_to_trx(bf_thd) : NULL;
-
-	WSREP_DEBUG("abort transaction: BF: %s victim: %s victim conf: %d",
-			wsrep_thd_query(bf_thd),
-			wsrep_thd_query(victim_thd),
-			wsrep_thd_conflict_state(victim_thd, FALSE));
-
-	if (victim_trx) {
-		lock_mutex_enter();
-		trx_mutex_enter(victim_trx);
-		wsrep_innobase_kill_one_trx(bf_thd, bf_trx, victim_trx, signal);
-		lock_mutex_exit();
-		trx_mutex_exit(victim_trx);
-		wsrep_srv_conc_cancel_wait(victim_trx);
-		DBUG_VOID_RETURN;
-	} else {
-		WSREP_DEBUG("victim does not have transaction");
-		wsrep_thd_LOCK(victim_thd);
-		wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
-		wsrep_thd_awake(victim_thd, signal);
-		wsrep_thd_UNLOCK(victim_thd);
-	}
+  DBUG_ENTER("wsrep_abort_transaction");
+  /* Note that victim thd is protected with
+  THD::LOCK_thd_data here. */
+  trx_t* victim_trx= thd_to_trx(victim_thd);
+  trx_t* bf_trx= thd_to_trx(bf_thd);
+
+  WSREP_DEBUG("wsrep_abort_transaction: BF:"
+	      " thread %ld query_state %s conflict_state %s"
+	      " exec %s query %s trx " TRX_ID_FMT,
+	      thd_get_thread_id(bf_thd),
+	      wsrep_thd_query_state_str(bf_thd),
+	      wsrep_thd_conflict_state_str(bf_thd),
+	      wsrep_thd_exec_mode_str(bf_thd),
+	      wsrep_thd_query(bf_thd),
+	      bf_trx ? bf_trx->id : 0);
+
+  WSREP_DEBUG("wsrep_abort_transaction: victim:"
+	      " thread %ld query_state %s conflict_state %s"
+	      " exec %s query %s trx " TRX_ID_FMT,
+	      thd_get_thread_id(victim_thd),
+	      wsrep_thd_query_state_str(victim_thd),
+	      wsrep_thd_conflict_state_str(victim_thd),
+	      wsrep_thd_exec_mode_str(victim_thd),
+	      wsrep_thd_query(victim_thd),
+	      victim_trx ? victim_trx->id : 0);
+
+  if (victim_trx) {
+    lock_mutex_enter();
+    trx_mutex_enter(victim_trx);
+    wsrep_kill_victim(bf_thd, bf_trx, victim_thd, victim_trx, signal);
+    lock_mutex_exit();
+    trx_mutex_exit(victim_trx);
+    wsrep_srv_conc_cancel_wait(victim_trx);
+  } else {
+    wsrep_thd_set_conflict_state(victim_thd, MUST_ABORT);
+    wsrep_thd_awake(victim_thd, signal);
+    wsrep_thd_UNLOCK(victim_thd);
+  }
 
-	DBUG_VOID_RETURN;
+  DBUG_VOID_RETURN;
 }
 
 static
diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h
index 3eab2135969..427e57f09d2 100644
--- a/storage/innobase/include/ha_prototypes.h
+++ b/storage/innobase/include/ha_prototypes.h
@@ -233,12 +233,11 @@ innobase_casedn_str(
 	char*	a);	/*!< in/out: string to put in lower case */
 
 #ifdef WITH_WSREP
-UNIV_INTERN
 void
 wsrep_innobase_kill_one_trx(MYSQL_THD const thd_ptr,
                             const trx_t * const bf_trx,
                             trx_t *victim_trx,
-                            ibool signal);
+                            my_bool signal);
 int wsrep_innobase_mysql_sort(int mysql_type, uint charset_number,
                              unsigned char* str, unsigned int str_length,
                              unsigned int buf_length);
diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc
index df1488b9df3..7da5e16a762 100644
--- a/storage/innobase/lock/lock0wait.cc
+++ b/storage/innobase/lock/lock0wait.cc
@@ -184,13 +184,11 @@ lock_wait_table_reserve_slot(
 check if lock timeout was for priority thread,
 as a side effect trigger lock monitor
 @param[in]    trx    transaction owning the lock
-@param[in]    locked true if trx and lock_sys_mutex is ownd
 @return	false for regular lock timeout */
 static
 bool
 wsrep_is_BF_lock_timeout(
-	const trx_t*	trx,
-	bool		locked = true)
+	const trx_t*	trx)
 {
 	bool long_wait= (trx->error_state != DB_DEADLOCK &&
 			 trx->is_wsrep() &&
@@ -204,18 +202,6 @@ wsrep_is_BF_lock_timeout(
 		ib::info() << "WSREP: BF lock wait long for trx:" << trx->id
 			   << " query: " << wsrep_thd_query(trx->mysql_thd);
 
-		if (!locked)
-			lock_mutex_enter();
-
-		ut_ad(lock_mutex_own());
-
-		wsrep_trx_print_locking(stderr, trx, 3000);
-		/* Note this will release lock_sys mutex */
-		lock_print_info_all_transactions(stderr);
-
-		if (locked)
-			lock_mutex_enter();
-
 		return was_wait;
 	} else
 		return false;
@@ -407,7 +393,7 @@ lock_wait_suspend_thread(
 	    && wait_time > (double) lock_wait_timeout
 #ifdef WITH_WSREP
 	    && (!trx->is_wsrep()
-		|| (!wsrep_is_BF_lock_timeout(trx, false)
+		|| (!wsrep_is_BF_lock_timeout(trx)
 		    && trx->error_state != DB_DEADLOCK))
 #endif /* WITH_WSREP */
 	    ) {
diff --git a/storage/tokudb/PerconaFT/portability/toku_pthread.cc b/storage/tokudb/PerconaFT/portability/toku_pthread.cc
index fe8a4d485cd..c066a734bae 100644
--- a/storage/tokudb/PerconaFT/portability/toku_pthread.cc
+++ b/storage/tokudb/PerconaFT/portability/toku_pthread.cc
@@ -55,7 +55,9 @@ Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
 #include <toku_pthread.h>
 
 int toku_pthread_yield(void) {
-#if defined(HAVE_PTHREAD_YIELD)
+#if defined(HAVE_SCHED_YIELD)
+    return sched_yield();
+#elif defined(HAVE_PTHREAD_YIELD)
 # if defined(PTHREAD_YIELD_RETURNS_INT)
     return pthread_yield();
 # elif defined(PTHREAD_YIELD_RETURNS_VOID)