MDEV-23851 BF-BF Conflict issue because of UK GAP locks

Add missing dbug sync point and correct the test case.

3 files changed, 218 insertions, 37 deletions

diff --git a/mysql-test/suite/galera/r/galera_UK_conflict.result b/mysql-test/suite/galera/r/galera_UK_conflict.result
index 76649f1b268..44bb64c9d63 100644
--- a/mysql-test/suite/galera/r/galera_UK_conflict.result
+++ b/mysql-test/suite/galera/r/galera_UK_conflict.result
@@ -1,3 +1,5 @@
+connection node_2;
+connection node_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int,  unique key keyj (f2));
 INSERT INTO t1 VALUES (1, 1, 0);
 INSERT INTO t1 VALUES (3, 3, 0);
@@ -23,14 +25,14 @@ connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
 connection node_1;
 COMMIT;
 connection node_1a;
 SET SESSION wsrep_on = 0;
 SET SESSION wsrep_on = 1;
 SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 SET GLOBAL DEBUG_DBUG = "";
 SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
@@ -39,13 +41,13 @@ SET debug_sync='RESET';
 SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
 SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
 SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
-SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 SET GLOBAL DEBUG_DBUG = "";
 SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
 SET GLOBAL debug_dbug = NULL;
 SET debug_sync='RESET';
-SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync';
 SET GLOBAL wsrep_provider_options = 'dbug=';
 connection node_1;
 SELECT * FROM t1;
@@ -55,8 +57,6 @@ f1	f2	f3
 4	4	2
 5	5	2
 10	10	0
-wsrep_local_replays
-1
 SET GLOBAL wsrep_slave_threads = DEFAULT;
 connection node_2;
 SELECT * FROM t1;
@@ -68,22 +68,64 @@ f1	f2	f3
 10	10	0
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
-SELECT * FROM t1;
-f1	f2	f3
-1	1	0
-3	3	1
-4	4	2
-5	5	2
-7	7	7
-8	8	8
-10	10	0
+DROP TABLE t1;
+test scenario 2
 connection node_1;
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int,  unique key keyj (f2));
+INSERT INTO t1 VALUES (1, 1, 0);
+INSERT INTO t1 VALUES (3, 3, 0);
+INSERT INTO t1 VALUES (10, 10, 0);
+SET GLOBAL wsrep_slave_threads = 3;
+SET GLOBAL wsrep_provider_options = 'dbug=d,apply_monitor_slave_enter_sync';
+connection node_1;
+SET SESSION wsrep_sync_wait=0;
+START TRANSACTION;
+DELETE FROM t1 WHERE f2 = 3;
+INSERT INTO t1 VALUES (3, 3, 1);
+connection node_1a;
+SET SESSION wsrep_sync_wait=0;
+connection node_2;
+INSERT INTO t1 VALUES (5, 5, 2);
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_master_enter_sync';
+connection node_1;
+COMMIT;
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_replay_cb";
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_master_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'signal=apply_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_replay_cb_reached";
+SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_slave_enter_sync';
+connection node_2;
+INSERT INTO t1 VALUES (4, 4, 2);
+connection node_1a;
+SET SESSION wsrep_on = 0;
+SET SESSION wsrep_on = 1;
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_slave_enter_sync';
+SET GLOBAL wsrep_provider_options = 'dbug=';
+SET GLOBAL DEBUG_DBUG = "";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_replay_cb";
+SET GLOBAL debug_dbug = NULL;
+SET debug_sync='RESET';
+connection node_1;
+SET GLOBAL wsrep_slave_threads = DEFAULT;
+connection node_2;
 SELECT * FROM t1;
 f1	f2	f3
 1	1	0
 3	3	1
 4	4	2
 5	5	2
-7	7	7
 10	10	0
+INSERT INTO t1 VALUES (7,7,7);
+INSERT INTO t1 VALUES (8,8,8);
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_UK_conflict.test b/mysql-test/suite/galera/t/galera_UK_conflict.test
index 57bafbf8ae0..9978ba9b8bf 100644
--- a/mysql-test/suite/galera/t/galera_UK_conflict.test
+++ b/mysql-test/suite/galera/t/galera_UK_conflict.test
@@ -1,14 +1,14 @@
 #
 # This test tests the operation of transaction replay with a scenario
-# where two subsequent write sets in applying conflict with local transaction
+# where two subsequent write sets being applied conflict with local transaction
 # in commit phase. The conflict is "false positive" confict on GAP lock in
 # secondary unique index.
 # The first applier will cause BF abort for the local committer, which
 # starts replaying because of positive certification.
-# In buggy version, scenatio continues so that ehile the local transaction
+# In buggy version, the test scenario continues so that while the local transaction
 # is replaying, the latter applier experiences similar UK GAP lock conflict
 # and forces the replayer to abort second time.
-# In fixed version, this latter BF abort should not happen.
+# In fixed version, this latter replayer BF abort should not happen.
 #
 
 --source include/galera_cluster.inc
@@ -16,15 +16,16 @@
 --source include/have_debug_sync.inc
 --source include/galera_have_debug_sync.inc
 
---let $wsrep_local_replays_old = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
+
+--let $expected_wsrep_local_replays = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
 
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int,  unique key keyj (f2));
 INSERT INTO t1 VALUES (1, 1, 0);
 INSERT INTO t1 VALUES (3, 3, 0);
 INSERT INTO t1 VALUES (10, 10, 0);
 
-# we will need 2 appliers threads for applyin two write sets in parallel in node1
-# and 1 applier thread for handling replaying
+# we will need 2 appliers threads for applyin two writes ets in parallel in node1
+# and 1 applier thread for handling replaying 
 SET GLOBAL wsrep_slave_threads = 3;
 SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
 
@@ -42,7 +43,7 @@ INSERT INTO t1 VALUES (3, 3, 1);
 --connection node_1a
 SET SESSION wsrep_sync_wait=0;
 
-# send from node 2 first INSERT transaction,  which will conflict on GAP lock in node 1
+# send from node 2 first an INSERT transaction,  which will conflict on GAP lock in node 1
 --connection node_2
 INSERT INTO t1 VALUES (5, 5, 2);
 
@@ -67,7 +68,7 @@ INSERT INTO t1 VALUES (4, 4, 2);
 # both appliers are now waiting in separate sync points
 
 # Block the local commit, send the COMMIT and wait until it gets blocked
---let $galera_sync_point = commit_monitor_enter_sync
+--let $galera_sync_point = commit_monitor_master_enter_sync
 --source include/galera_set_sync_point.inc
 
 --connection node_1
@@ -75,12 +76,12 @@ INSERT INTO t1 VALUES (4, 4, 2);
 
 --connection node_1a
 # wait for the local commit to enter in commit monitor wait state
---let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_enter_sync
+--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync
 --source include/galera_wait_sync_point.inc
 --source include/galera_clear_sync_point.inc
 
 # release the local transaction to continue with commit
---let $galera_sync_point = commit_monitor_enter_sync
+--let $galera_sync_point = commit_monitor_master_enter_sync
 --source include/galera_signal_sync_point.inc
 --source include/galera_clear_sync_point.inc
 
@@ -90,6 +91,11 @@ SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
 SET GLOBAL debug_dbug = NULL;
 SET debug_sync='RESET';
 
+# wait for BF abort to happen and replaying begin
+--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_local_replays FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
+--let $wait_condition_on_error_output= SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
+--source include/wait_condition_with_debug.inc
+
 # set another sync point for second applier
 SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
 
@@ -97,11 +103,11 @@ SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_apply_cb";
 --let $galera_sync_point = apply_monitor_slave_enter_sync
 --source include/galera_signal_sync_point.inc
 
-# waiting until second applier is in wait
+# waiting until second applier is in wait state
 SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_apply_cb_reached";
 
 # stopping second applier before commit
---let $galera_sync_point = commit_monitor_enter_sync
+--let $galera_sync_point = commit_monitor_slave_enter_sync
 --source include/galera_set_sync_point.inc
 --source include/galera_clear_sync_point.inc
 
@@ -112,9 +118,9 @@ SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
 SET GLOBAL debug_dbug = NULL;
 SET debug_sync='RESET';
 
-# with fixed version, second applier has reached commit monitor, and we can
+# with fixed version, second applier has reached comit monitor, and we can
 # release it to complete
---let $galera_sync_point = commit_monitor_enter_sync
+--let $galera_sync_point = commit_monitor_slave_enter_sync
 --source include/galera_signal_sync_point.inc
 --source include/galera_clear_sync_point.inc
 
@@ -124,12 +130,6 @@ SET debug_sync='RESET';
 
 SELECT * FROM t1;
 
-# wsrep_local_replays has increased by 1
---let $wsrep_local_replays_new = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
---disable_query_log
---eval SELECT $wsrep_local_replays_new - $wsrep_local_replays_old = 1 AS wsrep_local_replays;
---enable_query_log
-
 # returning original slave thread count
 SET GLOBAL wsrep_slave_threads = DEFAULT;
 
@@ -140,9 +140,137 @@ SELECT * FROM t1;
 # original state in node 1
 INSERT INTO t1 VALUES (7,7,7);
 INSERT INTO t1 VALUES (8,8,8);
-SELECT * FROM t1;
 
+DROP TABLE t1;
+
+##################################################################################
+# test scenario 2
+#
+# commit order is now: INSERT-1, local COMMIT, INSERT-2
+# while local trx is replaying, the latter applier has applied and is waiting
+# for commit.
+# The point in this scenario is to verify that replayer does not try to abort
+# the latter applier
+#################################################################################
+
+--echo test scenario 2
+
+--connection node_1
+--let $expected_wsrep_local_replays = `SELECT VARIABLE_VALUE+1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays'`
+
+CREATE TABLE t1 (f1 INTEGER PRIMARY KEY, f2 int, f3 int,  unique key keyj (f2));
+INSERT INTO t1 VALUES (1, 1, 0);
+INSERT INTO t1 VALUES (3, 3, 0);
+INSERT INTO t1 VALUES (10, 10, 0);
+
+# we will need 2 appliers threads for applyin two writes sets in parallel in node1
+# and 1 applier thread for handling replaying 
+SET GLOBAL wsrep_slave_threads = 3;
+
+# set sync point for the first INSERT applier
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_set_sync_point.inc
+
+--connection node_1
+# starting a transaction, which deletes and inserts the middle row in test table
+# this will be victim of false positive conflict with appliers
+SET SESSION wsrep_sync_wait=0;
+START TRANSACTION;
+
+DELETE FROM t1 WHERE f2 = 3;
+INSERT INTO t1 VALUES (3, 3, 1);
+
+# Control connection to manage sync points for appliers
+--connection node_1a
+SET SESSION wsrep_sync_wait=0;
+
+# send from node 2 first an INSERT transaction,  which will conflict on GAP lock in node 1
+--connection node_2
+INSERT INTO t1 VALUES (5, 5, 2);
+
+--connection node_1a
+# wait to see the INSERT in apply_cb sync point
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# Block the local commit, send the COMMIT and wait until it gets blocked
+--let $galera_sync_point = commit_monitor_master_enter_sync
+--source include/galera_set_sync_point.inc
+
+--connection node_1
+--send COMMIT
+
+--connection node_1a
+# wait for the local commit to enter in commit monitor wait state
+--let $galera_sync_point = apply_monitor_slave_enter_sync commit_monitor_master_enter_sync
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# set sync point before replaying
+SET GLOBAL DEBUG_DBUG = "d,sync.wsrep_replay_cb";
+
+# release the local transaction to continue with commit
+# it should advance and end up waiting in commit monitor for his turn
+--let $galera_sync_point = commit_monitor_master_enter_sync
+--source include/galera_signal_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# and now release the first applier, it should force local trx to abort
+--let $galera_sync_point = apply_monitor_slave_enter_sync
+--source include/galera_signal_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# wait for BF abort to happen and replaying begin
+--let $wait_condition = SELECT VARIABLE_VALUE= $expected_wsrep_local_replays FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
+--let $wait_condition_on_error_output= SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_local_replays';
+--source include/wait_condition_with_debug.inc
+
+# replayer should now be in stopped in sync point
+SET SESSION DEBUG_SYNC = "now WAIT_FOR sync.wsrep_replay_cb_reached";
+
+# set sync point for the second INSERT
+--let $galera_sync_point = commit_monitor_slave_enter_sync
+--source include/galera_set_sync_point.inc
+
+--connection node_2
+# send second insert into same GAP in test table
+INSERT INTO t1 VALUES (4, 4, 2);
+
+--connection node_1a
+# wait for the second applier to enter in commit monitor wait state
+--let $galera_sync_point = commit_monitor_slave_enter_sync
+--source include/galera_wait_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# and, letting the second appier to move forward, it will stop naturally
+# to wait for commit order after replayer's commit
+--let $galera_sync_point = commit_monitor_slave_enter_sync
+--source include/galera_signal_sync_point.inc
+--source include/galera_clear_sync_point.inc
+
+# and now release the replayer, if all is good,it will commit before the second applier
+SET GLOBAL DEBUG_DBUG = "";
+SET DEBUG_SYNC = "now SIGNAL signal.wsrep_replay_cb";
+SET GLOBAL debug_dbug = NULL;
+SET debug_sync='RESET';
+
+# local commit should succeed
 --connection node_1
+--reap
+
+--let $wait_condition = SELECT COUNT(*)=5 FROM t1;
+--source include/wait_condition.inc
+
+# returning original slave thread count
+SET GLOBAL wsrep_slave_threads = DEFAULT;
+
+--connection node_2
 SELECT * FROM t1;
 
+# replicate some transactions, so that wsrep slave thread count can reach
+# original state in node 1
+INSERT INTO t1 VALUES (7,7,7);
+INSERT INTO t1 VALUES (8,8,8);
+
 DROP TABLE t1;
diff --git a/sql/wsrep_high_priority_service.cc b/sql/wsrep_high_priority_service.cc
index 1adbb312ac0..0da71c3eda5 100644
--- a/sql/wsrep_high_priority_service.cc
+++ b/sql/wsrep_high_priority_service.cc
@@ -673,6 +673,17 @@ int Wsrep_replayer_service::apply_write_set(const wsrep::ws_meta& ws_meta,
   DBUG_ASSERT(thd->wsrep_trx().active());
   DBUG_ASSERT(thd->wsrep_trx().state() == wsrep::transaction::s_replaying);
 
+  /* Allow tests to block the replayer thread using the DBUG facilities */
+  DBUG_EXECUTE_IF("sync.wsrep_replay_cb",
+                 {
+                   const char act[]=
+                     "now "
+                     "SIGNAL sync.wsrep_replay_cb_reached "
+                     "WAIT_FOR signal.wsrep_replay_cb";
+                   DBUG_ASSERT(!debug_sync_set_action(thd,
+                                                      STRING_WITH_LEN(act)));
+                 };);
+
   wsrep_setup_uk_and_fk_checks(thd);
 
   int ret= 0;