Make test galera_parallel_apply_3nodes deterministic

Test galera_parallel_apply_3nodes started to failed occasionally.
The test assumes that one round of autocommit retry is sufficient in
order to avoid a deadlock error when two conflicting UPDATE statements
run concurrently.
This assumption no longer holds after galera library has changed
last_committed() to return the seqno of the last transaction that left
apply monitor, rather than commit monitor. So it is possible that
after a BF abort, a command is re-executed before it's BF abortee has
left the apply monitor. Thus causing another retry or a deadlock error.

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

2 files changed, 59 insertions, 26 deletions

diff --git a/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result b/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result
index 2acfe56bede..087f9e7e279 100644
--- a/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result
+++ b/mysql-test/suite/galera_3nodes/r/galera_parallel_apply_3nodes.result
@@ -1,27 +1,35 @@
 connection node_2;
 connection node_1;
+connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3;
+connect node_1_ctrl, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 connection node_3;
 SET GLOBAL wsrep_slave_threads = 2;
+connection node_1_ctrl;
+SET SESSION wsrep_sync_wait=0;
 connection node_1;
+SET DEBUG_SYNC = 'wsrep_before_certification SIGNAL before_cert WAIT_FOR continue';
 UPDATE t1 SET f1 = f1 + 10;;
+connection node_1_ctrl;
+SET DEBUG_SYNC = 'now WAIT_FOR before_cert';
+SET GLOBAL debug_dbug = '+d,sync.wsrep_retry_autocommit';
 connection node_2;
 UPDATE t1 SET f1 = f1 + 100;;
+connection node_1_ctrl;
+SET DEBUG_SYNC = 'now WAIT_FOR wsrep_retry_autocommit_reached';
+SET GLOBAL debug_dbug = NULL;
+SET DEBUG_SYNC = 'now SIGNAL wsrep_retry_autocommit_continue';
 connection node_1;
-SELECT COUNT(*) FROM t1;
-COUNT(*)
-1
 connection node_2;
-SELECT COUNT(*) FROM t1;
-COUNT(*)
-1
 connection node_3;
-SELECT COUNT(*) FROM t1;
-COUNT(*)
+SELECT f1 = 111 FROM t1;
+f1 = 111
 1
 SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%committed%';
 COUNT(*) IN (1, 2)
 1
-SET GLOBAL wsrep_slave_threads = 1;;
+SET GLOBAL wsrep_slave_threads = DEFAULT;
 DROP TABLE t1;
+connection node_1;
+SET DEBUG_SYNC= 'RESET';
diff --git a/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test b/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test
index f1168e59193..84629c96c65 100644
--- a/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test
+++ b/mysql-test/suite/galera_3nodes/t/galera_parallel_apply_3nodes.test
@@ -5,46 +5,71 @@
 
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
+--source include/have_debug.inc
+--source include/have_debug_sync.inc
 
---let $galera_connection_name = node_3
---let $galera_server_number = 3
---source include/galera_connect.inc
+--connect node_3, 127.0.0.1, root, , test, $NODE_MYPORT_3
+--connect node_1_ctrl, 127.0.0.1, root, , test, $NODE_MYPORT_1
 
 CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 
+--let $wsrep_last_committed_before = `SELECT VARIABLE_VALUE FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'`
+
 --connection node_3
---let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
 SET GLOBAL wsrep_slave_threads = 2;
 
+--connection node_1_ctrl
+SET SESSION wsrep_sync_wait=0;
+
+#
+# We will make the following UPDATE depend on the UPDATE below
+#
 --connection node_1
+SET DEBUG_SYNC = 'wsrep_before_certification SIGNAL before_cert WAIT_FOR continue';
 --send UPDATE t1 SET f1 = f1 + 10;
 
+--connection node_1_ctrl
+SET DEBUG_SYNC = 'now WAIT_FOR before_cert';
+SET GLOBAL debug_dbug = '+d,sync.wsrep_retry_autocommit';
+
 --connection node_2
 --send UPDATE t1 SET f1 = f1 + 100;
 
---connection node_1
 #
-# Note that test is not deterministic. We have following cases possible
-# (1) Both updates are certified locally and then executed by the applier
-# (2) Certification of update in node_1 fails because applier has started
-#     update from node_2
-# (3) Certification of update in node_2 fails because applier has started
-#     update from node_1
+# Let's wait for the first UPDATE the be BF aborted
+#
+--connection node_1_ctrl
+SET DEBUG_SYNC = 'now WAIT_FOR wsrep_retry_autocommit_reached';
+
+#
+# and make sure the second has committed
 #
---error 0,ER_LOCK_DEADLOCK
+--let $wait_condition = SELECT VARIABLE_VALUE > $wsrep_last_committed_before FROM INFORMATION_SCHEMA.SESSION_STATUS WHERE VARIABLE_NAME = 'wsrep_last_committed'
+--source include/wait_condition.inc
+
+#
+# now release the first UPDATE.
+#
+SET GLOBAL debug_dbug = NULL;
+SET DEBUG_SYNC = 'now SIGNAL wsrep_retry_autocommit_continue';
+
+#
+# Both UPDATEs should succeed.
+#
+--connection node_1
 --reap
-SELECT COUNT(*) FROM t1;
 
 --connection node_2
---error 0,ER_LOCK_DEADLOCK
 --reap
-SELECT COUNT(*) FROM t1;
 
 --connection node_3
-SELECT COUNT(*) FROM t1;
+SELECT f1 = 111 FROM t1;
 SELECT COUNT(*) IN (1, 2) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%committed%';
 
---eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig;
+SET GLOBAL wsrep_slave_threads = DEFAULT;
 
 DROP TABLE t1;
+
+--connection node_1
+SET DEBUG_SYNC= 'RESET';