Fix bad galera tests

* galera_kill_applier : we should make sure that node has
  correct number of wsrep appliers
* galera_bad_wsrep_new_cluster: This test restarts both nodes,
so it is bad on mtr. Make sure it is run alone
* galera_update_limit : Make sure we have PK when needed
galera_as_slave_replay : bf abort was not consistent
* galera_unicode_pk : Add wait_conditions so that all nodes
are part of cluster and DDL and INSERT has replicated before
any further operations are done.
* galera_bf_abort_at_after_statement : Add wait_conditions
to make sure all nodes are part of cluster and that DDL
and INSERT has replicated. Make sure we reset DEBUG_SYNC.

18 files changed, 100 insertions, 112 deletions

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index e4210734f5e..f1eb959e5bd 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -37,3 +37,5 @@ mysql-wsrep#198 : MDEV-24446: galera.mysql-wsrep#198 MTR failed: query 'reap' fa
 partition : MDEV-19958 Galera test failure on galera.partition
 query_cache: MDEV-15805 Test failure on galera.query_cache
 versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch
+galera_ssl_mode_server : Certificate CA mismatch
+galera_bf_abort_at_after_statement : Unstable
diff --git a/mysql-test/suite/galera/r/galera_as_slave_replay.result b/mysql-test/suite/galera/r/galera_as_slave_replay.result
index d81795eeed9..05bfeed8f01 100644
--- a/mysql-test/suite/galera/r/galera_as_slave_replay.result
+++ b/mysql-test/suite/galera/r/galera_as_slave_replay.result
@@ -60,44 +60,6 @@ f1	f2
 2	b
 3	c
 SET DEBUG_SYNC = "RESET";
-#
-# test phase with real abort
-#
-connection node_3;
-set binlog_format=ROW;
-insert into t1 values (4, 'd');
-SET AUTOCOMMIT=ON;
-START TRANSACTION;
-UPDATE t1 SET f2 = 'd' WHERE f1 = 3;
-connection node_2a;
-SET GLOBAL wsrep_provider_options = 'dbug=d,commit_monitor_enter_sync';
-SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
-connection node_1;
-UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3;
-connection node_3;
-COMMIT;
-connection node_2a;
-SET GLOBAL debug_dbug = "";
-SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
-connection node_2a;
-SET GLOBAL wsrep_provider_options = 'dbug=';
-SET GLOBAL wsrep_provider_options = 'signal=commit_monitor_enter_sync';
-SET DEBUG_SYNC = "RESET";
-connection node_2a;
-set session wsrep_sync_wait=0;
-SELECT * from test.t1;
-f1	f2
-1	a
-2	b
-3	e
-4	d
-connection node_1;
-SELECT * from test.t1;
-f1	f2
-1	a
-2	b
-3	e
-4	d
 connection node_2a;
 STOP SLAVE;
 RESET SLAVE;
@@ -105,3 +67,6 @@ DROP TABLE t1;
 connection node_3;
 DROP TABLE t1;
 RESET MASTER;
+connection node_1;
+disconnect node_2a;
+disconnect node_3;
diff --git a/mysql-test/suite/galera/r/galera_bf_abort_at_after_statement.result b/mysql-test/suite/galera/r/galera_bf_abort_at_after_statement.result
index e44a925baf4..f6f0803227f 100644
--- a/mysql-test/suite/galera/r/galera_bf_abort_at_after_statement.result
+++ b/mysql-test/suite/galera/r/galera_bf_abort_at_after_statement.result
@@ -4,6 +4,7 @@ connection node_1;
 CREATE TABLE t1 (id INT PRIMARY KEY, val INT);
 INSERT INTO t1 VALUES (1, 1);
 connection node_2;
+SET DEBUG_SYNC = 'RESET';
 START TRANSACTION;
 SET DEBUG_SYNC = 'wsrep_after_statement_enter SIGNAL blocked WAIT_FOR continue';
 UPDATE t1 SET val=2 WHERE id=1;
@@ -18,4 +19,5 @@ ERROR 40001: Deadlock found when trying to get lock; try restarting transaction
 COMMIT;
 SET DEBUG_SYNC = 'RESET';
 connection node_1;
+SET DEBUG_SYNC = 'RESET';
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/r/galera_kill_applier.result b/mysql-test/suite/galera/r/galera_kill_applier.result
index 25600744d73..78a91d2638f 100644
--- a/mysql-test/suite/galera/r/galera_kill_applier.result
+++ b/mysql-test/suite/galera/r/galera_kill_applier.result
@@ -1,20 +1,26 @@
 connection node_2;
 connection node_1;
 connection node_2;
+SELECT @@wsrep_slave_threads;
+@@wsrep_slave_threads
+1
 SET GLOBAL wsrep_slave_threads=2;
 Got one of the listed errors
 Got one of the listed errors
 Got one of the listed errors
 Got one of the listed errors
-SET GLOBAL wsrep_slave_threads=1;
+SET GLOBAL wsrep_slave_threads=DEFAULT;
 connection node_1;
 create table t1(a int not null primary key) engine=innodb;
 insert into t1 values (1);
 insert into t1 values (2);
 connection node_2;
-set global wsrep_sync_wait=15;
+# Wait until one of the appliers has exited
 select count(*) from t1;
 count(*)
 2
+SELECT @@wsrep_slave_threads;
+@@wsrep_slave_threads
+1
 connection node_1;
 drop table t1;
diff --git a/mysql-test/suite/galera/r/galera_unicode_pk.result b/mysql-test/suite/galera/r/galera_unicode_pk.result
index bb36fd4f369..8ed1a3a077e 100644
--- a/mysql-test/suite/galera/r/galera_unicode_pk.result
+++ b/mysql-test/suite/galera/r/galera_unicode_pk.result
@@ -1,5 +1,6 @@
 connection node_2;
 connection node_1;
+connection node_1;
 CREATE TABLE t1 (
 f1 VARCHAR(255) PRIMARY KEY
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
diff --git a/mysql-test/suite/galera/r/galera_update_limit.result b/mysql-test/suite/galera/r/galera_update_limit.result
index 30c89a38dff..bd021136ac2 100644
--- a/mysql-test/suite/galera/r/galera_update_limit.result
+++ b/mysql-test/suite/galera/r/galera_update_limit.result
@@ -1,9 +1,9 @@
 connection node_2;
 connection node_1;
 connection node_1;
-CREATE TABLE ten (f1 INTEGER) Engine=InnoDB;
+CREATE TABLE ten (f1 INTEGER not null primary key) Engine=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) Engine=InnoDB;
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) Engine=InnoDB;
 INSERT INTO t1 SELECT f1 FROM ten ORDER BY RAND();
 connection node_2;
 UPDATE IGNORE t1 SET f1 = FLOOR(1 + (RAND() * 10)) ORDER BY RAND() LIMIT 5;
diff --git a/mysql-test/suite/galera/t/galera_as_slave_replay.cnf b/mysql-test/suite/galera/t/galera_as_slave_replay.cnf
index b1f9d7e9cbd..c20c65f86d9 100644
--- a/mysql-test/suite/galera/t/galera_as_slave_replay.cnf
+++ b/mysql-test/suite/galera/t/galera_as_slave_replay.cnf
@@ -5,7 +5,9 @@ binlog-format=row
 
 [mysqld.1]
 wsrep_restart_slave=1
+wsrep-debug=1
 
 [mysqld.2]
 wsrep_restart_slave=1
+wsrep-debug=1
 
diff --git a/mysql-test/suite/galera/t/galera_as_slave_replay.test b/mysql-test/suite/galera/t/galera_as_slave_replay.test
index 2e8f45a047b..725d72c4144 100644
--- a/mysql-test/suite/galera/t/galera_as_slave_replay.test
+++ b/mysql-test/suite/galera/t/galera_as_slave_replay.test
@@ -131,70 +131,6 @@ set session wsrep_sync_wait=0;
 SELECT *  FROM t1;
 SET DEBUG_SYNC = "RESET";
 
-#********************************************************************************
-#	test phase 2
-#********************************************************************************
-
---echo #
---echo # test phase with real abort
---echo #
-
---connection node_3
-
-set binlog_format=ROW;
-
-insert into t1 values (4, 'd');
-
-SET AUTOCOMMIT=ON;
-START TRANSACTION;
-
-UPDATE t1 SET f2 = 'd' WHERE f1 = 3;
-
---connection node_2a
-# wait for the last insert to be replicated from master
---let $wait_condition = SELECT COUNT(*) = 4 FROM test.t1;
---source include/wait_condition.inc
-
-# Block the commit
---let $galera_sync_point = commit_monitor_enter_sync
---source include/galera_set_sync_point.inc
-
-# block applier
-SET GLOBAL debug_dbug = "d,sync.wsrep_apply_cb";
-
-# Inject a conflicting update from node 3
---connection node_1
-UPDATE test.t1 SET f2 = 'e' WHERE f1 = 3;
-
-# send the update from master
---connection node_3
---error 0
-COMMIT;
-
---connection node_2a
-
-# release the applier
-SET GLOBAL debug_dbug = "";
-SET DEBUG_SYNC = "now SIGNAL signal.wsrep_apply_cb";
-
-
-# Unblock the async slave commit
---connection node_2a
---source include/galera_clear_sync_point.inc
---source include/galera_signal_sync_point.inc
-SET DEBUG_SYNC = "RESET";
-
---connection node_2a
-set session wsrep_sync_wait=0;
---let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1 where f2 = 'e'
---source include/wait_condition.inc
-SELECT * from test.t1;
-
---connection node_1
---let $wait_condition = SELECT COUNT(*) = 1 FROM test.t1 where f2 = 'e'
---source include/wait_condition.inc
-SELECT * from test.t1;
-
 --connection node_2a
 STOP SLAVE;
 RESET SLAVE;
@@ -204,3 +140,7 @@ DROP TABLE t1;
 --connection node_3
 DROP TABLE t1;
 RESET MASTER;
+
+--connection node_1
+--disconnect node_2a
+--disconnect node_3
diff --git a/mysql-test/suite/galera/t/galera_bad_wsrep_new_cluster.cnf b/mysql-test/suite/galera/t/galera_bad_wsrep_new_cluster.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bad_wsrep_new_cluster.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.cnf b/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.test b/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.test
index b1fe3e64cd8..738aedbda9e 100644
--- a/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.test
+++ b/mysql-test/suite/galera/t/galera_bf_abort_at_after_statement.test
@@ -8,10 +8,21 @@
 --source include/have_debug_sync.inc
 
 --connection node_1
+# Mare sure both nodes are in the cluster
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
 CREATE TABLE t1 (id INT PRIMARY KEY, val INT);
 INSERT INTO t1 VALUES (1, 1);
 
 --connection node_2
+SET DEBUG_SYNC = 'RESET';
+# Mare sure that DLL has replicated and insert has replicated
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1
+--source include/wait_condition.inc
+
 START TRANSACTION;
 SET DEBUG_SYNC = 'wsrep_after_statement_enter SIGNAL blocked WAIT_FOR continue';
 --send UPDATE t1 SET val=2 WHERE id=1
@@ -34,4 +45,5 @@ COMMIT;
 SET DEBUG_SYNC = 'RESET';
 
 --connection node_1
+SET DEBUG_SYNC = 'RESET';
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/galera_kill_applier.cnf b/mysql-test/suite/galera/t/galera_kill_applier.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_kill_applier.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_kill_applier.test b/mysql-test/suite/galera/t/galera_kill_applier.test
index 4136bac5dc6..3a285822613 100644
--- a/mysql-test/suite/galera/t/galera_kill_applier.test
+++ b/mysql-test/suite/galera/t/galera_kill_applier.test
@@ -7,9 +7,10 @@
 --source include/force_restart.inc
 
 --connection node_2
+SELECT @@wsrep_slave_threads;
 SET GLOBAL wsrep_slave_threads=2;
 
---let $wait_condition = SELECT COUNT(*) >= 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep applier idle';
+--let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep applier idle';
 --let $wait_condition_on_error_output = SELECT COUNT(*), 2 as EXPECTED_VALUE FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep applier idle'; show processlist
 --source include/wait_condition_with_debug.inc
 
@@ -31,7 +32,7 @@ SET GLOBAL wsrep_slave_threads=2;
 --eval KILL QUERY $aborter_thread
 --enable_query_log
 
-SET GLOBAL wsrep_slave_threads=1;
+SET GLOBAL wsrep_slave_threads=DEFAULT;
 
 --connection node_1
 create table t1(a int not null primary key) engine=innodb;
@@ -39,8 +40,13 @@ insert into t1 values (1);
 insert into t1 values (2);
 
 --connection node_2
-set global wsrep_sync_wait=15;
+--echo # Wait until one of the appliers has exited
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_applier_thread_count';
+--let $wait_condition_on_error_output = SELECT COUNT(*), 1 as EXPECTED_VALUE FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE = 'wsrep applier idle'; show processlist
+--source include/wait_condition_with_debug.inc
+
 select count(*) from t1;
+SELECT @@wsrep_slave_threads;
 
 --connection node_1
 drop table t1;
diff --git a/mysql-test/suite/galera/t/galera_sp_bf_abort.cnf b/mysql-test/suite/galera/t/galera_sp_bf_abort.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_sp_bf_abort.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_unicode_pk.cnf b/mysql-test/suite/galera/t/galera_unicode_pk.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_unicode_pk.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_unicode_pk.test b/mysql-test/suite/galera/t/galera_unicode_pk.test
index 0d571f5cfd7..2da9b1b9b8a 100644
--- a/mysql-test/suite/galera/t/galera_unicode_pk.test
+++ b/mysql-test/suite/galera/t/galera_unicode_pk.test
@@ -5,6 +5,11 @@
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
 
+--connection node_1
+# Mare sure both nodes are in the cluster
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
 CREATE TABLE t1 (
 	f1 VARCHAR(255) PRIMARY KEY
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@@ -12,6 +17,12 @@ CREATE TABLE t1 (
 INSERT INTO t1 VALUES ('текст');
 
 --connection node_2
+# Mare sure that DLL has replicated and insert has replicated
+--let $wait_condition = SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1'
+--source include/wait_condition.inc
+--let $wait_condition = SELECT COUNT(*) = 1 FROM t1
+--source include/wait_condition.inc
+
 SELECT f1 = 'текст' FROM t1;
 
 #
diff --git a/mysql-test/suite/galera/t/galera_update_limit.cnf b/mysql-test/suite/galera/t/galera_update_limit.cnf
new file mode 100644
index 00000000000..62cf1854032
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_update_limit.cnf
@@ -0,0 +1,7 @@
+!include ../galera_2nodes.cnf
+
+[mysqld.1]
+wsrep-debug=1
+
+[mysqld.2]
+wsrep-debug=1
diff --git a/mysql-test/suite/galera/t/galera_update_limit.test b/mysql-test/suite/galera/t/galera_update_limit.test
index baacf2a60b2..e2823bf7124 100644
--- a/mysql-test/suite/galera/t/galera_update_limit.test
+++ b/mysql-test/suite/galera/t/galera_update_limit.test
@@ -4,17 +4,16 @@
 #
 
 --source include/galera_cluster.inc
---source include/have_innodb.inc
 
 #
 # With a PK
 #
 
 --connection node_1
-CREATE TABLE ten (f1 INTEGER) Engine=InnoDB;
+CREATE TABLE ten (f1 INTEGER not null primary key) Engine=InnoDB;
 INSERT INTO ten VALUES (1),(2),(3),(4),(5),(6),(7),(8),(9),(10);
 
-CREATE TABLE t1 (f1 INTEGER PRIMARY KEY) Engine=InnoDB;
+CREATE TABLE t1 (f1 INTEGER NOT NULL PRIMARY KEY) Engine=InnoDB;
 INSERT INTO t1 SELECT f1 FROM ten ORDER BY RAND();
 
 --connection node_2