MDEV-17062: Test failure on galera.MW-336bb-10.3-MDEV-17058

MDEV-17058: Test failure on wsrep.variables
MDEV-17060: Test failure on galera.galera_var_slave_threads

Fix incorrect calculation of increased applier (slave) threads.
Note that increase change takes effect "immediately" but we should
use proper wait condition to wait it. Reducing the number of
slave threads is not immediate as thread will only exit after a
replication event.

4 files changed, 95 insertions, 49 deletions

diff --git a/mysql-test/suite/galera/r/MW-336.result b/mysql-test/suite/galera/r/MW-336.result
index 0bf8d9d3909..fa5514dfc47 100644
--- a/mysql-test/suite/galera/r/MW-336.result
+++ b/mysql-test/suite/galera/r/MW-336.result
@@ -2,39 +2,76 @@ CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
 connection node_1;
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 1;
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+COUNT(*)
+11
+SHOW STATUS LIKE 'wsrep_thread_count';
+Variable_name	Value
+wsrep_thread_count	11
 connection node_2;
 INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
 connection node_1;
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 20;
 SET GLOBAL wsrep_slave_threads = 1;
 connection node_2;
 INSERT INTO t1 VALUES (1);
-INSERT INTO t1 VALUES (2);
-INSERT INTO t1 VALUES (3);
-INSERT INTO t1 VALUES (4);
-INSERT INTO t1 VALUES (5);
-INSERT INTO t1 VALUES (6);
-INSERT INTO t1 VALUES (7);
-INSERT INTO t1 VALUES (8);
-INSERT INTO t1 VALUES (9);
-connection node_1;
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 0;
 Warnings:
 Warning	1292	Truncated incorrect wsrep_slave_threads value: '0'
 connection node_2;
-INSERT INTO t1 VALUES (10);
-INSERT INTO t1 VALUES (11);
-INSERT INTO t1 VALUES (12);
-INSERT INTO t1 VALUES (13);
-INSERT INTO t1 VALUES (14);
-INSERT INTO t1 VALUES (15);
-INSERT INTO t1 VALUES (16);
-INSERT INTO t1 VALUES (17);
-INSERT INTO t1 VALUES (18);
-INSERT INTO t1 VALUES (19);
-INSERT INTO t1 VALUES (20);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
+INSERT INTO t1 VALUES (1);
 connection node_1;
 SET GLOBAL wsrep_slave_threads = 1;
 DROP TABLE t1;
diff --git a/mysql-test/suite/galera/t/MW-336.test b/mysql-test/suite/galera/t/MW-336.test
index 8cd363aa019..7af039e577b 100644
--- a/mysql-test/suite/galera/t/MW-336.test
+++ b/mysql-test/suite/galera/t/MW-336.test
@@ -8,15 +8,28 @@
 CREATE TABLE t1 (f1 INTEGER) Engine=InnoDB;
 
 --connection node_1
+
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 1;
+
+--let $wait_timeout=600
 --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
 --source include/wait_condition.inc
 
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
+SHOW STATUS LIKE 'wsrep_thread_count';
+
 --connection node_2
-INSERT INTO t1 VALUES (1);
+# Generate 11 replication events
+--let $count = 11
+while ($count)
+{
+  INSERT INTO t1 VALUES (1);
+  --dec $count
+}
 
 --connection node_1
+
 SET GLOBAL wsrep_slave_threads = 10;
 --let $wait_condition = SELECT COUNT(*) = 11 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
 --source include/wait_condition.inc
@@ -28,36 +41,25 @@ SET GLOBAL wsrep_slave_threads = 20;
 SET GLOBAL wsrep_slave_threads = 1;
 
 --connection node_2
-INSERT INTO t1 VALUES (1);
-INSERT INTO t1 VALUES (2);
-INSERT INTO t1 VALUES (3);
-INSERT INTO t1 VALUES (4);
-INSERT INTO t1 VALUES (5);
-INSERT INTO t1 VALUES (6);
-INSERT INTO t1 VALUES (7);
-INSERT INTO t1 VALUES (8);
-INSERT INTO t1 VALUES (9);
-
-
---connection node_1
---let $wait_condition = SELECT COUNT(*) = 12 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
---source include/wait_condition.inc
+# Generate 21 replication events
+--let $count = 21
+while ($count)
+{
+  INSERT INTO t1 VALUES (1);
+  --dec $count
+}
 
 SET GLOBAL wsrep_slave_threads = 10;
 SET GLOBAL wsrep_slave_threads = 0;
 
 --connection node_2
-INSERT INTO t1 VALUES (10);
-INSERT INTO t1 VALUES (11);
-INSERT INTO t1 VALUES (12);
-INSERT INTO t1 VALUES (13);
-INSERT INTO t1 VALUES (14);
-INSERT INTO t1 VALUES (15);
-INSERT INTO t1 VALUES (16);
-INSERT INTO t1 VALUES (17);
-INSERT INTO t1 VALUES (18);
-INSERT INTO t1 VALUES (19);
-INSERT INTO t1 VALUES (20);
+# Generate 21 replication events
+--let $count = 21
+while ($count)
+{
+  INSERT INTO t1 VALUES (1);
+  --dec $count
+}
 
 --connection node_1
 --let $wait_condition = SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND (STATE IS NULL OR STATE NOT LIKE 'InnoDB%');
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
index 1315f090d5c..40b869422ff 100644
--- a/mysql-test/suite/wsrep/t/variables.test
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -101,8 +101,10 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 --echo # Setting wsrep_cluster_address triggers the creation of
 --echo # applier/rollbacker threads.
 SET GLOBAL wsrep_cluster_address= 'gcomm://';
+
 --echo # Wait for applier threads to get created.
-sleep 3; 
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+--source include/wait_condition.inc
 
 --replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
@@ -113,8 +115,11 @@ SHOW STATUS LIKE 'wsrep_thread_count';
 
 SET @wsrep_slave_threads_saved= @@global.wsrep_slave_threads;
 SET GLOBAL wsrep_slave_threads= 10;
+
 --echo # Wait for applier threads to get created.
-sleep 3; 
+--let $wait_condition = SELECT VARIABLE_VALUE = 11 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_thread_count';
+--source include/wait_condition.inc
+
 SHOW STATUS LIKE 'threads_connected';
 SHOW STATUS LIKE 'wsrep_thread_count';
 
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
index 7fe120beb8e..aaae487b2e2 100644
--- a/sql/wsrep_var.cc
+++ b/sql/wsrep_var.cc
@@ -594,7 +594,9 @@ void wsrep_node_address_init (const char* value)
 
 static void wsrep_slave_count_change_update ()
 {
-  wsrep_slave_count_change += (wsrep_slave_threads - wsrep_prev_slave_threads);
+  wsrep_slave_count_change = (wsrep_slave_threads - wsrep_prev_slave_threads);
+  WSREP_DEBUG("Change on slave threads: New %lu old %lu difference %lu",
+	  wsrep_slave_threads, wsrep_prev_slave_threads, wsrep_slave_count_change);
   wsrep_prev_slave_threads = wsrep_slave_threads;
 }