MDEV-14311 wsrep.variables fails in buildbot with wrong resultbb-10.1-14311

 Removal of older code around wsrep_prev_slave_threads in
 wsrep_slave_count_change_update.

 Patch Credit:- Seppo

6 files changed, 261 insertions, 18 deletions

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index b67dea0f2f4..1007ad414c5 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -20,7 +20,6 @@ galera_binlog_rows_query_log_events: MariaDB does not support binlog_rows_query_
 GAL-419 : MDEV-13549 Galera test failures
 galera_toi_ddl_fk_insert : MDEV-13549 Galera test failures
 galera_var_notify_cmd : MDEV-13549 Galera test failures
-galera_var_slave_threads : MDEV-13549 Galera test failures
 mysql-wsrep#90 : MDEV-13549 Galera test failures
 galera_as_master_gtid : Requires MySQL GTID
 galera_as_master_gtid_change_master : Requires MySQL GTID
diff --git a/mysql-test/suite/galera/r/galera_var_slave_threads.result b/mysql-test/suite/galera/r/galera_var_slave_threads.result
index 2340d25d160..2d2ac726a76 100644
--- a/mysql-test/suite/galera/r/galera_var_slave_threads.result
+++ b/mysql-test/suite/galera/r/galera_var_slave_threads.result
@@ -11,12 +11,14 @@ SELECT @@wsrep_slave_threads = 1;
 @@wsrep_slave_threads = 1
 1
 SET GLOBAL wsrep_slave_threads = 1;
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
-COUNT(*) = 2
-1
 SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 COUNT(*) = 1
 1
+INSERT INTO t1 VALUES (100);
+SELECT COUNT(*) = 1 FROM t1;
+COUNT(*) = 1
+1
+truncate table t1;
 SET GLOBAL wsrep_slave_threads = 64;
 INSERT INTO t1 VALUES (1);
 SELECT COUNT(*) = 1 FROM t1;
@@ -96,13 +98,159 @@ INSERT INTO t2 VALUES (DEFAULT);
 SELECT COUNT(*) = 64 FROM t2;
 COUNT(*) = 64
 1
+SET wsrep_sync_wait=0;
 SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
 COUNT(*) = @@wsrep_slave_threads + 1
 1
 SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 COUNT(*) = 1
 1
+SET GLOBAL wsrep_slave_threads = 5;
+SET GLOBAL wsrep_slave_threads = 1;
+Shutting down server ...
+SET wsrep_sync_wait=0;
+show status like 'wsrep_cluster_size';
+Variable_name	Value
+wsrep_cluster_size	1
+SET GLOBAL wsrep_slave_threads = 6;
+SET GLOBAL wsrep_slave_threads = 1;
+SET GLOBAL wsrep_cluster_address='';
+SET GLOBAL wsrep_cluster_address='gcomm://';
+SET GLOBAL wsrep_slave_threads = 10;
+SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
+COUNT(*) = @@wsrep_slave_threads + 1
+1
 SET GLOBAL wsrep_slave_threads = 1;
+SET GLOBAL wsrep_slave_threads = 1;
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+SET GLOBAL wsrep_slave_threads = 1;
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
+INSERT INTO t2 VALUES (DEFAULT);
 DROP TABLE t1;
 DROP TABLE t2;
 #
diff --git a/mysql-test/suite/galera/t/galera_var_slave_threads.test b/mysql-test/suite/galera/t/galera_var_slave_threads.test
index 5e56800c5d6..bda21ffd491 100644
--- a/mysql-test/suite/galera/t/galera_var_slave_threads.test
+++ b/mysql-test/suite/galera/t/galera_var_slave_threads.test
@@ -6,6 +6,11 @@
 
 --source include/galera_cluster.inc
 --source include/have_innodb.inc
+# Auto Increment
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
 CALL mtr.add_suppression("WSREP: Refusing exit for the last slave thread.");
 --let $wsrep_slave_threads_orig = `SELECT @@wsrep_slave_threads`
 
@@ -22,8 +27,13 @@ SELECT @@wsrep_slave_threads = 1;
 
 SET GLOBAL wsrep_slave_threads = 1;
 # There is a separate wsrep_aborter thread at all times
-SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
 SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
+--connection node_1
+INSERT INTO t1 VALUES (100);
+
+--connection node_2
+SELECT COUNT(*) = 1 FROM t1;
+truncate table t1;
 
 #
 # Increase the number of slave threads. The change takes effect immediately
@@ -59,13 +69,104 @@ while ($count)
 
 --connection node_2
 SELECT COUNT(*) = 64 FROM t2;
+SET wsrep_sync_wait=0;
 
 SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
 SELECT COUNT(*) = 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user' AND STATE LIKE '%wsrep aborter%';
 
 
+--let $wsrep_cluster_address_node2 = `SELECT @@wsrep_cluster_address`
+--let $wsrep_provider_node2 = `SELECT @@wsrep_provider`
+
+SET GLOBAL wsrep_slave_threads = 5;
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
+SET GLOBAL wsrep_slave_threads = 1;
+
+#
+# test phase for bug https://github.com/codership/mysql-wsrep/issues/319
+#
+
+# shutdown node 2
+--connection node_2
+--echo Shutting down server ...
+--source include/shutdown_mysqld.inc
+
+# wait until node_1 is ready as one node cluster
+--connection node_1
+SET wsrep_sync_wait=0;
+--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'
+--source include/wait_condition.inc
+show status like 'wsrep_cluster_size';
+
+# step up slave threads to 6, and make sure all appliers and rollbacker thread are running
+SET GLOBAL wsrep_slave_threads = 6;
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
+# change to invalid cluster address
+SET GLOBAL wsrep_slave_threads = 1;
+SET GLOBAL wsrep_cluster_address='';
+
+# join back to single node cluster
+SET GLOBAL wsrep_cluster_address='gcomm://';
+--source include/wait_until_connected_again.inc
+
+# we should have 1 applier thread now
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
+# test if we can increase applier count now (fails in bug #319)
+SET GLOBAL wsrep_slave_threads = 10;
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
+# restart node 2
+--connection node_2
+--source include/start_mysqld.inc
+--source include/wait_until_connected_again.inc
+
+SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user';
+
+# 
+#
+# cleanup to original state
+#
+--connection node_1
+--eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig
+
+--connection node_2
+--eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig
+
+# Generate 64 replication events, to help node 1 to purge excessive applier threads
+--let $count = 64
+while ($count)
+{
+  INSERT INTO t2 VALUES (DEFAULT);
+  --dec $count
+}
+
+--connection node_1
 --eval SET GLOBAL wsrep_slave_threads = $wsrep_slave_threads_orig
 
+# Generate 64 replication events, to help node 2 to purge excessive applier threads
+--let $count = 64
+while ($count)
+{
+  INSERT INTO t2 VALUES (DEFAULT);
+  --dec $count
+}
+
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
+
+--connection node_2
+
+--let $wait_condition = SELECT COUNT(*) = @@wsrep_slave_threads + 1 FROM INFORMATION_SCHEMA.PROCESSLIST WHERE USER = 'system user'
+--source include/wait_condition.inc
+
 DROP TABLE t1;
 DROP TABLE t2;
 
@@ -81,4 +182,7 @@ SET GLOBAL wsrep_slave_threads = 4;
 SET GLOBAL wsrep_slave_threads = 1;
 DROP TABLE t1;
 
+# Restore original auto_increment_offset values.
+--source include/auto_increment_offset_restore.inc
+
 --echo # End of tests
diff --git a/mysql-test/suite/wsrep/r/variables.result b/mysql-test/suite/wsrep/r/variables.result
index 62d7f62440f..f74ef3d078f 100644
--- a/mysql-test/suite/wsrep/r/variables.result
+++ b/mysql-test/suite/wsrep/r/variables.result
@@ -19,7 +19,6 @@ SET GLOBAL wsrep_provider=none;
 # variables when using "_"
 #
 CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*");
-SHOW GLOBAL STATUS LIKE 'wsrep%';
 Variable_name	Value
 wsrep_apply_oooe	#
 wsrep_apply_oool	#
@@ -36,6 +35,7 @@ wsrep_commit_oooe	#
 wsrep_commit_oool	#
 wsrep_commit_window	#
 wsrep_connected	#
+wsrep_debug_sync_waiters	#
 wsrep_flow_control_paused	#
 wsrep_flow_control_paused_ns	#
 wsrep_flow_control_recv	#
@@ -74,7 +74,6 @@ wsrep_replicated	#
 wsrep_replicated_bytes	#
 wsrep_thread_count	#
 
-SHOW GLOBAL STATUS LIKE 'wsrep_%';
 Variable_name	Value
 wsrep_apply_oooe	#
 wsrep_apply_oool	#
@@ -91,6 +90,7 @@ wsrep_commit_oooe	#
 wsrep_commit_oool	#
 wsrep_commit_window	#
 wsrep_connected	#
+wsrep_debug_sync_waiters	#
 wsrep_flow_control_paused	#
 wsrep_flow_control_paused_ns	#
 wsrep_flow_control_recv	#
@@ -128,13 +128,10 @@ wsrep_repl_other_bytes	#
 wsrep_replicated	#
 wsrep_replicated_bytes	#
 wsrep_thread_count	#
-SHOW GLOBAL STATUS LIKE 'wsrep_local_state_comment';
 Variable_name	Value
 wsrep_local_state_comment	#
 # Should show nothing.
-SHOW STATUS LIKE 'x';
 Variable_name	Value
-SET GLOBAL wsrep_provider=none;
 #
 # MDEV#6079: xtrabackup SST failing with maria-10.0-galera
 #
diff --git a/mysql-test/suite/wsrep/t/variables.test b/mysql-test/suite/wsrep/t/variables.test
index 1769f567848..4301c9d123c 100644
--- a/mysql-test/suite/wsrep/t/variables.test
+++ b/mysql-test/suite/wsrep/t/variables.test
@@ -29,9 +29,6 @@ CALL mtr.add_suppression("WSREP: Could not open saved state file for reading.*")
 
 --disable_query_log
 eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
---let $galera_version=25.3.17
-source include/check_galera_version.inc;
---enable_query_log
 
 --replace_column 2 #
 SHOW GLOBAL STATUS LIKE 'wsrep%';
@@ -78,7 +75,7 @@ call mtr.add_suppression("WSREP: Failed to get provider options");
 --disable_query_log
 eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
 --enable_query_log
-
+--sleep 4
 --replace_regex /.*libgalera_smm.*/libgalera_smm.so/
 SELECT @@global.wsrep_provider;
 SELECT @@global.wsrep_slave_threads;
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
index ad1f4ec0eac..6ed4eafb059 100644
--- a/sql/wsrep_var.cc
+++ b/sql/wsrep_var.cc
@@ -35,8 +35,6 @@ const  char* wsrep_node_address     = 0;
 const  char* wsrep_node_incoming_address = 0;
 const  char* wsrep_start_position   = 0;
 
-static long wsrep_prev_slave_threads = wsrep_slave_threads;
-
 int wsrep_init_vars()
 {
   wsrep_provider        = my_strdup(WSREP_NONE, MYF(MY_WME));
@@ -539,8 +537,8 @@ void wsrep_node_address_init (const char* value)
 
 static void wsrep_slave_count_change_update ()
 {
-  wsrep_slave_count_change += (wsrep_slave_threads - wsrep_prev_slave_threads);
-  wsrep_prev_slave_threads = wsrep_slave_threads;
+  //wsrep running threads = applier thread + 1 rollback thread
+  wsrep_slave_count_change = (wsrep_slave_threads - wsrep_running_threads + 1);
 }
 
 bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type)