MDEV-19746: Galera test failures because of wsrep_slave_threads identification

Problem was that tests select INFORMATION_SCHEMA.PROCESSLIST processes
from user system user and empty state. Thus, there is not clear
state for slave threads.

Changes:
- Added new status variables that store current amount of applier threads
(wsrep_applier_thread_count) and rollbacker threads
(wsrep_rollbacker_thread_count). This will make clear how many slave threads
of certain type there is.
- Added THD state "wsrep applier idle" when applier slave thread is
waiting for work. This makes finding slave/applier threads easier.
- Added force-restart option for mtr to always restart servers between tests
to avoid race on start of the test
- Added wait_condition_with_debug to wait until the passed statement returns
true, or the operation times out. If operation times out, the additional error
statement will be executed

Changes to be committed:
	new file:   mysql-test/include/force_restart.inc
	new file:   mysql-test/include/wait_condition_with_debug.inc
	modified:   mysql-test/mysql-test-run.pl
	modified:   mysql-test/suite/galera/disabled.def
	modified:   mysql-test/suite/galera/r/MW-336.result
	modified:   mysql-test/suite/galera/r/galera_kill_applier.result
	modified:   mysql-test/suite/galera/r/galera_var_slave_threads.result
	new file:   mysql-test/suite/galera/t/MW-336.cnf
	modified:   mysql-test/suite/galera/t/MW-336.test
	modified:   mysql-test/suite/galera/t/galera_kill_applier.test
	modified:   mysql-test/suite/galera/t/galera_parallel_autoinc_largetrx.test
	modified:   mysql-test/suite/galera/t/galera_parallel_autoinc_manytrx.test
	modified:   mysql-test/suite/galera/t/galera_var_slave_threads.test
	modified:   mysql-test/suite/wsrep/disabled.def
	modified:   mysql-test/suite/wsrep/r/variables.result
	modified:   mysql-test/suite/wsrep/t/variables.test
	modified:   sql/mysqld.cc
	modified:   sql/wsrep_mysqld.cc
	modified:   sql/wsrep_mysqld.h
	modified:   sql/wsrep_thd.cc
	modified:   sql/wsrep_var.cc

1 files changed, 35 insertions, 3 deletions

diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl
index d88d99ce2f9..3095de2f819 100755
--- a/mysql-test/mysql-test-run.pl
+++ b/mysql-test/mysql-test-run.pl
@@ -842,6 +842,8 @@ sub run_test_server ($$$) {
 	    next if (defined $t->{reserved} and $t->{reserved} != $wid);
 	    if (! defined $t->{reserved})
 	    {
+	      # Force-restart not relevant when comparing *next* test
+	      $t->{criteria} =~ s/force-restart$/no-restart/;
 	      my $criteria= $t->{criteria};
 	      # Reserve similar tests for this worker, but not too many
 	      my $maxres= (@$tests - $i) / $opt_parallel + 1;
@@ -3699,6 +3701,25 @@ sub find_analyze_request
   return $analyze;
 }
 
+# The test can leave a file in var/tmp/ to signal
+# that all servers should be restarted
+sub restart_forced_by_test($)
+{
+  my $file = shift;
+  my $restart = 0;
+  foreach my $mysqld ( mysqlds() )
+  {
+    my $datadir = $mysqld->value('datadir');
+    my $force_restart_file = "$datadir/mtr/$file";
+    if ( -f $force_restart_file )
+    {
+      mtr_verbose("Restart of servers forced by test");
+      $restart = 1;
+      last;
+    }
+  }
+  return $restart;
+}
 
 # Return timezone value of tinfo or default value
 sub timezone {
@@ -4041,8 +4062,12 @@ sub run_testcase ($$) {
       if ( $res == 0 )
       {
         my $check_res;
-        if ( $opt_check_testcases and
-             $check_res= check_testcase($tinfo, "after"))
+	if ( restart_forced_by_test('force_restart') )
+        {
+          stop_all_servers($opt_shutdown_timeout);
+        }
+        elsif ( $opt_check_testcases and
+                $check_res= check_testcase($tinfo, "after"))
         {
           if ($check_res == 1) {
             # Test case had sideeffects, not fatal error, just continue
@@ -4077,7 +4102,8 @@ sub run_testcase ($$) {
         find_testcase_skipped_reason($tinfo);
         mtr_report_test_skipped($tinfo);
         # Restart if skipped due to missing perl, it may have had side effects
-        if ( $tinfo->{'comment'} =~ /^perl not found/ )
+	if ( restart_forced_by_test('force_restart_if_skipped') ||
+             $tinfo->{'comment'} =~ /^perl not found/ )
         {
           stop_all_servers($opt_shutdown_timeout);
         }
@@ -5227,6 +5253,11 @@ sub server_need_restart {
     return 0;
   }
 
+  if ( $tinfo->{'force_restart'} ) {
+    mtr_verbose_restart($server, "forced in .opt file");
+    return 1;
+  }
+
   if ( $opt_force_restart ) {
     mtr_verbose_restart($server, "forced restart turned on");
     return 1;
@@ -6209,6 +6240,7 @@ Misc options
                         servers to exit before finishing the process
   fast                  Run as fast as possible, don't wait for servers
                         to shutdown etc.
+  force-restart         Always restart servers between tests
   parallel=N            Run tests in N parallel threads (default 1)
                         Use parallel=auto for auto-setting of N
   repeat=N              Run each test N number of times