MDEV-20324: Galera threads are not registered to performance schema

Galera threads were not registered to performance schema and
used pthread_create when mysql_thread_create should have been
used.

Added test case to verify current galera performance schema
instrumentation does work.

7 files changed, 148 insertions, 7 deletions

diff --git a/mysql-test/suite/galera/r/galera_performance_schema.result b/mysql-test/suite/galera/r/galera_performance_schema.result
new file mode 100644
index 00000000000..5b4994556d6
--- /dev/null
+++ b/mysql-test/suite/galera/r/galera_performance_schema.result
@@ -0,0 +1,44 @@
+use performance_schema;
+SELECT name
+FROM threads
+WHERE name LIKE 'thread/sql/wsrep%'
+ORDER BY name;
+name	thread/sql/wsrep_applier_thread
+name	thread/sql/wsrep_rollbacker_thread
+use test;
+create table t1 (a int not null primary key) engine=innodb;
+insert into t1 values (1),(2);
+use performance_schema;
+select name from mutex_instances where name like 'wait/synch/mutex/sql/LOCK_wsrep%' order by name;
+name	wait/synch/mutex/sql/LOCK_wsrep_config_state
+name	wait/synch/mutex/sql/LOCK_wsrep_desync
+name	wait/synch/mutex/sql/LOCK_wsrep_ready
+name	wait/synch/mutex/sql/LOCK_wsrep_replaying
+name	wait/synch/mutex/sql/LOCK_wsrep_rollback
+name	wait/synch/mutex/sql/LOCK_wsrep_slave_threads
+name	wait/synch/mutex/sql/LOCK_wsrep_sst
+name	wait/synch/mutex/sql/LOCK_wsrep_sst_init
+select name from cond_instances where name like 'wait/synch/cond/sql/COND_wsrep%' order by name;
+name	wait/synch/cond/sql/COND_wsrep_ready
+name	wait/synch/cond/sql/COND_wsrep_replaying
+name	wait/synch/cond/sql/COND_wsrep_rollback
+name	wait/synch/cond/sql/COND_wsrep_sst
+name	wait/synch/cond/sql/COND_wsrep_sst_init
+connection node_2;
+use test;
+SET SESSION wsrep_on=OFF;
+CREATE TABLE t2 (f1 INTEGER) engine=innodb;
+connection node_1;
+use test;
+CREATE TABLE t2 (f1 INTEGER) engine=innodb;
+connection node_2;
+SET SESSION wsrep_on=ON;
+SELECT COUNT(*) FROM t1;
+COUNT(*)	2
+use performance_schema;
+select count(*)>=1 from file_instances where file_name like '%GRA_%.log';
+count(*)>=1	1
+CALL mtr.add_suppression("Slave SQL: Error 'Table 't2' already exists' on query");
+use test;
+drop table t1;
+drop table t2;
diff --git a/mysql-test/suite/galera/t/MW-328A.test b/mysql-test/suite/galera/t/MW-328A.test
index da1040b3e5d..dd692a292b8 100644
--- a/mysql-test/suite/galera/t/MW-328A.test
+++ b/mysql-test/suite/galera/t/MW-328A.test
@@ -13,6 +13,7 @@
 #
 
 --source include/galera_cluster.inc
+--source include/big_test.inc
 --source suite/galera/t/MW-328-header.inc
 
 --connection node_2
diff --git a/mysql-test/suite/galera/t/galera_performance_schema.test b/mysql-test/suite/galera/t/galera_performance_schema.test
new file mode 100644
index 00000000000..d54555ea301
--- /dev/null
+++ b/mysql-test/suite/galera/t/galera_performance_schema.test
@@ -0,0 +1,58 @@
+#
+# Test that wsrep mutexes, condition variables, files and
+# threads are shown in performance schema
+#
+--source include/galera_cluster.inc
+--source include/have_perfschema.inc
+
+use performance_schema;
+
+--vertical_results
+--disable_ps_protocol
+SELECT name
+FROM threads
+WHERE name LIKE 'thread/sql/wsrep%'
+ORDER BY name;
+--enable_ps_protocol
+
+use test;
+create table t1 (a int not null primary key) engine=innodb;
+insert into t1 values (1),(2);
+
+use performance_schema;
+select name from mutex_instances where name like 'wait/synch/mutex/sql/LOCK_wsrep%' order by name;
+select name from cond_instances where name like 'wait/synch/cond/sql/COND_wsrep%' order by name;
+# Whenever a node fails to apply an event on a slave node, the database server creates a
+# special binary log file of the event in the data directory. The naming convention the
+# node uses for the filename is GRA_*.log.
+# Thus, we need to produce a applier failure
+
+--connection node_2
+--exec rm -rf $MYSQLTEST_VARDIR/mysqld.2/data/GRA_*.log
+
+# Create applier failure
+
+use test;
+SET SESSION wsrep_on=OFF;
+CREATE TABLE t2 (f1 INTEGER) engine=innodb;
+
+--connection node_1
+use test;
+CREATE TABLE t2 (f1 INTEGER) engine=innodb;
+
+--connection node_2
+SET SESSION wsrep_on=ON;
+SELECT COUNT(*) FROM t1;
+
+use performance_schema;
+#
+# Below we can't just count number of files as if you run this test more
+# than once, test will create more files
+#
+select count(*)>=1 from file_instances where file_name like '%GRA_%.log';
+CALL mtr.add_suppression("Slave SQL: Error 'Table 't2' already exists' on query");
+
+use test;
+drop table t1;
+drop table t2;
+
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index 8c3be114c19..12a9dc35a6a 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -184,7 +184,19 @@ static PSI_file_info wsrep_files[]=
 {
   { &key_file_wsrep_gra_log, "wsrep_gra_log", 0}
 };
-#endif
+
+PSI_thread_key key_wsrep_sst_joiner, key_wsrep_sst_donor,
+  key_wsrep_rollbacker, key_wsrep_applier;
+
+static PSI_thread_info wsrep_threads[]=
+{
+ {&key_wsrep_sst_joiner, "wsrep_sst_joiner_thread", PSI_FLAG_GLOBAL},
+ {&key_wsrep_sst_donor, "wsrep_sst_donor_thread", PSI_FLAG_GLOBAL},
+ {&key_wsrep_rollbacker, "wsrep_rollbacker_thread", PSI_FLAG_GLOBAL},
+ {&key_wsrep_applier, "wsrep_applier_thread", PSI_FLAG_GLOBAL}
+};
+
+#endif /* HAVE_PSI_INTERFACE */
 
 my_bool wsrep_inited                   = 0; // initialized ?
 
@@ -789,6 +801,7 @@ void wsrep_thr_init()
   mysql_mutex_register("sql", wsrep_mutexes, array_elements(wsrep_mutexes));
   mysql_cond_register("sql", wsrep_conds, array_elements(wsrep_conds));
   mysql_file_register("sql", wsrep_files, array_elements(wsrep_files));
+  mysql_thread_register("sql", wsrep_threads, array_elements(wsrep_threads));
 #endif
 
   mysql_mutex_init(key_LOCK_wsrep_ready, &LOCK_wsrep_ready, MY_MUTEX_INIT_FAST);
@@ -804,6 +817,7 @@ void wsrep_thr_init()
   mysql_mutex_init(key_LOCK_wsrep_slave_threads, &LOCK_wsrep_slave_threads, MY_MUTEX_INIT_FAST);
   mysql_mutex_init(key_LOCK_wsrep_desync, &LOCK_wsrep_desync, MY_MUTEX_INIT_FAST);
   mysql_mutex_init(key_LOCK_wsrep_config_state, &LOCK_wsrep_config_state, MY_MUTEX_INIT_FAST);
+
   DBUG_VOID_RETURN;
 }
 
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
index 718e188cdc4..96b3e63914b 100644
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@ -291,7 +291,14 @@ extern PSI_mutex_key key_LOCK_wsrep_slave_threads;
 extern PSI_mutex_key key_LOCK_wsrep_desync;
 
 extern PSI_file_key key_file_wsrep_gra_log;
+
+extern PSI_thread_key key_wsrep_sst_joiner;
+extern PSI_thread_key key_wsrep_sst_donor;
+extern PSI_thread_key key_wsrep_rollbacker;
+extern PSI_thread_key key_wsrep_applier;
 #endif /* HAVE_PSI_INTERFACE */
+
+
 struct TABLE_LIST;
 class Alter_info;
 int wsrep_to_isolation_begin(THD *thd, char *db_, char *table_,
diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc
index 560db0b6c43..cb8b1cab225 100644
--- a/sql/wsrep_sst.cc
+++ b/sql/wsrep_sst.cc
@@ -735,10 +735,10 @@ static ssize_t sst_prepare_other (const char*  method,
   pthread_t tmp;
   sst_thread_arg arg(cmd_str(), env());
   mysql_mutex_lock (&arg.lock);
-  ret = pthread_create (&tmp, NULL, sst_joiner_thread, &arg);
+  ret = mysql_thread_create (key_wsrep_sst_joiner, &tmp, NULL, sst_joiner_thread, &arg);
   if (ret)
   {
-    WSREP_ERROR("sst_prepare_other(): pthread_create() failed: %d (%s)",
+    WSREP_ERROR("sst_prepare_other(): mysql_thread_create() failed: %d (%s)",
                 ret, strerror(ret));
     return -ret;
   }
@@ -1378,10 +1378,10 @@ static int sst_donate_other (const char*   method,
   pthread_t tmp;
   sst_thread_arg arg(cmd_str(), env);
   mysql_mutex_lock (&arg.lock);
-  ret = pthread_create (&tmp, NULL, sst_donor_thread, &arg);
+  ret = mysql_thread_create (key_wsrep_sst_donor, &tmp, NULL, sst_donor_thread, &arg);
   if (ret)
   {
-    WSREP_ERROR("sst_donate_other(): pthread_create() failed: %d (%s)",
+    WSREP_ERROR("sst_donate_other(): mysql_thread_create() failed: %d (%s)",
                 ret, strerror(ret));
     return ret;
   }
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index 31b6622d30c..da90faa74cf 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -418,9 +418,26 @@ static void wsrep_replication_process(THD *thd)
 
 static bool create_wsrep_THD(wsrep_thread_args* args)
 {
-  ulong old_wsrep_running_threads= wsrep_running_threads;
   mysql_mutex_lock(&LOCK_thread_count);
-  bool res= pthread_create(&args->thread_id, &connection_attrib, start_wsrep_THD,
+  ulong old_wsrep_running_threads= wsrep_running_threads;
+#ifdef HAVE_PSI_THREAD_INTERFACE
+  PSI_thread_key key;
+
+  switch (args->thread_type)
+  {
+    case WSREP_APPLIER_THREAD:
+      key= key_wsrep_applier;
+      break;
+    case WSREP_ROLLBACKER_THREAD:
+      key= key_wsrep_rollbacker;
+      break;
+    default:
+      assert(0);
+      break;
+  }
+#endif
+
+  bool res= mysql_thread_create(key, &args->thread_id, &connection_attrib, start_wsrep_THD,
                            (void*)args);
   /*
     if starting a thread on server startup, wait until the this thread's THD