MDEV-25502: rpl.rpl_perfschema_applier_status_by_worker failed in bb with: Test assertion failed

Problem:
=======
Test fails with 3 different symptoms
connection slave;
Assertion text: 'Last_Seen_Transaction should show .'
Assertion condition: '"0-1-1" = ""'
Assertion condition, interpolated: '"0-1-1" = ""'
Assertion result: '0'

connection slave;
Assertion text: 'Value returned by SSS and PS table for Last_Error_Number
                 should be same.'
Assertion condition: '"1146" = "0"'
Assertion condition, interpolated: '"1146" = "0"'
Assertion result: '0'

connection slave;
Assertion text: 'Value returned by PS table for worker_idle_time should be
                >= 1'
Assertion condition: '"0" >= "1"'
Assertion condition, interpolated: '"0" >= "1"'
Assertion result: '0'

Fix1:
====
Performance schema table's Last_Seen_Transaction is compared with 'SELECT
gtid_slave_pos'. Since DDLs are not transactional changes to user table and
gtid_slave_pos table are not guaranteed to be synchronous. To fix the
issue Gtid_IO_Pos value from SHOW SLAVE STATUS command will be used to
verify the correctness of Performance schema specific
Last_Seen_Transaction.

Fix2:
====
On error worker thread information is stored as part of backup pool. Access
to this backup pool should be protected by 'LOCK_rpl_thread_pool' mutex so
that simultaneous START SLAVE cannot destroy the backup pool, while it is
being queried by performance schema.

Fix3:
====
When a worker is waiting for events if performance schema table is queried,
at present it just returns the difference between current_time and
start_time.  This is incorrect. It should be worker_idle_time +
(current_time - start_time).

For example a worker thread was idle for 10 seconds and then it got events
to process. Upon completion it goes to idle state, now if the pfs table is
queried it should return current_idle time  + worker_idle_time.

1 files changed, 6 insertions, 3 deletions

diff --git a/storage/perfschema/table_replication_applier_status_by_worker.cc b/storage/perfschema/table_replication_applier_status_by_worker.cc
index 6d7866bc318..bb7ec50a533 100644
--- a/storage/perfschema/table_replication_applier_status_by_worker.cc
+++ b/storage/perfschema/table_replication_applier_status_by_worker.cc
@@ -92,9 +92,9 @@ ha_rows table_replication_applier_status_by_worker::get_row_count()
 
 int table_replication_applier_status_by_worker::rnd_next(void)
 {
-  if (global_rpl_thread_pool.inited && global_rpl_thread_pool.count)
+  rpl_parallel_thread_pool *pool= &global_rpl_thread_pool;
+  if (pool->inited && pool->count)
   {
-    rpl_parallel_thread_pool *pool= &global_rpl_thread_pool;
     mysql_mutex_lock(&pool->LOCK_rpl_thread_pool);
     uint worker_count= pool->count;
     for (m_pos.set_at(&m_next_pos);
@@ -111,7 +111,8 @@ int table_replication_applier_status_by_worker::rnd_next(void)
   }
   else
   {
-    struct pool_bkp_for_pfs *bkp_pool= &global_rpl_thread_pool.pfs_bkp;
+    mysql_mutex_lock(&pool->LOCK_rpl_thread_pool);
+    struct pool_bkp_for_pfs *bkp_pool= &pool->pfs_bkp;
     if (bkp_pool->inited && bkp_pool->count)
     {
       for (m_pos.set_at(&m_next_pos);
@@ -121,9 +122,11 @@ int table_replication_applier_status_by_worker::rnd_next(void)
         rpl_parallel_thread *rpt= bkp_pool->rpl_thread_arr[m_pos.m_index];
         make_row(rpt);
         m_next_pos.set_after(&m_pos);
+        mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool);
         return 0;
       }
     }
+    mysql_mutex_unlock(&pool->LOCK_rpl_thread_pool);
   }
   return HA_ERR_END_OF_FILE;
 }