MDEV-21326 : Address TSAN warnings in tpool.

1. Fix places where data race warnings were relevant.

tls_worker_data::m_state should be modified under mutex protection,
since both maintainence timer and current worker set this flag.

2. Suppress warnings that are legitimate, yet harmless.
Apparently, the dirty reads in waitable_task::get_ref_count() or
write_slots->pending_io_count()

Avoiding race entirely without side-effects here is tricky,
and the effects of race is harmless.

The worst thing that can happen due to race is an extra wait notification,
under rare circumstances.

3 files changed, 27 insertions, 7 deletions

diff --git a/tpool/tpool.h b/tpool/tpool.h
index 939ae919ff6..b4fa7210350 100644
--- a/tpool/tpool.h
+++ b/tpool/tpool.h
@@ -100,8 +100,8 @@ public:
   waitable_task(callback_func func, void* arg, task_group* group = nullptr);
   void add_ref() override;
   void release() override;
-  bool is_running() { return m_ref_count > 0; }
-  bool get_ref_count() {return m_ref_count;}
+  TPOOL_SUPPRESS_TSAN bool is_running() { return get_ref_count() > 0; }
+  TPOOL_SUPPRESS_TSAN int get_ref_count() {return m_ref_count;}
   void wait();
   virtual ~waitable_task() {};
 };
diff --git a/tpool/tpool_generic.cc b/tpool/tpool_generic.cc
index ec4f39b930b..715b75aa3c9 100644
--- a/tpool/tpool_generic.cc
+++ b/tpool/tpool_generic.cc
@@ -121,6 +121,10 @@ struct MY_ALIGNED(CPU_LEVEL1_DCACHE_LINESIZE)  worker_data
   {
     return m_state & LONG_TASK;
   }
+  bool is_waiting()
+  {
+    return m_state & WAITING;
+  }
   std::chrono::system_clock::time_point m_task_start_time;
   worker_data() :
     m_cv(),
@@ -738,10 +742,10 @@ void thread_pool_generic::submit_task(task* task)
 /* Notify thread pool that current thread is going to wait */
 void thread_pool_generic::wait_begin()
 {
-  if (!tls_worker_data || tls_worker_data->is_long_task())
+  if (!tls_worker_data || tls_worker_data->is_long_task() || tls_worker_data->is_waiting())
     return;
-  tls_worker_data->m_state |= worker_data::WAITING;
   std::unique_lock<std::mutex> lk(m_mtx);
+  tls_worker_data->m_state |= worker_data::WAITING;
   m_waiting_task_count++;
 
   /* Maintain concurrency */
@@ -754,10 +758,10 @@ void thread_pool_generic::wait_begin()
 
 void thread_pool_generic::wait_end()
 {
-  if (tls_worker_data && (tls_worker_data->m_state & worker_data::WAITING))
+  if (tls_worker_data && tls_worker_data->is_waiting())
   {
-    tls_worker_data->m_state &= ~worker_data::WAITING;
     std::unique_lock<std::mutex> lk(m_mtx);
+    tls_worker_data->m_state &= ~worker_data::WAITING;
     m_waiting_task_count--;
   }
 }
diff --git a/tpool/tpool_structs.h b/tpool/tpool_structs.h
index d1c078d11f1..7b0fb857695 100644
--- a/tpool/tpool_structs.h
+++ b/tpool/tpool_structs.h
@@ -21,6 +21,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02111 - 1301 USA*/
 #include <assert.h>
 #include <algorithm>
 
+
+/* Suppress TSAN warnings, that we believe are not critical. */
+#if defined(__has_feature)
+#define TPOOL_HAS_FEATURE(...) __has_feature(__VA_ARGS__)
+#else
+#define TPOOL_HAS_FEATURE(...) 0
+#endif
+
+#if TPOOL_HAS_FEATURE(address_sanitizer)
+#define TPOOL_SUPPRESS_TSAN  __attribute__((no_sanitize("thread"),noinline))
+#elif defined(__GNUC__) && defined (__SANITIZE_THREAD__)
+#define TPOOL_SUPPRESS_TSAN  __attribute__((no_sanitize_thread,noinline))
+#else
+#define TPOOL_SUPPRESS_TSAN
+#endif
+
 namespace tpool
 {
 
@@ -106,7 +122,7 @@ public:
     m_waiters--;
   }
 
-  size_t size()
+  TPOOL_SUPPRESS_TSAN size_t size()
   {
     return m_cache.size();
   }