MDEV-17062 : Test failure on galera.MW-336bb-10.2-MDEV-17062

Add mutex protection while we calculate required slave
thread change and create them. Add error handling.

4 files changed, 45 insertions, 17 deletions

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index 8efb75b0160..581e9e80daa 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -14,7 +14,6 @@ MW-286 : MDEV-18464 Killing thread can cause mutex deadlock if done concurrently
 MW-328A : MDEV-21483 galera.MW-328A galera.MW-328B
 MW-328B : MDEV-21483 galera.MW-328A galera.MW-328B
 MW-329 : MDEV-19962 Galera test failure on MW-329
-MW-336 : MDEV-17062 Test failure on galera.MW-336
 galera.galera_defaults : MDEV-21494 Galera test sporadic failure on galera.galera_defaults
 galera_account_management : MariaDB 10.0 does not support ALTER USER
 galera_as_master_gtid : Requires MySQL GTID
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index 413d7f97214..de2ad6d1a1f 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -416,29 +416,46 @@ static void wsrep_replication_process(THD *thd)
   DBUG_VOID_RETURN;
 }
 
-static bool create_wsrep_THD(wsrep_thread_args* args)
+static bool create_wsrep_THD(wsrep_thread_args* args, bool thread_count_lock)
 {
-  mysql_mutex_lock(&LOCK_thread_count);
+  if (!thread_count_lock)
+    mysql_mutex_lock(&LOCK_thread_count);
+
   ulong old_wsrep_running_threads= wsrep_running_threads;
+
   DBUG_ASSERT(args->thread_type == WSREP_APPLIER_THREAD ||
               args->thread_type == WSREP_ROLLBACKER_THREAD);
+
   bool res= mysql_thread_create(args->thread_type == WSREP_APPLIER_THREAD
                                 ? key_wsrep_applier : key_wsrep_rollbacker,
                                 &args->thread_id, &connection_attrib,
                                 start_wsrep_THD, (void*)args);
+
+  if (res)
+  {
+    WSREP_ERROR("Can't create wsrep thread");
+  }
+
   /*
     if starting a thread on server startup, wait until the this thread's THD
     is fully initialized (otherwise a THD initialization code might
     try to access a partially initialized server data structure - MDEV-8208).
   */
   if (!mysqld_server_initialized)
+  {
     while (old_wsrep_running_threads == wsrep_running_threads)
+    {
       mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
-  mysql_mutex_unlock(&LOCK_thread_count);
+    }
+  }
+
+  if (!thread_count_lock)
+    mysql_mutex_unlock(&LOCK_thread_count);
+
   return res;
 }
 
-void wsrep_create_appliers(long threads)
+bool wsrep_create_appliers(long threads, bool thread_count_lock)
 {
   if (!wsrep_connected)
   {
@@ -450,26 +467,32 @@ void wsrep_create_appliers(long threads)
                   "connection at '%s'", wsrep_cluster_address);
       assert(0);
     }
-    return;
+    return false;
   }
 
-  long wsrep_threads=0;
+  long wsrep_threads= 0;
+
   while (wsrep_threads++ < threads) {
     wsrep_thread_args* arg;
-    if((arg = (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL) {
+
+    if((arg= (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL)
+    {
       WSREP_ERROR("Can't allocate memory for wsrep replication thread %ld\n", wsrep_threads);
       assert(0);
     }
 
-    arg->thread_type = WSREP_APPLIER_THREAD;
-    arg->processor = wsrep_replication_process;
+    arg->thread_type= WSREP_APPLIER_THREAD;
+    arg->processor= wsrep_replication_process;
 
-    if (create_wsrep_THD(arg)) {
-      WSREP_WARN("Can't create thread to manage wsrep replication");
+    if (create_wsrep_THD(arg, thread_count_lock))
+    {
+      WSREP_ERROR("Can't create thread to manage wsrep replication");
       my_free(arg);
-      return;
+      return true;
     }
   }
+
+  return false;
 }
 
 static void wsrep_rollback_process(THD *thd)
@@ -565,7 +588,7 @@ void wsrep_create_rollbacker()
     arg->processor = wsrep_rollback_process;
 
     /* create rollbacker */
-    if (create_wsrep_THD(arg)) {
+    if (create_wsrep_THD(arg, false)) {
       WSREP_WARN("Can't create thread to manage wsrep rollback");
       my_free(arg);
       return;
diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h
index 6ce14a4eb0e..8d928014518 100644
--- a/sql/wsrep_thd.h
+++ b/sql/wsrep_thd.h
@@ -26,7 +26,7 @@ int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff,
                           enum enum_var_type scope);
 void wsrep_client_rollback(THD *thd);
 void wsrep_replay_transaction(THD *thd);
-void wsrep_create_appliers(long threads);
+bool wsrep_create_appliers(long threads, bool thread_count_lock=false);
 void wsrep_create_rollbacker();
 
 int  wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr,
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
index 0f7f7ef2acf..9777cc6ec62 100644
--- a/sql/wsrep_var.cc
+++ b/sql/wsrep_var.cc
@@ -602,16 +602,22 @@ static void wsrep_slave_count_change_update ()
 
 bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type)
 {
+  mysql_mutex_lock(&LOCK_thread_count);
+  bool res= false;
+
   wsrep_slave_count_change_update();
 
   if (wsrep_slave_count_change > 0)
   {
     WSREP_DEBUG("Creating %d applier threads, total %ld", wsrep_slave_count_change, wsrep_slave_threads);
-    wsrep_create_appliers(wsrep_slave_count_change);
+    res= wsrep_create_appliers(wsrep_slave_count_change, true);
     WSREP_DEBUG("Running %lu applier threads", wsrep_running_applier_threads);
     wsrep_slave_count_change = 0;
   }
-  return false;
+
+  mysql_mutex_unlock(&LOCK_thread_count);
+
+  return res;
 }
 
 bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var)