MDEV-17062 : Test failure on galera.MW-336

Add mutex protection while we calculate required slave
thread change and create them. Add error handling.

4 files changed, 38 insertions, 12 deletions

diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
index 22b3d081087..e00ce9e5d6f 100644
--- a/mysql-test/suite/galera/disabled.def
+++ b/mysql-test/suite/galera/disabled.def
@@ -18,7 +18,6 @@ MW-286 : MDEV-18464 Killing thread can cause mutex deadlock if done concurrently
 MW-328A : MDEV-21483 galera.MW-328A galera.MW-328B
 MW-328B : MDEV-21483 galera.MW-328A galera.MW-328B
 MW-329 : MDEV-19962 Galera test failure on MW-329
-MW-336 : MDEV-21409: Galera test failure on MW-336
 MW-360 : needs rewrite to be MariaDB gtid compatible
 galera.galera_defaults : MDEV-21494 Galera test sporadic failure on galera.galera_defaults
 galera_account_management : MariaDB 10.0 does not support ALTER USER
diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc
index 7f1818def73..36a58b3c9b0 100644
--- a/sql/wsrep_thd.cc
+++ b/sql/wsrep_thd.cc
@@ -84,42 +84,59 @@ static void wsrep_replication_process(THD *thd,
   DBUG_VOID_RETURN;
 }
 
-static bool create_wsrep_THD(Wsrep_thd_args* args)
+static bool create_wsrep_THD(Wsrep_thd_args* args, bool mutex_protected)
 {
+  if (!mutex_protected)
+    mysql_mutex_lock(&LOCK_wsrep_slave_threads);
+
   ulong old_wsrep_running_threads= wsrep_running_threads;
+
   DBUG_ASSERT(args->thread_type() == WSREP_APPLIER_THREAD ||
               args->thread_type() == WSREP_ROLLBACKER_THREAD);
+
   bool res= mysql_thread_create(args->thread_type() == WSREP_APPLIER_THREAD
                                 ? key_wsrep_applier : key_wsrep_rollbacker,
                                 args->thread_id(), &connection_attrib,
                                 start_wsrep_THD, (void*)args);
+
+  if (res)
+  {
+    WSREP_ERROR("Canät create wsrep thread");
+  }
+
   /*
     if starting a thread on server startup, wait until the this thread's THD
     is fully initialized (otherwise a THD initialization code might
     try to access a partially initialized server data structure - MDEV-8208).
   */
-  mysql_mutex_lock(&LOCK_wsrep_slave_threads);
   if (!mysqld_server_initialized)
+  {
     while (old_wsrep_running_threads == wsrep_running_threads)
+    {
       mysql_cond_wait(&COND_wsrep_slave_threads, &LOCK_wsrep_slave_threads);
-  mysql_mutex_unlock(&LOCK_wsrep_slave_threads);
+    }
+  }
+
+  if (!mutex_protected)
+    mysql_mutex_unlock(&LOCK_wsrep_slave_threads);
+
   return res;
 }
 
-void wsrep_create_appliers(long threads)
+bool wsrep_create_appliers(long threads, bool mutex_protected)
 {
   /*  Dont' start slave threads if wsrep-provider or wsrep-cluster-address
       is not set.
   */
   if (!WSREP_PROVIDER_EXISTS)
   {
-    return;
+    return false;
   }
 
   if (!wsrep_cluster_address || wsrep_cluster_address[0]== 0)
   {
     WSREP_DEBUG("wsrep_create_appliers exit due to empty address");
-    return;
+    return false;
   }
 
   long wsrep_threads=0;
@@ -129,11 +146,14 @@ void wsrep_create_appliers(long threads)
     Wsrep_thd_args* args(new Wsrep_thd_args(wsrep_replication_process,
                                             WSREP_APPLIER_THREAD,
                                             pthread_self()));
-    if (create_wsrep_THD(args))
+    if (create_wsrep_THD(args, mutex_protected))
     {
       WSREP_WARN("Can't create thread to manage wsrep replication");
+      return true;
     }
   }
+
+  return false;
 }
 
 static void wsrep_remove_streaming_fragments(THD* thd, const char* ctx)
@@ -279,7 +299,7 @@ void wsrep_create_rollbacker()
                                             pthread_self()));
 
     /* create rollbacker */
-    if (create_wsrep_THD(args))
+    if (create_wsrep_THD(args, false))
       WSREP_WARN("Can't create thread to manage wsrep rollback");
    }
 }
diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h
index f98c15a5a85..d24d8e6358f 100644
--- a/sql/wsrep_thd.h
+++ b/sql/wsrep_thd.h
@@ -84,7 +84,7 @@ private:
 
 int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff,
                           enum enum_var_type scope);
-void wsrep_create_appliers(long threads);
+bool wsrep_create_appliers(long threads, bool mutex_protected=false);
 void wsrep_create_rollbacker();
 
 bool wsrep_bf_abort(const THD*, THD*);
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
index 315cfe2c53e..5f76f650b34 100644
--- a/sql/wsrep_var.cc
+++ b/sql/wsrep_var.cc
@@ -594,15 +594,22 @@ static void wsrep_slave_count_change_update ()
 
 bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type)
 {
+  mysql_mutex_lock(&LOCK_wsrep_slave_threads);
+  bool res= false;
+
   wsrep_slave_count_change_update();
+
   if (wsrep_slave_count_change > 0)
   {
     WSREP_DEBUG("Creating %d applier threads, total %ld", wsrep_slave_count_change, wsrep_slave_threads);
-    wsrep_create_appliers(wsrep_slave_count_change);
+    res= wsrep_create_appliers(wsrep_slave_count_change, true);
     WSREP_DEBUG("Running %lu applier threads", wsrep_running_applier_threads);
     wsrep_slave_count_change = 0;
   }
-  return false;
+
+  mysql_mutex_unlock(&LOCK_wsrep_slave_threads);
+
+  return res;
 }
 
 bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var)