From 3acab2de200f5a3e07bc5cf655f6decd217424e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Thu, 16 Jan 2020 13:18:44 +0200 Subject: MDEV-17062 : Test failure on galera.MW-336 Add mutex protection while we calculate required slave thread change and create them. Add error handling. --- mysql-test/suite/galera/disabled.def | 1 - sql/wsrep_thd.cc | 49 ++++++++++++++++++++++++++---------- sql/wsrep_thd.h | 2 +- sql/wsrep_var.cc | 10 ++++++-- 4 files changed, 45 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def index 8efb75b0160..581e9e80daa 100644 --- a/mysql-test/suite/galera/disabled.def +++ b/mysql-test/suite/galera/disabled.def @@ -14,7 +14,6 @@ MW-286 : MDEV-18464 Killing thread can cause mutex deadlock if done concurrently MW-328A : MDEV-21483 galera.MW-328A galera.MW-328B MW-328B : MDEV-21483 galera.MW-328A galera.MW-328B MW-329 : MDEV-19962 Galera test failure on MW-329 -MW-336 : MDEV-17062 Test failure on galera.MW-336 galera.galera_defaults : MDEV-21494 Galera test sporadic failure on galera.galera_defaults galera_account_management : MariaDB 10.0 does not support ALTER USER galera_as_master_gtid : Requires MySQL GTID diff --git a/sql/wsrep_thd.cc b/sql/wsrep_thd.cc index 413d7f97214..de2ad6d1a1f 100644 --- a/sql/wsrep_thd.cc +++ b/sql/wsrep_thd.cc @@ -416,29 +416,46 @@ static void wsrep_replication_process(THD *thd) DBUG_VOID_RETURN; } -static bool create_wsrep_THD(wsrep_thread_args* args) +static bool create_wsrep_THD(wsrep_thread_args* args, bool thread_count_lock) { - mysql_mutex_lock(&LOCK_thread_count); + if (!thread_count_lock) + mysql_mutex_lock(&LOCK_thread_count); + ulong old_wsrep_running_threads= wsrep_running_threads; + DBUG_ASSERT(args->thread_type == WSREP_APPLIER_THREAD || args->thread_type == WSREP_ROLLBACKER_THREAD); + bool res= mysql_thread_create(args->thread_type == WSREP_APPLIER_THREAD ? key_wsrep_applier : key_wsrep_rollbacker, &args->thread_id, &connection_attrib, start_wsrep_THD, (void*)args); + + if (res) + { + WSREP_ERROR("Can't create wsrep thread"); + } + /* if starting a thread on server startup, wait until the this thread's THD is fully initialized (otherwise a THD initialization code might try to access a partially initialized server data structure - MDEV-8208). */ if (!mysqld_server_initialized) + { while (old_wsrep_running_threads == wsrep_running_threads) + { mysql_cond_wait(&COND_thread_count, &LOCK_thread_count); - mysql_mutex_unlock(&LOCK_thread_count); + } + } + + if (!thread_count_lock) + mysql_mutex_unlock(&LOCK_thread_count); + return res; } -void wsrep_create_appliers(long threads) +bool wsrep_create_appliers(long threads, bool thread_count_lock) { if (!wsrep_connected) { @@ -450,26 +467,32 @@ void wsrep_create_appliers(long threads) "connection at '%s'", wsrep_cluster_address); assert(0); } - return; + return false; } - long wsrep_threads=0; + long wsrep_threads= 0; + while (wsrep_threads++ < threads) { wsrep_thread_args* arg; - if((arg = (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL) { + + if((arg= (wsrep_thread_args*)my_malloc(sizeof(wsrep_thread_args), MYF(0))) == NULL) + { WSREP_ERROR("Can't allocate memory for wsrep replication thread %ld\n", wsrep_threads); assert(0); } - arg->thread_type = WSREP_APPLIER_THREAD; - arg->processor = wsrep_replication_process; + arg->thread_type= WSREP_APPLIER_THREAD; + arg->processor= wsrep_replication_process; - if (create_wsrep_THD(arg)) { - WSREP_WARN("Can't create thread to manage wsrep replication"); + if (create_wsrep_THD(arg, thread_count_lock)) + { + WSREP_ERROR("Can't create thread to manage wsrep replication"); my_free(arg); - return; + return true; } } + + return false; } static void wsrep_rollback_process(THD *thd) @@ -565,7 +588,7 @@ void wsrep_create_rollbacker() arg->processor = wsrep_rollback_process; /* create rollbacker */ - if (create_wsrep_THD(arg)) { + if (create_wsrep_THD(arg, false)) { WSREP_WARN("Can't create thread to manage wsrep rollback"); my_free(arg); return; diff --git a/sql/wsrep_thd.h b/sql/wsrep_thd.h index 6ce14a4eb0e..8d928014518 100644 --- a/sql/wsrep_thd.h +++ b/sql/wsrep_thd.h @@ -26,7 +26,7 @@ int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff, enum enum_var_type scope); void wsrep_client_rollback(THD *thd); void wsrep_replay_transaction(THD *thd); -void wsrep_create_appliers(long threads); +bool wsrep_create_appliers(long threads, bool thread_count_lock=false); void wsrep_create_rollbacker(); int wsrep_abort_thd(void *bf_thd_ptr, void *victim_thd_ptr, diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc index 0f7f7ef2acf..9777cc6ec62 100644 --- a/sql/wsrep_var.cc +++ b/sql/wsrep_var.cc @@ -602,16 +602,22 @@ static void wsrep_slave_count_change_update () bool wsrep_slave_threads_update (sys_var *self, THD* thd, enum_var_type type) { + mysql_mutex_lock(&LOCK_thread_count); + bool res= false; + wsrep_slave_count_change_update(); if (wsrep_slave_count_change > 0) { WSREP_DEBUG("Creating %d applier threads, total %ld", wsrep_slave_count_change, wsrep_slave_threads); - wsrep_create_appliers(wsrep_slave_count_change); + res= wsrep_create_appliers(wsrep_slave_count_change, true); WSREP_DEBUG("Running %lu applier threads", wsrep_running_applier_threads); wsrep_slave_count_change = 0; } - return false; + + mysql_mutex_unlock(&LOCK_thread_count); + + return res; } bool wsrep_desync_check (sys_var *self, THD* thd, set_var* var) -- cgit v1.2.1