MDEV-8208: Sporadic SEGFAULT on startup

Problem:
When mysqld starts as a galera node, it creates 2 system threads
(applier & rollbacker) using start_wsrep_THD(). These threads are
created before plugin initialization (plugin_init()) for SST methods
like rsync and xtrabackup.

The threads' initialization itself can proceed in parallel to mysqld's
main thread of execution. As a result, the thread initialization code
(start_wsrep_THD()) can end up accessing some un/partially initialized
structures (like maria_hton, in this particular case) resulting in
segfault.

Solution:
Fixed by calling THD::init_for_queries() (which accesses maria_hton)
only after the plugins have been initialized.

3 files changed, 57 insertions, 18 deletions

diff --git a/sql/mysqld.cc b/sql/mysqld.cc
index a59780f8f4f..415f6c3d617 100644
--- a/sql/mysqld.cc
+++ b/sql/mysqld.cc
@@ -4507,11 +4507,6 @@ will be ignored as the --log-bin option is not defined.");
                      // to know mysqld_port now - lp:1071882
 #endif /* !EMBEDDED_LIBRARY */
 
-        /*
-          Plugin initialization (plugin_init()) hasn't happened yet, set
-          maria_hton to 0.
-        */
-        maria_hton= 0;
         wsrep_init_startup(true);
       }
     }
@@ -4593,6 +4588,29 @@ a file name for --log-bin-index option", opt_binlog_index_name);
   }
   plugins_are_initialized= TRUE;  /* Don't separate from init function */
 
+#ifdef WITH_WSREP
+  /* Wait for wsrep threads to get created. */
+  if (wsrep_creating_startup_threads == 1) {
+    mysql_mutex_lock(&LOCK_thread_count);
+    while (wsrep_running_threads < 2)
+    {
+      mysql_cond_wait(&COND_thread_count, &LOCK_thread_count);
+    }
+
+    /* Now is the time to initialize threads for queries. */
+    THD *tmp;
+    I_List_iterator<THD> it(threads);
+    while ((tmp= it++))
+    {
+      if (tmp->wsrep_applier == true)
+      {
+        tmp->init_for_queries();
+      }
+    }
+    mysql_mutex_unlock(&LOCK_thread_count);
+  }
+#endif
+
   have_csv= plugin_status(STRING_WITH_LEN("csv"),
                           MYSQL_STORAGE_ENGINE_PLUGIN);
   have_ndbcluster= plugin_status(STRING_WITH_LEN("ndbcluster"),
@@ -4825,16 +4843,11 @@ pthread_handler_t start_wsrep_THD(void *arg)
   THD *thd;
   wsrep_thd_processor_fun processor= (wsrep_thd_processor_fun)arg;
 
-  if (my_thread_init())
+  if (my_thread_init() || (!(thd= new THD(true))))
   {
-    WSREP_ERROR("Could not initialize thread");
-    return(NULL);
+    goto error;
   }
 
-  if (!(thd= new THD(true)))
-  {
-    return(NULL);
-  }
   mysql_mutex_lock(&LOCK_thread_count);
   thd->thread_id=thread_id++;
 
@@ -4859,13 +4872,13 @@ pthread_handler_t start_wsrep_THD(void *arg)
 
   mysql_thread_set_psi_id(thd->thread_id);
   thd->thr_create_utime=  microsecond_interval_timer();
+
   if (MYSQL_CALLBACK_ELSE(thread_scheduler, init_new_connection_thread, (), 0))
   {
     close_connection(thd, ER_OUT_OF_RESOURCES);
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
-
-    return(NULL);
+    goto error;
   }
 
 // </5.1.17>
@@ -4888,8 +4901,7 @@ pthread_handler_t start_wsrep_THD(void *arg)
     statistic_increment(aborted_connects,&LOCK_status);
     MYSQL_CALLBACK(thread_scheduler, end_thread, (thd, 0));
     delete thd;
-
-    return(NULL);
+    goto error;
   }
 
   thd->system_thread= SYSTEM_THREAD_SLAVE_SQL;
@@ -4899,12 +4911,21 @@ pthread_handler_t start_wsrep_THD(void *arg)
   //thd->version= refresh_version;
   thd->proc_info= 0;
   thd->command= COM_SLEEP;
-  thd->set_time();
-  thd->init_for_queries();
+
+  if (plugins_are_initialized)
+  {
+    thd->init_for_queries();
+  }
 
   mysql_mutex_lock(&LOCK_thread_count);
   wsrep_running_threads++;
   mysql_cond_broadcast(&COND_thread_count);
+
+  if (wsrep_running_threads > 2)
+  {
+    wsrep_creating_startup_threads= 0;
+  }
+
   mysql_mutex_unlock(&LOCK_thread_count);
 
   processor(thd);
@@ -4942,6 +4963,15 @@ pthread_handler_t start_wsrep_THD(void *arg)
     mysql_mutex_unlock(&LOCK_thread_count);
   }
   return(NULL);
+
+error:
+  WSREP_ERROR("Failed to create/initialize system thread");
+
+  /* Abort if its the first applier/rollbacker thread. */
+  if (wsrep_creating_startup_threads < 2)
+    unireg_abort(1);
+  else
+    return NULL;
 }
 
 /**/
diff --git a/sql/wsrep_mysqld.cc b/sql/wsrep_mysqld.cc
index a154809d132..c570c7b831d 100644
--- a/sql/wsrep_mysqld.cc
+++ b/sql/wsrep_mysqld.cc
@@ -65,6 +65,13 @@ my_bool wsrep_restart_slave_activated  = 0; // node has dropped, and slave
                                             // restart will be needed
 my_bool wsrep_slave_UK_checks          = 0; // slave thread does UK checks
 my_bool wsrep_slave_FK_checks          = 0; // slave thread does FK checks
+
+/*
+  Set during the creation of first wsrep applier and rollback threads.
+  Since these threads are critical, abort if the thread creation fails.
+*/
+my_bool wsrep_creating_startup_threads = 0;
+
 /*
  * End configuration options
  */
@@ -700,6 +707,7 @@ void wsrep_init_startup (bool first)
 
   if (!wsrep_start_replication()) unireg_abort(1);
 
+  wsrep_creating_startup_threads= 1;
   wsrep_create_rollbacker();
   wsrep_create_appliers(1);
 
diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h
index dd47d4dc1a4..e6fe0c04fac 100644
--- a/sql/wsrep_mysqld.h
+++ b/sql/wsrep_mysqld.h
@@ -99,6 +99,7 @@ extern my_bool     wsrep_restart_slave;
 extern my_bool     wsrep_restart_slave_activated;
 extern my_bool     wsrep_slave_FK_checks;
 extern my_bool     wsrep_slave_UK_checks;
+extern my_bool     wsrep_creating_startup_threads;
 
 enum enum_wsrep_OSU_method { WSREP_OSU_TOI, WSREP_OSU_RSU };
 enum enum_wsrep_sync_wait {