MDEV-7914: spider/bg.ha, spider/bg.ha_part crash server sporadically in buildbot

The crash occurs when a thread that is closing its connection attempts to
access Spider transaction information when another thread has freed that memory
while processing Spider plugin deinit.  This occurs because Spider does not
adjust the plugin's reference count when it sets a transaction information
pointer for the plugin.

The fix I implemented changes the way Spider sets the transaction information
pointer to use thd_set_ha_data() so that Spider's plugin reference counter is
adjusted as well.

Author:
  Jacob Mathew.

Reviewer:
  Kentoku Shiba.

Merged From:
  bb-10.3-MDEV-7914

4 files changed, 105 insertions, 46 deletions

diff --git a/storage/spider/ha_spider.cc b/storage/spider/ha_spider.cc
index b1d50391a08..9668e4cfbdb 100644
--- a/storage/spider/ha_spider.cc
+++ b/storage/spider/ha_spider.cc
@@ -725,7 +725,7 @@ int ha_spider::close()
     }
   }
 
-  if (!thd || !*thd_ha_data(thd, spider_hton_ptr))
+  if (!thd || !thd_get_ha_data(thd, spider_hton_ptr))
   {
     for (roop_count = 0; roop_count < (int) share->link_count; roop_count++)
       conns[roop_count] = NULL;
diff --git a/storage/spider/spd_malloc.h b/storage/spider/spd_malloc.h
index 42e6abd407c..a46172c57e6 100644
--- a/storage/spider/spd_malloc.h
+++ b/storage/spider/spd_malloc.h
@@ -19,7 +19,7 @@
 #define spider_bulk_malloc(A,B,C,...) \
   spider_bulk_alloc_mem(A,B,__func__,__FILE__,__LINE__,C,__VA_ARGS__)
 #define spider_current_trx \
-  (current_thd ? ((SPIDER_TRX *) *thd_ha_data(current_thd, spider_hton_ptr)) : NULL)
+  (current_thd ? ((SPIDER_TRX *) thd_get_ha_data(current_thd, spider_hton_ptr)) : NULL)
 
 #define init_calc_mem(A) init_mem_calc(A,__func__,__FILE__,__LINE__)
 
diff --git a/storage/spider/spd_table.cc b/storage/spider/spd_table.cc
index fde470daadd..76032ca80f0 100644
--- a/storage/spider/spd_table.cc
+++ b/storage/spider/spd_table.cc
@@ -75,7 +75,7 @@ void destroy_thd(MYSQL_THD thd)
   delete thd;
 }
 #endif
-inline MYSQL_THD spider_create_thd(SPIDER_THREAD *thread)
+inline MYSQL_THD spider_create_sys_thd(SPIDER_THREAD *thread)
 {
   THD *thd = create_thd();
   if (thd)
@@ -86,10 +86,30 @@ inline MYSQL_THD spider_create_thd(SPIDER_THREAD *thread)
   }
   return thd;
 }
-inline void spider_destroy_thd(MYSQL_THD thd)
+inline void spider_destroy_sys_thd(MYSQL_THD thd)
 {
   destroy_thd(thd);
 }
+inline MYSQL_THD spider_create_thd()
+{
+  THD *thd;
+  my_thread_init();
+  if (!(thd = new THD(next_thread_id())))
+    my_thread_end();
+  else
+  {
+#ifdef HAVE_PSI_INTERFACE
+    mysql_thread_set_psi_id(thd->thread_id);
+#endif
+    thd->thread_stack = (char *) &thd;
+    thd->store_globals();
+  }
+  return thd;
+}
+inline void spider_destroy_thd(MYSQL_THD thd)
+{
+  delete thd;
+}
 
 #ifdef SPIDER_XID_USES_xid_cache_iterate
 #else
@@ -5714,6 +5734,8 @@ int spider_free_share(
 ) {
   DBUG_ENTER("spider_free_share");
   pthread_mutex_lock(&spider_tbl_mutex);
+  bool do_delete_thd = false;
+  THD *thd = current_thd;
   if (!--share->use_count)
   {
 #ifndef WITHOUT_SPIDER_BG_SEARCH
@@ -5735,8 +5757,16 @@ int spider_free_share(
       share->sts_init &&
       spider_param_store_last_sts(share->store_last_sts)
     ) {
+      if (!thd)
+      {
+        /* Create a thread for Spider system table update */
+        thd = spider_create_thd();
+        if (!thd)
+          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+        do_delete_thd = TRUE;
+      }
       spider_sys_insert_or_update_table_sts(
-        current_thd,
+        thd,
         share->lgtm_tblhnd_share->table_name,
         share->lgtm_tblhnd_share->table_name_length,
         &share->data_file_length,
@@ -5754,8 +5784,16 @@ int spider_free_share(
       share->crd_init &&
       spider_param_store_last_crd(share->store_last_crd)
     ) {
+      if (!thd)
+      {
+        /* Create a thread for Spider system table update */
+        thd = spider_create_thd();
+        if (!thd)
+          DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+        do_delete_thd = TRUE;
+      }
       spider_sys_insert_or_update_table_crd(
-        current_thd,
+        thd,
         share->lgtm_tblhnd_share->table_name,
         share->lgtm_tblhnd_share->table_name_length,
         share->cardinality,
@@ -5777,6 +5815,8 @@ int spider_free_share(
     free_root(&share->mem_root, MYF(0));
     spider_free(spider_current_trx, share, MYF(0));
   }
+  if (do_delete_thd)
+    spider_destroy_thd(thd);
   pthread_mutex_unlock(&spider_tbl_mutex);
   DBUG_RETURN(0);
 }
@@ -6501,7 +6541,7 @@ int spider_close_connection(
   SPIDER_CONN *conn;
   SPIDER_TRX *trx;
   DBUG_ENTER("spider_close_connection");
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
   trx->tmp_spider->conns = &conn;
@@ -6556,6 +6596,7 @@ int spider_db_done(
   void *p
 ) {
   int roop_count;
+  bool do_delete_thd;
   THD *thd = current_thd, *tmp_thd;
   SPIDER_CONN *conn;
   SPIDER_INIT_ERROR_TABLE *spider_init_error_table;
@@ -6563,6 +6604,18 @@ int spider_db_done(
   SPIDER_LGTM_TBLHND_SHARE *lgtm_tblhnd_share;
   DBUG_ENTER("spider_db_done");
 
+  /* Begin Spider plugin deinit */
+  if (thd)
+    do_delete_thd = FALSE;
+  else
+  {
+    /* Create a thread for Spider plugin deinit */
+    thd = spider_create_thd();
+    if (!thd)
+      DBUG_RETURN(HA_ERR_OUT_OF_MEM);
+    do_delete_thd = TRUE;
+  }
+
 #ifndef WITHOUT_SPIDER_BG_SEARCH
   spider_free_trx(spider_global_trx, TRUE);
 #endif
@@ -6619,21 +6672,22 @@ int spider_db_done(
     pthread_mutex_destroy(&spider_udf_table_mon_mutexes[roop_count]);
   spider_free(NULL, spider_udf_table_mon_mutexes, MYF(0));
 
-  if (thd && thd_sql_command(thd) == SQLCOM_UNINSTALL_PLUGIN) {
-    pthread_mutex_lock(&spider_allocated_thds_mutex);
-    while ((tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, 0)))
+  pthread_mutex_lock(&spider_allocated_thds_mutex);
+  while ((tmp_thd = (THD *) my_hash_element(&spider_allocated_thds, 0)))
+  {
+    SPIDER_TRX *trx = (SPIDER_TRX *)
+                      thd_get_ha_data(tmp_thd, spider_hton_ptr);
+    if (trx)
     {
-      SPIDER_TRX *trx = (SPIDER_TRX *) *thd_ha_data(tmp_thd, spider_hton_ptr);
-      if (trx)
-      {
-        DBUG_ASSERT(tmp_thd == trx->thd);
-        spider_free_trx(trx, FALSE);
-        *thd_ha_data(tmp_thd, spider_hton_ptr) = (void *) NULL;
-      } else
-        my_hash_delete(&spider_allocated_thds, (uchar *) tmp_thd);
+      DBUG_ASSERT(tmp_thd == trx->thd);
+      spider_free_trx(trx, FALSE);
+      thd_set_ha_data(tmp_thd, spider_hton_ptr, NULL);
     }
-    pthread_mutex_unlock(&spider_allocated_thds_mutex);
+    else
+      my_hash_delete(&spider_allocated_thds, (uchar *) tmp_thd);
   }
+  pthread_mutex_unlock(&spider_allocated_thds_mutex);
+
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   pthread_mutex_lock(&spider_hs_w_conn_mutex);
   while ((conn = (SPIDER_CONN*) my_hash_element(&spider_hs_w_conn_hash, 0)))
@@ -6784,6 +6838,11 @@ int spider_db_done(
         spider_current_alloc_mem[roop_count] ? "NG" : "OK"
       ));
   }
+
+  /* End Spider plugin deinit */
+  if (do_delete_thd)
+    spider_destroy_thd(thd);
+
 /*
 DBUG_ASSERT(0);
 */
@@ -9865,7 +9924,7 @@ void *spider_table_bg_sts_action(
   DBUG_ENTER("spider_table_bg_sts_action");
   /* init start */
   pthread_mutex_lock(&thread->mutex);
-  if (!(thd = spider_create_thd(thread)))
+  if (!(thd = spider_create_sys_thd(thread)))
   {
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
@@ -9880,7 +9939,7 @@ void *spider_table_bg_sts_action(
   thd_proc_info(thd, "Spider table background statistics action handler");
   if (!(trx = spider_get_trx(NULL, FALSE, &error_num)))
   {
-    spider_destroy_thd(thd);
+    spider_destroy_sys_thd(thd);
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
     pthread_mutex_unlock(&thread->mutex);
@@ -9901,7 +9960,7 @@ void *spider_table_bg_sts_action(
       DBUG_PRINT("info",("spider bg sts kill start"));
       trx->thd = NULL;
       spider_free_trx(trx, TRUE);
-      spider_destroy_thd(thd);
+      spider_destroy_sys_thd(thd);
       pthread_cond_signal(&thread->sync_cond);
       pthread_mutex_unlock(&thread->mutex);
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
@@ -10017,7 +10076,7 @@ void *spider_table_bg_crd_action(
   DBUG_ENTER("spider_table_bg_crd_action");
   /* init start */
   pthread_mutex_lock(&thread->mutex);
-  if (!(thd = spider_create_thd(thread)))
+  if (!(thd = spider_create_sys_thd(thread)))
   {
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
@@ -10032,7 +10091,7 @@ void *spider_table_bg_crd_action(
   thd_proc_info(thd, "Spider table background cardinality action handler");
   if (!(trx = spider_get_trx(NULL, FALSE, &error_num)))
   {
-    spider_destroy_thd(thd);
+    spider_destroy_sys_thd(thd);
     thread->thd_wait = FALSE;
     thread->killed = FALSE;
     pthread_mutex_unlock(&thread->mutex);
@@ -10053,7 +10112,7 @@ void *spider_table_bg_crd_action(
       DBUG_PRINT("info",("spider bg crd kill start"));
       trx->thd = NULL;
       spider_free_trx(trx, TRUE);
-      spider_destroy_thd(thd);
+      spider_destroy_sys_thd(thd);
       pthread_cond_signal(&thread->sync_cond);
       pthread_mutex_unlock(&thread->mutex);
 #if !defined(MYSQL_DYNAMIC_PLUGIN) || !defined(_WIN32)
diff --git a/storage/spider/spd_trx.cc b/storage/spider/spd_trx.cc
index 179156a0950..0a56eafeb6e 100644
--- a/storage/spider/spd_trx.cc
+++ b/storage/spider/spd_trx.cc
@@ -1195,7 +1195,7 @@ SPIDER_TRX *spider_get_trx(
 
   if (
     !thd ||
-    !(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr))
+    !(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr))
   ) {
     DBUG_PRINT("info",("spider create new trx"));
     if (!(trx = (SPIDER_TRX *)
@@ -1233,7 +1233,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_hash;
     spider_alloc_calc_mem_init(trx->trx_conn_hash, 151);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_conn_hash,
       trx->trx_conn_hash.array.max_element *
       trx->trx_conn_hash.array.size_of_element);
@@ -1245,7 +1245,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_another_hash;
     spider_alloc_calc_mem_init(trx->trx_another_conn_hash, 152);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_another_conn_hash,
       trx->trx_another_conn_hash.array.max_element *
       trx->trx_another_conn_hash.array.size_of_element);
@@ -1258,7 +1258,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_hs_r_init_hash;
     spider_alloc_calc_mem_init(trx->trx_hs_r_conn_hash, 153);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_hs_r_conn_hash,
       trx->trx_hs_r_conn_hash.array.max_element *
       trx->trx_hs_r_conn_hash.array.size_of_element);
@@ -1270,7 +1270,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_hs_w_init_hash;
     spider_alloc_calc_mem_init(trx->trx_hs_w_conn_hash, 154);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_hs_w_conn_hash,
       trx->trx_hs_w_conn_hash.array.max_element *
       trx->trx_hs_w_conn_hash.array.size_of_element);
@@ -1284,7 +1284,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_direct_hs_r_init_hash;
     spider_alloc_calc_mem_init(trx->trx_direct_hs_r_conn_hash, 155);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_direct_hs_r_conn_hash,
       trx->trx_direct_hs_r_conn_hash.array.max_element *
       trx->trx_direct_hs_r_conn_hash.array.size_of_element);
@@ -1296,7 +1296,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_direct_hs_w_init_hash;
     spider_alloc_calc_mem_init(trx->trx_direct_hs_w_conn_hash, 156);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_direct_hs_w_conn_hash,
       trx->trx_direct_hs_w_conn_hash.array.max_element *
       trx->trx_direct_hs_w_conn_hash.array.size_of_element);
@@ -1309,7 +1309,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_alter_hash;
     spider_alloc_calc_mem_init(trx->trx_alter_table_hash, 157);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_alter_table_hash,
       trx->trx_alter_table_hash.array.max_element *
       trx->trx_alter_table_hash.array.size_of_element);
@@ -1321,7 +1321,7 @@ SPIDER_TRX *spider_get_trx(
       goto error_init_trx_ha_hash;
     spider_alloc_calc_mem_init(trx->trx_ha_hash, 158);
     spider_alloc_calc_mem(
-      thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+      thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
       trx->trx_ha_hash,
       trx->trx_ha_hash.array.max_element *
       trx->trx_ha_hash.array.size_of_element);
@@ -1443,7 +1443,7 @@ SPIDER_TRX *spider_get_trx(
         pthread_mutex_unlock(&spider_allocated_thds_mutex);
         trx->registed_allocated_thds = TRUE;
       }
-      *thd_ha_data(thd, spider_hton_ptr) = (void *) trx;
+      thd_set_ha_data(thd, spider_hton_ptr, trx);
     }
   }
 
@@ -1489,7 +1489,7 @@ error_set_connect_info_default:
   my_hash_free(&trx->trx_ha_hash);
 error_init_trx_ha_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_alter_table_hash_id,
     trx->trx_alter_table_hash.array.max_element *
     trx->trx_alter_table_hash.array.size_of_element);
@@ -1497,14 +1497,14 @@ error_init_trx_ha_hash:
 error_init_alter_hash:
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_direct_hs_w_conn_hash_id,
     trx->trx_direct_hs_w_conn_hash.array.max_element *
     trx->trx_direct_hs_w_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_direct_hs_w_conn_hash);
 error_direct_hs_w_init_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_direct_hs_r_conn_hash_id,
     trx->trx_direct_hs_r_conn_hash.array.max_element *
     trx->trx_direct_hs_r_conn_hash.array.size_of_element);
@@ -1513,14 +1513,14 @@ error_direct_hs_r_init_hash:
 #endif
 #if defined(HS_HAS_SQLCOM) && defined(HAVE_HANDLERSOCKET)
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_hs_w_conn_hash_id,
     trx->trx_hs_w_conn_hash.array.max_element *
     trx->trx_hs_w_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_hs_w_conn_hash);
 error_hs_w_init_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_hs_r_conn_hash_id,
     trx->trx_hs_r_conn_hash.array.max_element *
     trx->trx_hs_r_conn_hash.array.size_of_element);
@@ -1528,14 +1528,14 @@ error_hs_w_init_hash:
 error_hs_r_init_hash:
 #endif
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_another_conn_hash_id,
     trx->trx_another_conn_hash.array.max_element *
     trx->trx_another_conn_hash.array.size_of_element);
   my_hash_free(&trx->trx_another_conn_hash);
 error_init_another_hash:
   spider_free_mem_calc(
-    thd ? ((SPIDER_TRX *) *thd_ha_data(thd, spider_hton_ptr)) : NULL,
+    thd ? ((SPIDER_TRX *) thd_get_ha_data(thd, spider_hton_ptr)) : NULL,
     trx->trx_conn_hash_id,
     trx->trx_conn_hash.array.max_element *
     trx->trx_conn_hash.array.size_of_element);
@@ -1574,7 +1574,7 @@ int spider_free_trx(
       if (need_lock)
         pthread_mutex_unlock(&spider_allocated_thds_mutex);
     }
-    *thd_ha_data(trx->thd, spider_hton_ptr) = (void *) NULL;
+    thd_set_ha_data(trx->thd, spider_hton_ptr, NULL);
   }
   spider_free_trx_alloc(trx);
   spider_merge_mem_calc(trx, TRUE);
@@ -3374,7 +3374,7 @@ int spider_commit(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_commit");
 
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
 #ifdef HA_CAN_BULK_ACCESS
@@ -3466,7 +3466,7 @@ int spider_rollback(
   SPIDER_CONN *conn;
   DBUG_ENTER("spider_rollback");
 
-  if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+  if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
     DBUG_RETURN(0); /* transaction is not started */
 
 #ifdef HA_CAN_BULK_ACCESS
@@ -3543,7 +3543,7 @@ int spider_xa_prepare(
 
   if (all || (!thd_test_options(thd, OPTION_NOT_AUTOCOMMIT | OPTION_BEGIN)))
   {
-    if (!(trx = (SPIDER_TRX*) *thd_ha_data(thd, spider_hton_ptr)))
+    if (!(trx = (SPIDER_TRX*) thd_get_ha_data(thd, spider_hton_ptr)))
       DBUG_RETURN(0); /* transaction is not started */
 
     DBUG_PRINT("info",("spider trx_start=%s",