wt: comments, OOM checks, test case for deadlock detection

include/waiting_threads.h:
  make wt_thd_dontwait private
mysql-test/r/maria.result:
  deadlock example
mysql-test/t/maria.test:
  deadlock example
mysys/waiting_threads.c:
  comments, OOM checks
sql/mysqld.cc:
  fix variables
sql/sql_class.cc:
  move wt_lazy_init to THD constructor
sql/sql_class.h:
  move wt_lazy_init to THD constructor
storage/maria/ha_maria.cc:
  backport from 6.0
storage/maria/ma_write.c:
  poset-review fixes, set thd->proc_info
storage/maria/trnman.c:
  bugfixing
storage/myisam/mi_check.c:
  warnings
storage/myisam/mi_page.c:
  warnings
storage/myisam/mi_search.c:
  warnings
storage/myisammrg/myrg_create.c:
  warnings
unittest/mysys/waiting_threads-t.c:
  fixes

3 files changed, 40 insertions, 31 deletions

diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index 001aa1262ce..78edf7d6545 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -2346,7 +2346,8 @@ int ha_maria::external_lock(THD *thd, int lock_type)
             This is a bit excessive, ACID requires this only if there are some
             changes to commit (rollback shouldn't be tested).
           */
-          DBUG_ASSERT(!thd->main_da.is_sent);
+          DBUG_ASSERT(!thd->main_da.is_sent ||
+                      thd->killed == THD::KILL_CONNECTION);
           /* autocommit ? rollback a transaction */
 #ifdef MARIA_CANNOT_ROLLBACK
           if (ma_commit(trn))
diff --git a/storage/maria/ma_write.c b/storage/maria/ma_write.c
index 09391ad829c..8add20d46ef 100644
--- a/storage/maria/ma_write.c
+++ b/storage/maria/ma_write.c
@@ -182,23 +182,20 @@ int maria_write(MARIA_HA *info, uchar *record)
       {
         while (keyinfo->ck_insert(info,
                                   (*keyinfo->make_key)(info, &int_key, i,
-                                                        buff, record, filepos,
-                                                        info->trn->trid)))
+                                                       buff, record, filepos,
+                                                       info->trn->trid)))
         {
           TRN *blocker;
           DBUG_PRINT("error",("Got error: %d on write",my_errno));
           /*
-            explicit check for our own trid, because temp tables
-            aren't transactional and don't have a proper TRN so the code
-            below doesn't work for them
-            XXX a better test perhaps ?
+            explicit check to filter out temp tables, they aren't
+            transactional and don't have a proper TRN so the code
+            below doesn't work for them.
+            Also, filter out non-thread maria use, and table modified in
+            the same transaction.
           */
-          if (info->dup_key_trid == info->trn->trid)
-          {
-            if (local_lock_tree)
-              rw_unlock(&keyinfo->root_lock);
+          if (!local_lock_tree || info->dup_key_trid == info->trn->trid)
             goto err;
-          }
           blocker= trnman_trid_to_trn(info->trn, info->dup_key_trid);
           /*
             if blocker TRN was not found, it means that the conflicting
@@ -206,16 +203,16 @@ int maria_write(MARIA_HA *info, uchar *record)
             aborted, as it would have to wait on the key tree lock
             to remove the conflicting key it has inserted.
           */
-          if (local_lock_tree)
+          if (!blocker || blocker->commit_trid != ~(TrID)0)
+          { /* committed */
+            if (blocker)
+              pthread_mutex_unlock(& blocker->state_lock);
             rw_unlock(&keyinfo->root_lock);
-          if (!blocker)
-            goto err;
-          if (blocker->commit_trid != ~(TrID)0)
-          { /* committed, albeit recently */
-            pthread_mutex_unlock(& blocker->state_lock);
             goto err;
           }
-          { /* running. now we wait */
+          rw_unlock(&keyinfo->root_lock);
+          {
+            /* running. now we wait */
             WT_RESOURCE_ID rc;
             int res;
 
@@ -225,15 +222,26 @@ int maria_write(MARIA_HA *info, uchar *record)
             if (res != WT_OK)
             {
               pthread_mutex_unlock(& blocker->state_lock);
+              my_errno= HA_ERR_LOCK_DEADLOCK;
               goto err;
             }
-            res=wt_thd_cond_timedwait(info->trn->wt, & blocker->state_lock);
+            {
+              const char *old_proc_info= proc_info_hook(0,
+                    "waiting for a resource", __func__, __FILE__, __LINE__);
+
+              res= wt_thd_cond_timedwait(info->trn->wt, & blocker->state_lock);
+
+              proc_info_hook(0, old_proc_info, __func__, __FILE__, __LINE__);
+            }
             pthread_mutex_unlock(& blocker->state_lock);
             if (res != WT_OK)
+            {
+              my_errno= res == WT_TIMEOUT ? HA_ERR_LOCK_WAIT_TIMEOUT
+                                          : HA_ERR_LOCK_DEADLOCK;
               goto err;
+            }
           }
-          if (local_lock_tree)
-            rw_wrlock(&keyinfo->root_lock);
+          rw_wrlock(&keyinfo->root_lock);
         }
       }
 
@@ -643,7 +651,7 @@ static int w_search(register MARIA_HA *info, uint32 comp_flag, MARIA_KEY *key,
     {
       DBUG_PRINT("warning", ("Duplicate key"));
       /*
-        FIXME
+        TODO
         When the index will support true versioning - with multiple
         identical values in the UNIQUE index, invisible to each other -
         the following should be changed to "continue inserting keys, at the
diff --git a/storage/maria/trnman.c b/storage/maria/trnman.c
index 42c32db3cab..4a9004596ae 100644
--- a/storage/maria/trnman.c
+++ b/storage/maria/trnman.c
@@ -89,6 +89,7 @@ static void wt_thd_release_self(TRN *trn)
     rc.type= &ma_rc_dup_unique;
     rc.value.ptr= trn;
     wt_thd_release(trn->wt, & rc);
+    trn->wt= 0;
   }
 }
 
@@ -296,8 +297,8 @@ TRN *trnman_new_trn(WT_THD *wt)
     }
     trnman_allocated_transactions++;
     pthread_mutex_init(&trn->state_lock, MY_MUTEX_INIT_FAST);
-    trn->wt= wt;
   }
+  trn->wt= wt;
   trn->pins= lf_hash_get_pins(&trid_to_trn);
   if (!trn->pins)
   {
@@ -415,17 +416,17 @@ my_bool trnman_end_trn(TRN *trn, my_bool commit)
     }
   }
 
+  pthread_mutex_lock(&trn->state_lock);
+  trn->commit_trid= global_trid_generator;
+  wt_thd_release_self(trn);
+  pthread_mutex_unlock(&trn->state_lock);
+
   /*
     if transaction is committed and it was not the only active transaction -
     add it to the committed list
   */
   if (commit && active_list_min.next != &active_list_max)
   {
-    pthread_mutex_lock(&trn->state_lock);
-    trn->commit_trid= global_trid_generator;
-    wt_thd_release_self(trn);
-    pthread_mutex_unlock(&trn->state_lock);
-
     trn->next= &committed_list_max;
     trn->prev= committed_list_max.prev;
     trnman_committed_transactions++;
@@ -440,6 +441,7 @@ my_bool trnman_end_trn(TRN *trn, my_bool commit)
                                active_list_min.next != &active_list_max))
     res= -1;
   trnman_active_transactions--;
+
   pthread_mutex_unlock(&LOCK_trn_list);
 
   /* the rest is done outside of a critical section */
@@ -492,8 +494,6 @@ void trnman_free_trn(TRN *trn)
 
   pthread_mutex_lock(&trn->state_lock);
   trn->short_id= 0;
-  wt_thd_release_self(trn);
-  trn->wt= 0; /* just in case */
   pthread_mutex_unlock(&trn->state_lock);
 
   tmp.trn= pool;