Fixes for Aria transaction handling with lock tables

MDEV-10130 Assertion `share->in_trans == 0' failed in storage/maria/ma_close.c
MDEV-10378 Assertion `trn' failed in virtual int ha_maria::start_stmt

The problem was that maria_handler->trn was not properly reset
at commit/rollback and ha_maria::exernal_lock() could get confused
because.

There was some old code in ha_maria::implicit_commit() that tried
to take care of this, but it was not bullet proof.

Fixed by adding list of all tables that is part of the maria transaction to
TRN.

A nice side effect was of the fix is that loops in
ha_maria::implict_commit() got to be much simpler.

Other things:
- Fixed a bug in mysql_admin_table() where argument open_for_modify
  was wrongly reset for the next table in the chain
- rollback admin command also in case of fatal error.
- Split _ma_set_trn_for_table() to three version to simplify code
  and debugging.
- Several new asserts to detect the original problem (that file was
  not properly removed from trn before calling ma_close())

2 files changed, 9 insertions, 7 deletions

diff --git a/sql/sql_admin.cc b/sql/sql_admin.cc
index 4fe51a2189e..10fcbd56e18 100644
--- a/sql/sql_admin.cc
+++ b/sql/sql_admin.cc
@@ -302,7 +302,7 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
                               HA_CHECK_OPT* check_opt,
                               const char *operator_name,
                               thr_lock_type lock_type,
-                              bool open_for_modify,
+                              bool org_open_for_modify,
                               bool repair_table_use_frm,
                               uint extra_open_options,
                               int (*prepare_func)(THD *, TABLE_LIST *,
@@ -359,10 +359,10 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
   for (table= tables; table; table= table->next_local)
   {
     char table_name[SAFE_NAME_LEN*2+2];
-    char* db = table->db;
+    char *db= table->db;
     bool fatal_error=0;
     bool open_error;
-
+    bool open_for_modify= org_open_for_modify;
     DBUG_PRINT("admin", ("table: '%s'.'%s'", table->db, table->table_name));
     strxmov(table_name, db, ".", table->table_name, NullS);
     thd->open_options|= extra_open_options;
@@ -395,8 +395,8 @@ static bool mysql_admin_table(THD* thd, TABLE_LIST* tables,
 
       /*
         CHECK TABLE command is allowed for views as well. Check on alter flags
-        to differentiate from ALTER TABLE...CHECK PARTITION on which view is not
-        allowed.
+        to differentiate from ALTER TABLE...CHECK PARTITION on which view is
+        not allowed.
       */
       if (lex->alter_info.flags & Alter_info::ALTER_ADMIN_PARTITION ||
           view_operator_func == NULL)
@@ -1053,7 +1053,7 @@ send_result_message:
       }
     }
     /* Error path, a admin command failed. */
-    if (thd->transaction_rollback_request)
+    if (thd->transaction_rollback_request || fatal_error)
     {
       /*
         Unlikely, but transaction rollback was requested by one of storage
diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc
index 5fc7c20d409..52ce42e2e6a 100644
--- a/sql/sql_handler.cc
+++ b/sql/sql_handler.cc
@@ -155,10 +155,11 @@ static void mysql_ha_close_table(SQL_HANDLER *handler)
 {
   THD *thd= handler->thd;
   TABLE *table= handler->table;
+  DBUG_ENTER("mysql_ha_close_table");
 
   /* check if table was already closed */
   if (!table)
-    return;
+    DBUG_VOID_RETURN;
 
   if (!table->s->tmp_table)
   {
@@ -184,6 +185,7 @@ static void mysql_ha_close_table(SQL_HANDLER *handler)
   }
   my_free(handler->lock);
   handler->init();
+  DBUG_VOID_RETURN;
 }
 
 /*