MDEV-23824 SIGSEGV in end_io_cache on REPAIR LOCAL TABLE for Aria table

Bugs fixed:
- prepare_for_repair() didn't close all open files if table opened failed
  because of out-of-memory
- If dd_recreate_table() failed, the data file was not properly restored
  from it's temporary name
- Aria repair initializing code didn't properly clear all used structs
  before calling error, which caused crashed in memory-free calls.
- maria_delete_table() didn't register if table open failed. This could
  calls my_error() to be called without returning 1 to the caller, which
  cased failures in my_ok()

Note when merging to 10.5:
- Remove the #if MYSQL_VERSION from sql_admin.cc

3 files changed, 33 insertions, 12 deletions

diff --git a/storage/maria/ma_check.c b/storage/maria/ma_check.c
index 20f8eca2828..0f4d79032f8 100644
--- a/storage/maria/ma_check.c
+++ b/storage/maria/ma_check.c
@@ -2344,6 +2344,14 @@ static int initialize_variables_for_repair(HA_CHECK *param,
 {
   MARIA_SHARE *share= info->s;
 
+  /*
+    We have to clear these variables first, as the cleanup-in-case-of-error
+    handling may touch these.
+  */
+  bzero((char*) sort_info,  sizeof(*sort_info));
+  bzero((char*) sort_param, sizeof(*sort_param));
+  bzero(&info->rec_cache, sizeof(info->rec_cache));
+
   if (share->data_file_type == NO_RECORD)
   {
     _ma_check_print_error(param,
@@ -2358,9 +2366,6 @@ static int initialize_variables_for_repair(HA_CHECK *param,
   if (share->lock.update_status)
     (*share->lock.update_status)(info);
 
-  bzero((char*) sort_info,  sizeof(*sort_info));
-  bzero((char*) sort_param, sizeof(*sort_param));
-
   param->testflag|= T_REP;                     /* for easy checking */
   if (share->options & (HA_OPTION_CHECKSUM | HA_OPTION_COMPRESS_RECORD))
     param->testflag|= T_CALC_CHECKSUM;
@@ -2388,7 +2393,6 @@ static int initialize_variables_for_repair(HA_CHECK *param,
   set_data_file_type(sort_info, info->s);
   sort_info->org_data_file_type= share->data_file_type;
 
-  bzero(&info->rec_cache, sizeof(info->rec_cache));
   info->rec_cache.file= info->dfile.file;
   info->update= (short) (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED);
 
@@ -2869,9 +2873,13 @@ err:
   _ma_reset_state(info);
 
   end_io_cache(&param->read_cache);
-  end_io_cache(&sort_info.new_info->rec_cache);
+  if (sort_info.new_info)
+  {
+    end_io_cache(&sort_info.new_info->rec_cache);
+    sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
+  }
   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
-  sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
+
   sort_param.sort_info->info->in_check_table= 0;
   /* this below could fail, shouldn't we detect error? */
   if (got_error)
@@ -4086,10 +4094,13 @@ err:
     maria_scan_end(sort_info.info);
   _ma_reset_state(info);
 
-  end_io_cache(&sort_info.new_info->rec_cache);
+  if (sort_info.new_info)
+  {
+    end_io_cache(&sort_info.new_info->rec_cache);
+    sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
+  }
   end_io_cache(&param->read_cache);
   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
-  sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
   if (got_error)
   {
     if (! param->error_printed)
@@ -4618,10 +4629,13 @@ err:
     the share by remove_io_thread() or it was not yet started (if the
     error happend before creating the thread).
   */
-  end_io_cache(&sort_info.new_info->rec_cache);
+  if (sort_info.new_info)
+  {
+    end_io_cache(&sort_info.new_info->rec_cache);
+    sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
+  }
   end_io_cache(&param->read_cache);
   info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
-  sort_info.new_info->opt_flag&= ~(READ_CACHE_USED | WRITE_CACHE_USED);
   /*
     Destroy the new data cache in case of non-quick repair. All slave
     threads did either detach from the share by remove_io_thread()
diff --git a/storage/maria/ma_delete_table.c b/storage/maria/ma_delete_table.c
index fee001df1e1..0c78476ad44 100644
--- a/storage/maria/ma_delete_table.c
+++ b/storage/maria/ma_delete_table.c
@@ -30,6 +30,7 @@ int maria_delete_table(const char *name)
 {
   MARIA_HA *info;
   myf sync_dir;
+  int got_error= 0, error;
   DBUG_ENTER("maria_delete_table");
 
 #ifdef EXTRA_DEBUG
@@ -41,9 +42,13 @@ int maria_delete_table(const char *name)
     Unfortunately it is necessary to open the table just to check this. We use
     'open_for_repair' to be able to open even a crashed table.
   */
+  my_errno= 0;
   if (!(info= maria_open(name, O_RDONLY, HA_OPEN_FOR_REPAIR)))
   {
     sync_dir= 0;
+    /* Ignore not found errors and wrong symlink errors */
+    if (my_errno != ENOENT && my_errno != HA_WRONG_CREATE_OPTION)
+      got_error= my_errno;;
   }
   else
   {
@@ -78,7 +83,9 @@ int maria_delete_table(const char *name)
       DBUG_RETURN(1);
   }
 
-  DBUG_RETURN(maria_delete_table_files(name, 0, sync_dir));
+  if (!(error= maria_delete_table_files(name, 0, sync_dir)))
+    error= got_error;
+  DBUG_RETURN(error);
 }
 
 
diff --git a/storage/maria/ma_open.c b/storage/maria/ma_open.c
index 226ebe8c1dd..3fa0e4293a6 100644
--- a/storage/maria/ma_open.c
+++ b/storage/maria/ma_open.c
@@ -1334,7 +1334,7 @@ static void setup_key_functions(register MARIA_KEYDEF *keyinfo)
 
 
 /**
-   @brief Function to save and store the header in the index file (.MYI)
+   @brief Function to save and store the header in the index file (.MAI)
 
    Operates under MARIA_SHARE::intern_lock if requested.
    Sets MARIA_SHARE::MARIA_STATE_INFO::is_of_horizon if transactional table.