MDEV-23842 Atomic RENAME TABLE

- Major rewrite of ddl_log.cc and ddl_log.h
  - ddl_log.cc described in the beginning how the recovery works.
  - ddl_log.log has unique signature and is dynamic. It's easy to
    add more information to the header and other ddl blocks while still
    being able to execute old ddl entries.
  - IO_SIZE for ddl blocks is now dynamic. Can be changed without affecting
    recovery of old logs.
  - Code is more modular and is now usable outside of partition handling.
  - Renamed log file to dll_recovery.log and added option --log-ddl-recovery
    to allow one to specify the path & filename.
- Added ddl_log_entry_phase[], number of phases for each DDL action,
  which allowed me to greatly simply set_global_from_ddl_log_entry()
- Changed how strings are stored in log entries, which allows us to
  store much more information in a log entry.
- ddl log is now always created at start and deleted on normal shutdown.
  This simplices things notable.
- Added probes debug_crash_here() and debug_simulate_error() to simply
  crash testing and allow crash after a given number of times a probe
  is executed. See comments in debug_sync.cc and rename_table.test for
  how this can be used.
- Reverting failed table and view renames is done trough the ddl log.
  This ensures that the ddl log is tested also outside of recovery.
- Added helper function 'handler::needs_lower_case_filenames()'
- Extend binary log with Q_XID events. ddl log handling is using this
  to check if a ddl log entry was logged to the binary log (if yes,
  it will be deleted from the log during ddl_log_close_binlogged_events()
- If a DDL entry fails 3 time, disable it. This is to ensure that if
  we have a crash in ddl recovery code the server will not get stuck
  in a forever crash-restart-crash loop.

mysqltest.cc changes:
- --die will now replace $variables with their values
- $error will contain the error of the last failed statement

storage engine changes:
- maria_rename() was changed to be more robust against crashes during
  rename.

1 files changed, 29 insertions, 25 deletions

diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc
index 9afdd6f9dc0..22efecb9210 100644
--- a/sql/sql_partition.cc
+++ b/sql/sql_partition.cc
@@ -6203,15 +6203,17 @@ static bool write_log_replace_delete_frm(ALTER_PARTITION_PARAM_TYPE *lpt,
   DDL_LOG_MEMORY_ENTRY *log_entry;
   DBUG_ENTER("write_log_replace_delete_frm");
 
+  bzero(&ddl_log_entry, sizeof(ddl_log_entry));
   if (replace_flag)
     ddl_log_entry.action_type= DDL_LOG_REPLACE_ACTION;
   else
     ddl_log_entry.action_type= DDL_LOG_DELETE_ACTION;
   ddl_log_entry.next_entry= next_entry;
-  ddl_log_entry.handler_name= reg_ext;
-  ddl_log_entry.name= to_path;
+  lex_string_set(&ddl_log_entry.handler_name, reg_ext);
+  lex_string_set(&ddl_log_entry.name, to_path);
+
   if (replace_flag)
-    ddl_log_entry.from_name= from_path;
+    lex_string_set(&ddl_log_entry.from_name, from_path);
   if (ddl_log_write_entry(&ddl_log_entry, &log_entry))
   {
     DBUG_RETURN(TRUE);
@@ -6263,6 +6265,7 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
     if (part_elem->part_state == PART_IS_CHANGED ||
         (part_elem->part_state == PART_IS_ADDED && temp_partitions))
     {
+      bzero(&ddl_log_entry, sizeof(ddl_log_entry));
       if (part_info->is_sub_partitioned())
       {
         List_iterator<partition_element> sub_it(part_elem->subpartitions);
@@ -6272,8 +6275,9 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
         {
           partition_element *sub_elem= sub_it++;
           ddl_log_entry.next_entry= *next_entry;
-          ddl_log_entry.handler_name=
-               ha_resolve_storage_engine_name(sub_elem->engine_type);
+          lex_string_set(&ddl_log_entry.handler_name,
+                         ha_resolve_storage_engine_name(sub_elem->
+                                                        engine_type));
           if (create_subpartition_name(tmp_path, sizeof(tmp_path), path,
                                        part_elem->partition_name,
                                        sub_elem->partition_name,
@@ -6283,16 +6287,15 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
                                        sub_elem->partition_name,
                                        NORMAL_PART_NAME))
             DBUG_RETURN(TRUE);
-          ddl_log_entry.name= normal_path;
-          ddl_log_entry.from_name= tmp_path;
+          lex_string_set(&ddl_log_entry.name, normal_path);
+          lex_string_set(&ddl_log_entry.from_name, tmp_path);
           if (part_elem->part_state == PART_IS_CHANGED)
             ddl_log_entry.action_type= DDL_LOG_REPLACE_ACTION;
           else
             ddl_log_entry.action_type= DDL_LOG_RENAME_ACTION;
           if (ddl_log_write_entry(&ddl_log_entry, &log_entry))
-          {
             DBUG_RETURN(TRUE);
-          }
+
           *next_entry= log_entry->entry_pos;
           sub_elem->log_entry= log_entry;
           insert_part_info_log_entry_list(part_info, log_entry);
@@ -6301,8 +6304,8 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
       else
       {
         ddl_log_entry.next_entry= *next_entry;
-        ddl_log_entry.handler_name=
-               ha_resolve_storage_engine_name(part_elem->engine_type);
+        lex_string_set(&ddl_log_entry.handler_name,
+                       ha_resolve_storage_engine_name(part_elem->engine_type));
         if (create_partition_name(tmp_path, sizeof(tmp_path), path,
                                   part_elem->partition_name, TEMP_PART_NAME,
                                   TRUE) ||
@@ -6310,8 +6313,8 @@ static bool write_log_changed_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
                                   part_elem->partition_name, NORMAL_PART_NAME,
                                   TRUE))
           DBUG_RETURN(TRUE);
-        ddl_log_entry.name= normal_path;
-        ddl_log_entry.from_name= tmp_path;
+        lex_string_set(&ddl_log_entry.name, normal_path);
+        lex_string_set(&ddl_log_entry.from_name, tmp_path);
         if (part_elem->part_state == PART_IS_CHANGED)
           ddl_log_entry.action_type= DDL_LOG_REPLACE_ACTION;
         else
@@ -6355,6 +6358,7 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
   uint num_elements= part_info->partitions.elements;
   DBUG_ENTER("write_log_dropped_partitions");
 
+  bzero(&ddl_log_entry, sizeof(ddl_log_entry));
   ddl_log_entry.action_type= DDL_LOG_DELETE_ACTION;
   if (temp_list)
     num_elements= num_temp_partitions;
@@ -6385,13 +6389,14 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
         {
           partition_element *sub_elem= sub_it++;
           ddl_log_entry.next_entry= *next_entry;
-          ddl_log_entry.handler_name=
-               ha_resolve_storage_engine_name(sub_elem->engine_type);
+          lex_string_set(&ddl_log_entry.handler_name,
+                         ha_resolve_storage_engine_name(sub_elem->
+                                                        engine_type));
           if (create_subpartition_name(tmp_path, sizeof(tmp_path), path,
                                        part_elem->partition_name,
                                        sub_elem->partition_name, name_variant))
             DBUG_RETURN(TRUE);
-          ddl_log_entry.name= tmp_path;
+          lex_string_set(&ddl_log_entry.name, tmp_path);
           if (ddl_log_write_entry(&ddl_log_entry, &log_entry))
           {
             DBUG_RETURN(TRUE);
@@ -6404,13 +6409,13 @@ static bool write_log_dropped_partitions(ALTER_PARTITION_PARAM_TYPE *lpt,
       else
       {
         ddl_log_entry.next_entry= *next_entry;
-        ddl_log_entry.handler_name=
-               ha_resolve_storage_engine_name(part_elem->engine_type);
+        lex_string_set(&ddl_log_entry.handler_name,
+                       ha_resolve_storage_engine_name(part_elem->engine_type));
         if (create_partition_name(tmp_path, sizeof(tmp_path), path,
                                   part_elem->partition_name, name_variant,
                                   TRUE))
           DBUG_RETURN(TRUE);
-        ddl_log_entry.name= tmp_path;
+        lex_string_set(&ddl_log_entry.name, tmp_path);
         if (ddl_log_write_entry(&ddl_log_entry, &log_entry))
         {
           DBUG_RETURN(TRUE);
@@ -6474,7 +6479,7 @@ static bool write_log_drop_shadow_frm(ALTER_PARTITION_PARAM_TYPE *lpt)
     goto error;
   log_entry= part_info->first_log_entry;
   if (ddl_log_write_execute_entry(log_entry->entry_pos,
-                                    FALSE, &exec_log_entry))
+                                  &exec_log_entry))
     goto error;
   mysql_mutex_unlock(&LOCK_gdl);
   set_part_info_exec_log_entry(part_info, exec_log_entry);
@@ -6521,7 +6526,7 @@ static bool write_log_rename_frm(ALTER_PARTITION_PARAM_TYPE *lpt)
   log_entry= part_info->first_log_entry;
   part_info->frm_log_entry= log_entry;
   if (ddl_log_write_execute_entry(log_entry->entry_pos,
-                                    FALSE, &exec_log_entry))
+                                  &exec_log_entry))
     goto error;
   release_part_info_log_entries(old_first_log_entry);
   mysql_mutex_unlock(&LOCK_gdl);
@@ -6576,7 +6581,7 @@ static bool write_log_drop_partition(ALTER_PARTITION_PARAM_TYPE *lpt)
   log_entry= part_info->first_log_entry;
   part_info->frm_log_entry= log_entry;
   if (ddl_log_write_execute_entry(log_entry->entry_pos,
-                                    FALSE, &exec_log_entry))
+                                  &exec_log_entry))
     goto error;
   release_part_info_log_entries(old_first_log_entry);
   mysql_mutex_unlock(&LOCK_gdl);
@@ -6635,7 +6640,6 @@ static bool write_log_add_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt)
   log_entry= part_info->first_log_entry;
 
   if (ddl_log_write_execute_entry(log_entry->entry_pos,
-                                  FALSE,
                                   /* Reuse the old execute ddl_log_entry */
                                   &exec_log_entry))
     goto error;
@@ -6705,7 +6709,7 @@ static bool write_log_final_change_partition(ALTER_PARTITION_PARAM_TYPE *lpt)
   part_info->frm_log_entry= log_entry;
   /* Overwrite the revert execute log entry with this retry execute entry */
   if (ddl_log_write_execute_entry(log_entry->entry_pos,
-                                    FALSE, &exec_log_entry))
+                                  &exec_log_entry))
     goto error;
   release_part_info_log_entries(old_first_log_entry);
   mysql_mutex_unlock(&LOCK_gdl);
@@ -6741,7 +6745,7 @@ static void write_log_completed(ALTER_PARTITION_PARAM_TYPE *lpt,
 
   DBUG_ASSERT(log_entry);
   mysql_mutex_lock(&LOCK_gdl);
-  if (ddl_log_write_execute_entry(0UL, TRUE, &log_entry))
+  if (ddl_log_disable_execute_entry(&log_entry))
   {
     /*
       Failed to write, Bad...