MDEV-25803 Inplace ALTER breaks MyISAM/Aria table when order of keys is changed

mysql_prepare_create_table() does my_qsort(sort_keys) on key
info. This sorting is indeterministic: a table is created with one
order and inplace alter may overwrite frm with another order. Since
inplace alter does nothing about key info for MyISAM/Aria storage
engines this results in discrepancy between frm and storage engine key
definitions.

The fix avoids the sorting of keys when no new keys added by ALTER
(and this is ok for MyISAM/Aria since it cannot add new keys inplace).

Notes:

mi_keydef_write()/mi_keyseg_write() are used only in mi_create(). They
should be used in ha_inplace_alter_table() as well.

Aria corruption detection is unimplemented: maria_check_definition()
is never used!

MySQL 8.0 has this bug as well as of 8.0.26.

This breaks main.long_unique in 10.4. The new result is correct and
should be applied as it just different (original) order of keys.

5 files changed, 92 insertions, 4 deletions

diff --git a/mysql-test/main/alter_table.result b/mysql-test/main/alter_table.result
index 8a3fc640301..522c4bf872b 100644
--- a/mysql-test/main/alter_table.result
+++ b/mysql-test/main/alter_table.result
@@ -2584,5 +2584,21 @@ set max_statement_time= 0;
 drop table t1;
 drop view v1;
 #
+# MDEV-25803 Inplace ALTER breaks MyISAM/Aria tables when order of keys is changed
+#
+set @save_default_engine= @@default_storage_engine;
+create or replace table t1 (x int, y int, unique (y), unique (x), primary key(x)) engine myisam;
+alter table t1 change x xx int, algorithm=inplace;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+create or replace table t1 (x int, y int, unique (y), unique (x), primary key(x));
+alter table t1 change x xx int, algorithm=inplace;
+check table t1;
+Table	Op	Msg_type	Msg_text
+test.t1	check	status	OK
+drop table t1;
+set @@default_storage_engine= @save_default_engine;
+#
 # End of 10.3 tests
 #
diff --git a/mysql-test/main/alter_table.test b/mysql-test/main/alter_table.test
index fab57a49f5b..fd50366311f 100644
--- a/mysql-test/main/alter_table.test
+++ b/mysql-test/main/alter_table.test
@@ -2100,5 +2100,46 @@ drop table t1;
 drop view v1;
 
 --echo #
+--echo # MDEV-25803 Inplace ALTER breaks MyISAM/Aria tables when order of keys is changed
+--echo #
+set @save_default_engine= @@default_storage_engine;
+--disable_query_log
+if ($MTR_COMBINATION_INNODB)
+{
+  set default_storage_engine= innodb;
+}
+if ($MTR_COMBINATION_ARIA)
+{
+  set default_storage_engine= aria;
+}
+--enable_query_log
+
+if (!$MTR_COMBINATION_INNODB)
+{
+  --disable_query_log
+  --disable_result_log
+  # There is no inplace ADD INDEX for MyISAM/Aria:
+  create or replace table t1 (x int);
+  --error ER_ALTER_OPERATION_NOT_SUPPORTED
+  alter table t1 add unique (x), algorithm=inplace;
+  --error ER_ALTER_OPERATION_NOT_SUPPORTED
+  alter table t1 add primary key(x), algorithm=inplace;
+  --error ER_ALTER_OPERATION_NOT_SUPPORTED
+  alter table t1 add index(x), algorithm=inplace;
+  --enable_query_log
+  --enable_result_log
+}
+
+create or replace table t1 (x int, y int, unique (y), unique (x), primary key(x)) engine myisam;
+alter table t1 change x xx int, algorithm=inplace;
+check table t1;
+create or replace table t1 (x int, y int, unique (y), unique (x), primary key(x));
+alter table t1 change x xx int, algorithm=inplace;
+check table t1;
+# cleanup
+drop table t1;
+set @@default_storage_engine= @save_default_engine;
+
+--echo #
 --echo # End of 10.3 tests
 --echo #
diff --git a/sql/sql_table.cc b/sql/sql_table.cc
index 2b796403dc8..fb3ac735c5d 100644
--- a/sql/sql_table.cc
+++ b/sql/sql_table.cc
@@ -4202,9 +4202,31 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
     my_message(ER_WRONG_AUTO_KEY, ER_THD(thd, ER_WRONG_AUTO_KEY), MYF(0));
     DBUG_RETURN(TRUE);
   }
-  /* Sort keys in optimized order */
-  my_qsort((uchar*) *key_info_buffer, *key_count, sizeof(KEY),
-	   (qsort_cmp) sort_keys);
+  /*
+    We cannot do qsort of key info if MyISAM/Aria does inplace. These engines
+    do not synchronise key info on inplace alter and that qsort is
+    indeterministic (MDEV-25803).
+
+    Yet we do not know whether we do inplace or not. That detection is done
+    after this create_table_impl() and that cannot be changed because of chicken
+    and egg problem (inplace processing requires key info made by
+    create_table_impl()).
+
+    MyISAM/Aria cannot add index inplace so we are safe to qsort key info in
+    that case. And if we don't add index then we do not need qsort at all.
+  */
+  if (!(create_info->options & HA_CREATE_TMP_ALTER) ||
+      alter_info->flags & ALTER_ADD_INDEX)
+  {
+    /*
+      Sort keys in optimized order.
+
+      Note: PK must be always first key, otherwise init_from_binary_frm_image()
+      can not understand it.
+    */
+    my_qsort((uchar*) *key_info_buffer, *key_count, sizeof(KEY),
+             (qsort_cmp) sort_keys);
+  }
   create_info->null_bits= null_fields;
 
   /* Check fields. */
diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index c0163473f3a..12a55f2349e 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -707,6 +707,11 @@ static int table2maria(TABLE *table_arg, data_file_type row_type,
     - compare SPATIAL keys;
     - compare FIELD_SKIP_ZERO which is converted to FIELD_NORMAL correctly
       (should be correctly detected in table2maria).
+
+  FIXME:
+    maria_check_definition() is never used! CHECK TABLE does not detect the
+    corruption! Do maria_check_definition() like check_definition() is done
+    by MyISAM (related to MDEV-25803).
 */
 
 int maria_check_definition(MARIA_KEYDEF *t1_keyinfo,
diff --git a/storage/myisam/mi_create.c b/storage/myisam/mi_create.c
index de516d9fb6a..ebe139bb342 100644
--- a/storage/myisam/mi_create.c
+++ b/storage/myisam/mi_create.c
@@ -711,7 +711,11 @@ int mi_create(const char *name,uint keys,MI_KEYDEF *keydefs,
   }
 #endif
 
-  /* Write key and keyseg definitions */
+  /* Write key and keyseg definitions
+
+    TODO: update key and keyseg definitions for inplace alter (grep sql layer by
+          MDEV-25803). Do the same for Aria.
+  */
   DBUG_PRINT("info", ("write key and keyseg definitions"));
   for (i=0 ; i < share.base.keys - uniques; i++)
   {