From 0a92ef458bba67223555066cc866a3b553768906 Mon Sep 17 00:00:00 2001
From: Monty <monty@mariadb.org>
Date: Wed, 16 Feb 2022 14:47:26 +0200
Subject: MDEV-17223 Assertion `thd->killed != 0' failed in
 ha_maria::enable_indexes MDEV-22500 Assertion `thd->killed != 0' failed in
 ha_maria::enable_indexes

For MDEV-17223 the issue was an assert that didn't take into account that
we could get duplicate key errors when enablling unique indexes.
Fixed by not retrying repair in case of duplicate key error for this
case, which avoids the assert.

For MDEV-22500 I removed the assert, as it's not critical (just a way to
find potential wrong code) and we will anyway get things logged in the
error log if this happens. This case cannot triggered an assert in 10.3
but I verified that it would trigger in 10.5 and that this patch fixes
it.
---
 mysql-test/suite/maria/repair.result | 31 +++++++++++++++++++++++++++++++
 mysql-test/suite/maria/repair.test   | 33 +++++++++++++++++++++++++++++++++
 storage/maria/ha_maria.cc            | 12 +++++++++---
 3 files changed, 73 insertions(+), 3 deletions(-)

diff --git a/mysql-test/suite/maria/repair.result b/mysql-test/suite/maria/repair.result
index 722d9f28712..458b5503602 100644
--- a/mysql-test/suite/maria/repair.result
+++ b/mysql-test/suite/maria/repair.result
@@ -31,3 +31,34 @@ SET max_session_mem_used=50000;
 REPAIR LOCAL TABLE t1 USE_FRM;
 REPAIR LOCAL TABLE t1;
 DROP TABLE t1;
+SET max_session_mem_used=default;
+
+# MDEV-17223 Assertion `thd->killed != 0' failed in
+# ha_maria::enable_indexes
+#
+CREATE OR REPLACE TABLE t1 (c VARCHAR(1024) NOT NULL) ENGINE=Aria ROW_FORMAT FIXED;
+insert into t1 select char(seq) from seq_65_to_256;
+insert into t1 values ("a");
+ALTER TABLE t1 ADD PRIMARY KEY(c(67));
+ERROR 23000: Duplicate entry 'a' for key 'PRIMARY'
+select count(*) from t1;
+count(*)
+193
+drop table t1;
+
+# MDEV-17223 Assertion `thd->killed != 0' failed in
+# ha_maria::enable_indexes
+#
+SET SESSION aria_sort_buffer_size=1023;
+Warnings:
+Warning	1292	Truncated incorrect aria_sort_buffer_size value: '1023'
+CREATE TABLE t2 (c TEXT,INDEX(c(1000))) ENGINE=Aria;
+INSERT INTO t2 select char(seq) from seq_65_to_255;
+SELECT COUNT(*) FROM t2;
+COUNT(*)
+191
+DROP TABLE t2;
+SET SESSION aria_sort_buffer_size=default;
+#
+# End of 10.3 tests
+#
diff --git a/mysql-test/suite/maria/repair.test b/mysql-test/suite/maria/repair.test
index 571f861c512..688ac076e1d 100644
--- a/mysql-test/suite/maria/repair.test
+++ b/mysql-test/suite/maria/repair.test
@@ -3,6 +3,7 @@
 # as memory usage is different compared to normal server.
 
 --source include/not_embedded.inc
+--source include/have_sequence.inc
 
 #
 # MDEV-11539 test_if_reopen: Assertion `strcmp(share->unique_file_name,filename) || share->last_version' failed upon select from I_S
@@ -41,3 +42,35 @@ REPAIR LOCAL TABLE t1 USE_FRM;
 REPAIR LOCAL TABLE t1;
 --enable_result_log
 DROP TABLE t1;
+SET max_session_mem_used=default;
+
+--echo
+--echo # MDEV-17223 Assertion `thd->killed != 0' failed in
+--echo # ha_maria::enable_indexes
+--echo #
+
+CREATE OR REPLACE TABLE t1 (c VARCHAR(1024) NOT NULL) ENGINE=Aria ROW_FORMAT FIXED;
+insert into t1 select char(seq) from seq_65_to_256;
+insert into t1 values ("a");
+--error ER_DUP_ENTRY
+ALTER TABLE t1 ADD PRIMARY KEY(c(67));
+select count(*) from t1;
+drop table t1;
+
+--echo
+--echo # MDEV-17223 Assertion `thd->killed != 0' failed in
+--echo # ha_maria::enable_indexes
+--echo #
+
+SET SESSION aria_sort_buffer_size=1023;
+CREATE TABLE t2 (c TEXT,INDEX(c(1000))) ENGINE=Aria;
+--disable_warnings
+INSERT INTO t2 select char(seq) from seq_65_to_255;
+--enable_warnings
+SELECT COUNT(*) FROM t2;
+DROP TABLE t2;
+SET SESSION aria_sort_buffer_size=default;
+
+--echo #
+--echo # End of 10.3 tests
+--echo #
diff --git a/storage/maria/ha_maria.cc b/storage/maria/ha_maria.cc
index 12a55f2349e..d43d98821be 100644
--- a/storage/maria/ha_maria.cc
+++ b/storage/maria/ha_maria.cc
@@ -1958,13 +1958,19 @@ int ha_maria::enable_indexes(uint mode)
     param->sort_buffer_length= THDVAR(thd,sort_buffer_size);
     param->stats_method= (enum_handler_stats_method)THDVAR(thd,stats_method);
     param->tmpdir= &mysql_tmpdir_list;
-    if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair)
+
+    /*
+      Don't retry repair if we get duplicate key error if
+      create_unique_index_by_sort is enabled
+      This can be set when doing an ALTER TABLE and enabling unique keys
+    */
+    if ((error= (repair(thd, param, 0) != HA_ADMIN_OK)) && param->retry_repair &&
+        (my_errno != HA_ERR_FOUND_DUPP_KEY ||
+         !file->create_unique_index_by_sort))
     {
       sql_print_warning("Warning: Enabling keys got errno %d on %s.%s, "
                         "retrying",
                         my_errno, param->db_name, param->table_name);
-      /* This should never fail normally */
-      DBUG_ASSERT(thd->killed != 0);
       /* Repairing by sort failed. Now try standard repair method. */
       param->testflag &= ~T_REP_BY_SORT;
       file->state->records= start_rows;
-- 
cgit v1.2.1