BUG#23175 - MYISAM crash/repair failed during repair

Repair table could crash a server if there is not sufficient
memory (myisam_sort_buffer_size) to operate. Affects not only
repair, but also all statements that use create index by sort:
repair by sort, parallel repair, bulk insert.

Return an error if there is not sufficient memory to store at
least one key per BUFFPEK.

Also fixed memory leak if thr_find_all_keys returns an error.


myisam/sort.c:
  maxbuffer is number of BUFFPEK-s for repair. It is calculated
  as records / keys. keys is number of keys that can be stored
  in memory (myisam_sort_buffer_size). There must be sufficient
  memory to store both BUFFPEK-s and keys. It was checked
  correctly before this patch. However there is another
  requirement that wasn't checked: there must be sufficient
  memory for at least one key per BUFFPEK, otherwise repair
  by sort/parallel repair cannot operate.
  
  Return an error if there is not sufficient memory to store at
  least one key per BUFFPEK.
  
  Also fixed memory leak if thr_find_all_keys returns an error.
mysql-test/r/repair.result:
  A test case for BUG#23175.
mysql-test/t/repair.test:
  A test case for BUG#23175.

3 files changed, 62 insertions, 2 deletions

diff --git a/myisam/sort.c b/myisam/sort.c
index 727840709da..154d50d4d39 100644
--- a/myisam/sort.c
+++ b/myisam/sort.c
@@ -148,7 +148,8 @@ int _create_index_by_sort(MI_SORT_PARAM *info,my_bool no_messages,
 	skr=maxbuffer;
 	if (memavl < sizeof(BUFFPEK)*(uint) maxbuffer ||
 	    (keys=(memavl-sizeof(BUFFPEK)*(uint) maxbuffer)/
-	     (sort_length+sizeof(char*))) <= 1)
+             (sort_length+sizeof(char*))) <= 1 ||
+            keys < (uint) maxbuffer)
 	{
 	  mi_check_print_error(info->sort_info->param,
 			       "sort_buffer_size is to small");
@@ -363,7 +364,8 @@ pthread_handler_decl(thr_find_all_keys,arg)
         skr=maxbuffer;
         if (memavl < sizeof(BUFFPEK)*maxbuffer ||
             (keys=(memavl-sizeof(BUFFPEK)*maxbuffer)/
-             (sort_length+sizeof(char*))) <= 1)
+             (sort_length+sizeof(char*))) <= 1 ||
+            keys < (uint) maxbuffer)
         {
           mi_check_print_error(sort_param->sort_info->param,
                                "sort_buffer_size is to small");
@@ -497,6 +499,8 @@ int thr_write_keys(MI_SORT_PARAM *sort_param)
     if (!sinfo->sort_keys)
     {
       got_error=1;
+      my_free(mi_get_rec_buff_ptr(info, sinfo->rec_buff),
+              MYF(MY_ALLOW_ZERO_PTR));
       continue;
     }
     if (!got_error)
diff --git a/mysql-test/r/repair.result b/mysql-test/r/repair.result
index c069824e9f0..8b3782ec2a4 100644
--- a/mysql-test/r/repair.result
+++ b/mysql-test/r/repair.result
@@ -48,3 +48,31 @@ Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_par
 t1	1	a	1	a	A	5	NULL	NULL	YES	BTREE	
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 DROP TABLE t1;
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4096;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	repair	error	sort_buffer_size is to small
+test.t1	repair	warning	Number of rows changed from 0 to 157
+test.t1	repair	status	OK
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;
diff --git a/mysql-test/t/repair.test b/mysql-test/t/repair.test
index f086d5b0c2a..b192d5c0a64 100644
--- a/mysql-test/t/repair.test
+++ b/mysql-test/t/repair.test
@@ -45,4 +45,32 @@ SHOW INDEX FROM t1;
 SET myisam_repair_threads=@@global.myisam_repair_threads;
 DROP TABLE t1;
 
+#
+# BUG#23175 - MYISAM crash/repair failed during repair
+#
+CREATE TABLE t1(a CHAR(255), KEY(a));
+SET myisam_sort_buffer_size=4096;
+INSERT INTO t1 VALUES
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),('0'),
+('0'),('0'),('0'),('0'),('0'),('0'),('0');
+SET myisam_repair_threads=2;
+REPAIR TABLE t1;
+SET myisam_repair_threads=@@global.myisam_repair_threads;
+SET myisam_sort_buffer_size=@@global.myisam_sort_buffer_size;
+DROP TABLE t1;
+
 # End of 4.1 tests