Fixed bug mdev-449.

The bug could caused a crash when the server executed a query with
ORDER by and sort_buffer_size was set to a small enough number.
It happened because the small sort buffer did not allow to allocate
all merge buffers in it.
Made sure that the allocated sort buffer would be big enough
to contain all possible merge buffers.  

3 files changed, 178 insertions, 5 deletions

diff --git a/mysql-test/r/order_by.result b/mysql-test/r/order_by.result
index c0a54817b4c..8fa1006b8fc 100644
--- a/mysql-test/r/order_by.result
+++ b/mysql-test/r/order_by.result
@@ -1783,3 +1783,135 @@ Warnings:
 Note	1003	select `test`.`t2`.`b` AS `b` from `test`.`t1` join `test`.`t2` where (`test`.`t2`.`a` = `test`.`t1`.`b`) group by `test`.`t2`.`b` order by `test`.`t1`.`b`
 drop table t1,t2;
 End of 5.2 tests
+#
+# Bug mdev-449: ORDER BY with small sort_buffer_size
+#
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10"),
+(11,"11"),(12,"12"),(13,"13"),(14,"14"),(15,"15"),
+(16,"16"),(17,"17"),(18,"18"),(19,"19"),(20,"20"),
+(21,"21"),(22,"22"),(23,"23"),(24,"24"),(25,"25"),
+(26,"26"),(27,"27"),(28,"28"),(29,"29"),(30,"30"),
+(31,"31"),(32,"32"),(33,"33"),(34,"34"),(35,"35"),
+(36,"36"),(37,"37"),(38,"38"),(39,"39"),(40,"40"),
+(41,"41"),(42,"42"),(43,"43"),(44,"44"),(45,"45"),
+(46,"46"),(47,"47"),(48,"48"),(49,"49"),(50,"50"),
+(51,"51"),(52,"52"),(53,"53"),(54,"54"),(55,"55"),
+(56,"56"),(57,"57"),(58,"58"),(59,"59"),(60,"60"),
+(61,"61"),(62,"62"),(63,"63"),(64,"64"),(65,"65"),
+(66,"66"),(67,"67"),(68,"68"),(69,"69"),(70,"70"),
+(71,"71"),(72,"72"),(73,"73"),(74,"74"),(75,"75"),
+(76,"76"),(77,"77"),(78,"78"),(79,"79"),(80,"80"),
+(81,"81"),(82,"82"),(83,"83"),(84,"84"),(85,"85"),
+(86,"86"),(87,"87"),(88,"88"),(89,"89"),(90,"90"),
+(91,"91"),(92,"92"),(93,"93"),(94,"94"),(95,"95"),
+(96,"96"),(97,"97"),(98,"98"),(99,"99");
+set @save_sort_buffer_size= @@sort_buffer_size;
+set sort_buffer_size= 2000;
+SELECT * FROM t1 ORDER BY f1 DESC, f0;
+f0	f1	f2
+100	99	99
+99	98	98
+98	97	97
+97	96	96
+96	95	95
+95	94	94
+94	93	93
+93	92	92
+92	91	91
+91	90	90
+90	89	89
+89	88	88
+88	87	87
+87	86	86
+86	85	85
+85	84	84
+84	83	83
+83	82	82
+82	81	81
+81	80	80
+80	79	79
+79	78	78
+78	77	77
+77	76	76
+76	75	75
+75	74	74
+74	73	73
+73	72	72
+72	71	71
+71	70	70
+70	69	69
+69	68	68
+68	67	67
+67	66	66
+66	65	65
+65	64	64
+64	63	63
+63	62	62
+62	61	61
+61	60	60
+60	59	59
+59	58	58
+58	57	57
+57	56	56
+56	55	55
+55	54	54
+54	53	53
+53	52	52
+52	51	51
+51	50	50
+50	49	49
+49	48	48
+48	47	47
+47	46	46
+46	45	45
+45	44	44
+44	43	43
+43	42	42
+42	41	41
+41	40	40
+40	39	39
+39	38	38
+38	37	37
+37	36	36
+36	35	35
+35	34	34
+34	33	33
+33	32	32
+32	31	31
+31	30	30
+30	29	29
+29	28	28
+28	27	27
+27	26	26
+26	25	25
+25	24	24
+24	23	23
+23	22	22
+22	21	21
+21	20	20
+20	19	19
+19	18	18
+18	17	17
+17	16	16
+16	15	15
+15	14	14
+14	13	13
+13	12	12
+12	11	11
+11	10	10
+10	9	9
+9	8	8
+8	7	7
+7	6	6
+6	5	5
+5	4	4
+4	3	3
+3	2	2
+2	1	1
+1	0	0
+set sort_buffer_size= @save_sort_buffer_size;
+DROP TABLE t1;
+End of 5.3 tests
diff --git a/mysql-test/t/order_by.test b/mysql-test/t/order_by.test
index 425bef6c679..1d81248b81e 100644
--- a/mysql-test/t/order_by.test
+++ b/mysql-test/t/order_by.test
@@ -1589,3 +1589,42 @@ SELECT t2.b FROM t1, t2 WHERE t1.b = t2.a GROUP BY t2.b ORDER BY t1.b;
 drop table t1,t2;
 
 --echo End of 5.2 tests
+
+--echo #
+--echo # Bug mdev-449: ORDER BY with small sort_buffer_size
+--echo #
+
+CREATE TABLE t1(f0 int auto_increment primary key, f1 int, f2 varchar(200));
+INSERT INTO t1(f1, f2) VALUES 
+(0,"0"),(1,"1"),(2,"2"),(3,"3"),(4,"4"),(5,"5"),
+(6,"6"),(7,"7"),(8,"8"),(9,"9"),(10,"10"),
+(11,"11"),(12,"12"),(13,"13"),(14,"14"),(15,"15"),
+(16,"16"),(17,"17"),(18,"18"),(19,"19"),(20,"20"),
+(21,"21"),(22,"22"),(23,"23"),(24,"24"),(25,"25"),
+(26,"26"),(27,"27"),(28,"28"),(29,"29"),(30,"30"),
+(31,"31"),(32,"32"),(33,"33"),(34,"34"),(35,"35"),
+(36,"36"),(37,"37"),(38,"38"),(39,"39"),(40,"40"),
+(41,"41"),(42,"42"),(43,"43"),(44,"44"),(45,"45"),
+(46,"46"),(47,"47"),(48,"48"),(49,"49"),(50,"50"),
+(51,"51"),(52,"52"),(53,"53"),(54,"54"),(55,"55"),
+(56,"56"),(57,"57"),(58,"58"),(59,"59"),(60,"60"),
+(61,"61"),(62,"62"),(63,"63"),(64,"64"),(65,"65"),
+(66,"66"),(67,"67"),(68,"68"),(69,"69"),(70,"70"),
+(71,"71"),(72,"72"),(73,"73"),(74,"74"),(75,"75"),
+(76,"76"),(77,"77"),(78,"78"),(79,"79"),(80,"80"),
+(81,"81"),(82,"82"),(83,"83"),(84,"84"),(85,"85"),
+(86,"86"),(87,"87"),(88,"88"),(89,"89"),(90,"90"),
+(91,"91"),(92,"92"),(93,"93"),(94,"94"),(95,"95"),
+(96,"96"),(97,"97"),(98,"98"),(99,"99");
+
+set @save_sort_buffer_size= @@sort_buffer_size;
+set sort_buffer_size= 2000;
+
+SELECT * FROM t1 ORDER BY f1 DESC, f0;
+
+set sort_buffer_size= @save_sort_buffer_size; 
+
+DROP TABLE t1;
+
+--echo End of 5.3 tests
+
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 772698c6e1a..4d06317447a 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -102,6 +102,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 {
   int error;
   ulong memavl, min_sort_memory;
+  ulong sort_buff_sz;
   uint maxbuffer;
   BUFFPEK *buffpek;
   ha_rows records= HA_POS_ERROR;
@@ -210,6 +211,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
 
   memavl= thd->variables.sortbuff_size;
   min_sort_memory= max(MIN_SORT_MEMORY, param.sort_length*MERGEBUFF2);
+  set_if_bigger(min_sort_memory, sizeof(BUFFPEK*)*MERGEBUFF2);
   if (!table_sort.sort_keys)
   {
     while (memavl >= min_sort_memory)
@@ -217,9 +219,10 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
       ulong old_memavl;
       ulong keys= memavl/(param.rec_length+sizeof(char*));
       table_sort.keys= (uint) min(records+1, keys);
+      sort_buff_sz= table_sort.keys*(param.rec_length+sizeof(char*));
+      set_if_bigger(sort_buff_sz, param.rec_length * MERGEBUFF2);   
       if ((table_sort.sort_keys=
-           (uchar**) my_malloc(table_sort.keys*(param.rec_length+sizeof(char*)),
-                               MYF(0))))
+           (uchar**) my_malloc(sort_buff_sz, MYF(0))))
         break;
       old_memavl=memavl;
       if ((memavl=memavl/4*3) < min_sort_memory &&
@@ -282,8 +285,7 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
       Use also the space previously used by string pointers in sort_buffer
       for temporary key storage.
     */
-    param.keys=((param.keys*(param.rec_length+sizeof(char*))) /
-		param.rec_length-1);
+    param.keys= sort_buff_sz / param.rec_length;
     maxbuffer--;				// Offset from 0
     if (merge_many_buff(&param,(uchar*) sort_keys,buffpek,&maxbuffer,
 			&tempfile))
@@ -1257,7 +1259,7 @@ int merge_buffers(SORTPARAM *param, IO_CACHE *from_file,
   to_start_filepos= my_b_tell(to_file);
   strpos= sort_buffer;
   org_max_rows=max_rows= param->max_rows;
-
+  
   /* The following will fire if there is not enough space in sort_buffer */
   DBUG_ASSERT(maxcount!=0);