MDEV-12350: Heap corruption, overrun buffer, ASAN errors, server crash in my_fill_8bit / filesort

In the function make_sortkey a tmp buffer was defined and in the absence of
param->tmp_buffer, tmp buffer used the sort_keys buffer. sort_keys buffer
has a length defined in sort_field->length, while param->tmp_buffer is
stored in param->rec_length. Make sure to use the appropriate length
based on which buffer we are using otherwise we'll overflow.

Also added a type cast to size_t during the calculation of the sort keys
buffer size to avoid an oveflow if the buffer size exceeds 32 bits.

3 files changed, 56 insertions, 2 deletions

diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result
index b948c9a6f88..52ee73e6612 100644
--- a/mysql-test/r/group_by.result
+++ b/mysql-test/r/group_by.result
@@ -2558,3 +2558,28 @@ create table t2 (c1 int, c2 int);
 select t1.c1 as c1, t2.c2 as c1 from t1, t2 where t1.c1 < 20 and t2.c2 > 5 group by t1.c1, t2.c2 having t1.c1 < 3;
 c1	c1
 drop table t1, t2;
+SET @old_sort_buff_size = @@sort_buffer_size;
+SET @@sort_buffer_size=256*1024;
+CREATE TABLE t1 (c INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES
+(2011),(1977),(1982),(2027),(2023),(NULL),(NULL),(2004),(1974),(2032),
+(1993),(NULL),(1995),(2034),(NULL),(2009),(1900),(NULL),(2025),(1900),
+(2033),(1900),(2012),(NULL),(2009),(1992),(1974),(1974),(2012),(2028),
+(2007),(2012),(1900),(1983),(1900),(2010),(1987),(1994),(1981),(2032),
+(2010),(1989),(2014),(1900),(1900),(1976),(1978),(2007),(2030),(NULL),
+(2002),(1997),(1900),(NULL),(2000),(2027),(1975),(2026),(1975),(2026),
+(2029),(1977),(1900),(1900),(2031),(1993),(1986),(2012),(1979),(2013),
+(1994),(2014),(2025),(2006),(1971),(1974),(2021),(2011),(NULL),(1991),
+(2001),(1977),(2023),(2012),(1900),(1978),(1998),(NULL),(1988),(1999),
+(2017),(2008),(1976),(1900),(2005),(2030),(2023),(1900),(1978),(1990),
+(1978),(1987),(2030),(1900),(2034),(2006),(2015),(2001),(2019),(2024),
+(2030),(1989),(1997),(2007),(2023),(1994),(1971),(2011),(2011),(2015),
+(1984),(1978),(1979),(1989),(2008),(2030);
+SELECT ExtractValue('<a></a>','/a') AS f1, SPACE(c) AS f2 FROM t1 GROUP BY f1, f2 WITH ROLLUP;
+f1	f2
+	NULL
+	                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           
+	NULL
+NULL	NULL
+SET @@sort_buffer_size = @old_sort_buff_size;
+DROP TABLE t1;
diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 80ecfea539b..43274532b3e 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1742,5 +1742,31 @@ select t1.c1 as c1, t2.c2 as c1 from t1, t2 where t1.c1 < 20 and t2.c2 > 5 group
 drop table t1, t2;
 
 #
+# MDEV-12350: Heap corruption, overrun buffer, ASAN errors, server crash in my_fill_8bit / filesort
+#
+
+SET @old_sort_buff_size = @@sort_buffer_size;
+SET @@sort_buffer_size=256*1024;
+CREATE TABLE t1 (c INT) ENGINE=MyISAM;
+INSERT INTO t1 VALUES
+  (2011),(1977),(1982),(2027),(2023),(NULL),(NULL),(2004),(1974),(2032),
+  (1993),(NULL),(1995),(2034),(NULL),(2009),(1900),(NULL),(2025),(1900),
+  (2033),(1900),(2012),(NULL),(2009),(1992),(1974),(1974),(2012),(2028),
+  (2007),(2012),(1900),(1983),(1900),(2010),(1987),(1994),(1981),(2032),
+  (2010),(1989),(2014),(1900),(1900),(1976),(1978),(2007),(2030),(NULL),
+  (2002),(1997),(1900),(NULL),(2000),(2027),(1975),(2026),(1975),(2026),
+  (2029),(1977),(1900),(1900),(2031),(1993),(1986),(2012),(1979),(2013),
+  (1994),(2014),(2025),(2006),(1971),(1974),(2021),(2011),(NULL),(1991),
+  (2001),(1977),(2023),(2012),(1900),(1978),(1998),(NULL),(1988),(1999),
+  (2017),(2008),(1976),(1900),(2005),(2030),(2023),(1900),(1978),(1990),
+  (1978),(1987),(2030),(1900),(2034),(2006),(2015),(2001),(2019),(2024),
+  (2030),(1989),(1997),(2007),(2023),(1994),(1971),(2011),(2011),(2015),
+  (1984),(1978),(1979),(1989),(2008),(2030);
+  
+SELECT ExtractValue('<a></a>','/a') AS f1, SPACE(c) AS f2 FROM t1 GROUP BY f1, f2 WITH ROLLUP;
+SET @@sort_buffer_size = @old_sort_buff_size;
+DROP TABLE t1;
+
+#
 # End of MariaDB 5.5 tests
 #
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 38404b01cf7..5674786d8b2 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -210,7 +210,9 @@ ha_rows filesort(THD *thd, TABLE *table, SORT_FIELD *sortorder, uint s_length,
     {
       ulonglong keys= memory_available / (param.rec_length + sizeof(char*));
       table_sort.keys= (uint) min(num_rows, keys);
-      sort_buff_sz= table_sort.keys*(param.rec_length+sizeof(char*));
+      /* Cast to size_t to avoid overflow when result is greater than uint. */
+      sort_buff_sz= ((size_t)table_sort.keys) *
+                    (param.rec_length + sizeof(char*));
       set_if_bigger(sort_buff_sz, param.rec_length * MERGEBUFF2);   
 
       DBUG_EXECUTE_IF("make_sort_keys_alloc_fail",
@@ -914,7 +916,8 @@ static void make_sortkey(register SORTPARAM *param,
         if (maybe_null)
           *to++=1;
         char *tmp_buffer= param->tmp_buffer ? param->tmp_buffer : (char*)to;
-        String tmp(tmp_buffer, param->sort_length, cs);
+        String tmp(tmp_buffer, param->tmp_buffer ? param->sort_length :
+                                                   sort_field->length, cs);
         String *res= item->str_result(&tmp);
         if (!res)
         {