MDEV-28490 Strange result truncation with group_concat_max_len=1GB.

Arythmetic can overrun the uint type when possible group_concat_max_len
is multiplied to collation.mbmaxlen (can easily be like 4).
So use ulonglong there for calculations.

3 files changed, 88 insertions, 3 deletions

diff --git a/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result b/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result
index 30e2639af37..01f44ae51be 100644
--- a/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result
+++ b/mysql-test/suite/sys_vars/r/group_concat_max_len_func.result
@@ -94,4 +94,58 @@ DROP TABLE t1;
 disconnect test_con2;
 disconnect test_con1;
 connection default;
+CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci;
+INSERT INTO t1 VALUES('bar');
+INSERT INTO t1 VALUES('foo');
+SET group_concat_max_len = 1073741823;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741823
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741824;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741824
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741825;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741825
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 1073741826;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	1073741826
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+SET group_concat_max_len = 2147483649;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+Variable_name	Value
+group_concat_max_len	2147483649
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+simple
+bar,foo
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+nested
+bar,foo
+DROP TABLE t1;
 SET @@global.group_concat_max_len = @save;
diff --git a/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test b/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test
index b053ee229d7..d90fc061289 100644
--- a/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test
+++ b/mysql-test/suite/sys_vars/t/group_concat_max_len_func.test
@@ -132,4 +132,35 @@ disconnect test_con1;
 
 connection default;
 
+CREATE TABLE t1(val VARCHAR(100) PRIMARY KEY) CHARACTER SET utf8mb4 COLLATE utf8mb4_danish_ci;
+INSERT INTO t1 VALUES('bar');
+INSERT INTO t1 VALUES('foo');
+
+SET group_concat_max_len = 1073741823;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741824;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741825;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 1073741826;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+SET group_concat_max_len = 2147483649;
+SHOW VARIABLES LIKE 'group_concat_max_len';
+SELECT GROUP_CONCAT(val) AS simple FROM t1;
+SELECT * FROM ( SELECT GROUP_CONCAT(val) AS nested FROM t1) As tmp;
+
+DROP TABLE t1;
+
 SET @@global.group_concat_max_len = @save;
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 5065aba922a..1c17b5c6409 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -4242,9 +4242,9 @@ Item_func_group_concat::fix_fields(THD *thd, Item **ref)
   result.set_charset(collation.collation);
   result_field= 0;
   null_value= 1;
-  max_length= (uint32)MY_MIN(thd->variables.group_concat_max_len
-                             / collation.collation->mbminlen
-                             * collation.collation->mbmaxlen, UINT_MAX32);
+  max_length= (uint32) MY_MIN((ulonglong) thd->variables.group_concat_max_len
+                              / collation.collation->mbminlen
+                              * collation.collation->mbmaxlen, UINT_MAX32);
 
   uint32 offset;
   if (separator->needs_conversion(separator->length(), separator->charset(),