MDEV-4677 GROUP_CONCAT not showing any output with group_concat_max_len >= 4Gb

don't allow group_concat_max_len values >= 4Gb
(they never worked anyway)

10 files changed, 41 insertions, 18 deletions

diff --git a/mysql-test/r/func_gconcat.result b/mysql-test/r/func_gconcat.result
index 992f935b14e..da33e7bbc2c 100644
--- a/mysql-test/r/func_gconcat.result
+++ b/mysql-test/r/func_gconcat.result
@@ -1280,5 +1280,17 @@ Name_exp_1
 DROP VIEW v1;
 DROP TABLE t1;
 #
+# MDEV-4677 GROUP_CONCAT not showing any output with group_concat_max_len >= 4Gb
+#
+set group_concat_max_len=1024*1024*1024*4;
+Warnings:
+Warning	1292	Truncated incorrect group_concat_max_len value: '4294967296'
+create table t1 (i int, j int);
+insert into t1 values (1,1),(1,2);
+select i, group_concat(j) from t1 group by i;
+i	group_concat(j)
+1	1,2
+drop table t1;
+#
 # End of 10.2 tests
 #
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
index 06886ff05c4..5ccabeecf2d 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result
@@ -944,10 +944,10 @@ READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	GROUP_CONCAT_MAX_LEN
 VARIABLE_SCOPE	SESSION
-VARIABLE_TYPE	BIGINT UNSIGNED
+VARIABLE_TYPE	INT UNSIGNED
 VARIABLE_COMMENT	The maximum length of the result of function GROUP_CONCAT()
 NUMERIC_MIN_VALUE	4
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	4294967295
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
index 4808997a87a..42c236bf54a 100644
--- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
+++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result
@@ -964,10 +964,10 @@ READ_ONLY	NO
 COMMAND_LINE_ARGUMENT	REQUIRED
 VARIABLE_NAME	GROUP_CONCAT_MAX_LEN
 VARIABLE_SCOPE	SESSION
-VARIABLE_TYPE	BIGINT UNSIGNED
+VARIABLE_TYPE	INT UNSIGNED
 VARIABLE_COMMENT	The maximum length of the result of function GROUP_CONCAT()
 NUMERIC_MIN_VALUE	4
-NUMERIC_MAX_VALUE	18446744073709551615
+NUMERIC_MAX_VALUE	4294967295
 NUMERIC_BLOCK_SIZE	1
 ENUM_VALUE_LIST	NULL
 READ_ONLY	NO
diff --git a/mysql-test/t/func_gconcat.test b/mysql-test/t/func_gconcat.test
index e752d9fcd50..b699d39e953 100644
--- a/mysql-test/t/func_gconcat.test
+++ b/mysql-test/t/func_gconcat.test
@@ -939,6 +939,16 @@ DROP VIEW v1;
 
 DROP TABLE t1;
 
+
+--echo #
+--echo # MDEV-4677 GROUP_CONCAT not showing any output with group_concat_max_len >= 4Gb
+--echo #
+set group_concat_max_len=1024*1024*1024*4;
+create table t1 (i int, j int);
+insert into t1 values (1,1),(1,2);
+select i, group_concat(j) from t1 group by i;
+drop table t1;
+
 --echo #
 --echo # End of 10.2 tests
 --echo #
diff --git a/sql/field.cc b/sql/field.cc
index fbce8b9041d..9ea1eac9db9 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -8255,7 +8255,7 @@ int Field_blob::store(const char *from,uint length,CHARSET_INFO *cs)
     DBUG_ASSERT(length <= max_data_length());
     
     new_length= length;
-    copy_length= (uint)MY_MIN(UINT_MAX,table->in_use->variables.group_concat_max_len);
+    copy_length= table->in_use->variables.group_concat_max_len;
     if (new_length > copy_length)
     {
       new_length= Well_formed_prefix(cs,
diff --git a/sql/item_sum.cc b/sql/item_sum.cc
index 9490c71c19e..dd65f04a652 100644
--- a/sql/item_sum.cc
+++ b/sql/item_sum.cc
@@ -3171,7 +3171,7 @@ int dump_leaf_key(void* key_arg, element_count count __attribute__((unused)),
 {
   Item_func_group_concat *item= (Item_func_group_concat *) item_arg;
   TABLE *table= item->table;
-  uint max_length= (uint)table->in_use->variables.group_concat_max_len;
+  uint max_length= table->in_use->variables.group_concat_max_len;
   String tmp((char *)table->record[1], table->s->reclength,
              default_charset_info);
   String tmp2;
@@ -3499,7 +3499,7 @@ bool Item_func_group_concat::repack_tree(THD *thd)
   DBUG_ASSERT(tree->size_of_element == st.tree.size_of_element);
   st.table= table;
   st.len= 0;
-  st.maxlen= (size_t)thd->variables.group_concat_max_len;
+  st.maxlen= thd->variables.group_concat_max_len;
   tree_walk(tree, &copy_to_tree, &st, left_root_right);
   if (st.len <= st.maxlen) // Copying aborted. Must be OOM
   {
@@ -3520,7 +3520,7 @@ bool Item_func_group_concat::repack_tree(THD *thd)
   decreases up to N=10 (that is, factor=1024) and then starts to increase,
   again, very slowly.
 */
-#define GCONCAT_REPACK_FACTOR (1 << 10)
+#define GCONCAT_REPACK_FACTOR 10
 
 bool Item_func_group_concat::add()
 {
@@ -3566,7 +3566,7 @@ bool Item_func_group_concat::add()
   {
     THD *thd= table->in_use;
     table->field[0]->store(row_str_len, FALSE);
-    if (tree_len > thd->variables.group_concat_max_len * GCONCAT_REPACK_FACTOR
+    if ((tree_len >> GCONCAT_REPACK_FACTOR) > thd->variables.group_concat_max_len
         && tree->elements_in_tree > 1)
       if (repack_tree(thd))
         return 1;
@@ -3624,9 +3624,9 @@ Item_func_group_concat::fix_fields(THD *thd, Item **ref)
   result.set_charset(collation.collation);
   result_field= 0;
   null_value= 1;
-  max_length= (uint32)(thd->variables.group_concat_max_len
-              / collation.collation->mbminlen
-              * collation.collation->mbmaxlen);
+  max_length= (uint32)MY_MIN(thd->variables.group_concat_max_len
+                             / collation.collation->mbminlen
+                             * collation.collation->mbmaxlen, UINT_MAX32);
 
   uint32 offset;
   if (separator->needs_conversion(separator->length(), separator->charset(),
diff --git a/sql/sql_cache.cc b/sql/sql_cache.cc
index 8555ae6ee3e..0de153a5e7d 100644
--- a/sql/sql_cache.cc
+++ b/sql/sql_cache.cc
@@ -1451,7 +1451,7 @@ void Query_cache::store_query(THD *thd, TABLE_LIST *tables_used)
     DBUG_PRINT("qcache", ("\
 long %d, 4.1: %d, eof: %d, bin_proto: %d, more results %d, pkt_nr: %d, \
 CS client: %u, CS result: %u, CS conn: %u, limit: %llu, TZ: %p, \
-sql mode: 0x%llx, sort len: %llu, conncat len: %llu, div_precision: %lu, \
+sql mode: 0x%llx, sort len: %llu, concat len: %u, div_precision: %lu, \
 def_week_frmt: %lu, in_trans: %d, autocommit: %d",
                           (int)flags.client_long_flag,
                           (int)flags.client_protocol_41,
@@ -1951,7 +1951,7 @@ Query_cache::send_result_to_client(THD *thd, char *org_sql, uint query_length)
   DBUG_PRINT("qcache", ("\
 long %d, 4.1: %d, eof: %d, bin_proto: %d, more results %d, pkt_nr: %d, \
 CS client: %u, CS result: %u, CS conn: %u, limit: %llu, TZ: %p, \
-sql mode: 0x%llx, sort len: %llu, conncat len: %llu, div_precision: %lu, \
+sql mode: 0x%llx, sort len: %llu, concat len: %u, div_precision: %lu, \
 def_week_frmt: %lu, in_trans: %d, autocommit: %d",
                           (int)flags.client_long_flag,
                           (int)flags.client_protocol_41,
diff --git a/sql/sql_cache.h b/sql/sql_cache.h
index 6ad73bbb0a2..d428c7fdb80 100644
--- a/sql/sql_cache.h
+++ b/sql/sql_cache.h
@@ -554,11 +554,11 @@ struct Query_cache_query_flags
   uint character_set_client_num;
   uint character_set_results_num;
   uint collation_connection_num;
+  uint group_concat_max_len;
   ha_rows limit;
   Time_zone *time_zone;
   sql_mode_t sql_mode;
   ulonglong max_sort_length;
-  ulonglong group_concat_max_len;
   ulong default_week_format;
   ulong div_precision_increment;
   MY_LOCALE *lc_time_names;
diff --git a/sql/sql_class.h b/sql/sql_class.h
index d2b0b40846b..35068245830 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -555,7 +555,6 @@ typedef struct system_variables
   ulonglong bulk_insert_buff_size;
   ulonglong join_buff_size;
   ulonglong sortbuff_size;
-  ulonglong group_concat_max_len;
   ulonglong default_regex_flags;
   ulonglong max_mem_used;
 
@@ -645,6 +644,8 @@ typedef struct system_variables
   uint32     gtid_domain_id;
   uint64     gtid_seq_no;
 
+  uint group_concat_max_len;
+
   /**
     Default transaction access mode. READ ONLY (true) or READ WRITE (false).
   */
diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc
index e1252acefcc..baf27a7d0af 100644
--- a/sql/sys_vars.cc
+++ b/sql/sys_vars.cc
@@ -4157,11 +4157,11 @@ static Sys_var_ulong Sys_default_week_format(
        SESSION_VAR(default_week_format), CMD_LINE(REQUIRED_ARG),
        VALID_RANGE(0, 7), DEFAULT(0), BLOCK_SIZE(1));
 
-static Sys_var_ulonglong Sys_group_concat_max_len(
+static Sys_var_uint Sys_group_concat_max_len(
        "group_concat_max_len",
        "The maximum length of the result of function GROUP_CONCAT()",
        SESSION_VAR(group_concat_max_len), CMD_LINE(REQUIRED_ARG),
-       VALID_RANGE(4, SIZE_T_MAX), DEFAULT(1024*1024), BLOCK_SIZE(1));
+       VALID_RANGE(4, UINT_MAX32), DEFAULT(1024*1024), BLOCK_SIZE(1));
 
 static char *glob_hostname_ptr;
 static Sys_var_charptr Sys_hostname(