MDEV-22819: Wrong result or Assertion `ix > 0' failed in read_to_buffer upon select with GROUP BY and GROUP_CONCAT10.5-mdev22819

In the merge_buffers phase for sorting, the sort buffer size is divided between the number of chunks.
The chunks have a start and end position (m_buffer_start and m_buffer_end).
Then we read the as many records that fit in this buffer for a chunk of the file.
The issue here was we were resetting the end of buffer(m_buffer_end) to the number of bytes that was
read, this was causing a problem because with dynamic size of sort keys it is possible that later
we would not be able to accommodate even one key inside a chunk of file.
So the fix was to not reset the end of buffer for a chunk of file.

4 files changed, 106 insertions, 6 deletions

diff --git a/mysql-test/main/order_by.result b/mysql-test/main/order_by.result
index c750b5ca10c..26e42e4d6f9 100644
--- a/mysql-test/main/order_by.result
+++ b/mysql-test/main/order_by.result
@@ -3965,4 +3965,62 @@ a
 1
 0
 DROP TABLE t1;
+#
+# MDEV-22819:Wrong result or Assertion `ix > 0' failed in read_to_buffer upon select
+# with GROUP BY and GROUP_CONCAT
+#
+CREATE TABLE t1 (a VARCHAR(1000), b CHAR(1));
+INSERT INTO t1 VALUES
+(REPEAT('a',1000),'a'),(REPEAT('t',932),'t'),('x',NULL),('x',NULL),
+(REPEAT('z',298),'z'),(REPEAT('p',1000),'p'),(REPEAT('k',468),'k'),
+(REPEAT('c',1000),'c'),(REPEAT('o',648),'o'),('x',NULL),('x',NULL),
+(REPEAT('c',258),'c'),(REPEAT('t',414),'t'),(REPEAT('f',966),'f'),
+(REPEAT('y',746),'y'),(REPEAT('f',1000),'f');
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET sort_buffer_size= 16384;
+SELECT LEFT(a,1), GROUP_CONCAT(b) FROM t1 GROUP BY a;
+LEFT(a,1)	GROUP_CONCAT(b)
+a	a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a
+c	c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c
+c	c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c
+f	f,f,f,f,f,f,f,f,f,f,f,f,f,f,f,f
+f	f,f,f,f,f,f,f,f,f,f,f,f,f,f,f,f
+k	k,k,k,k,k,k,k,k,k,k,k,k,k,k,k,k
+o	o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o
+p	p,p,p,p,p,p,p,p,p,p,p,p,p,p,p,p
+t	t,t,t,t,t,t,t,t,t,t,t,t,t,t,t,t
+t	t,t,t,t,t,t,t,t,t,t,t,t,t,t,t,t
+x	NULL
+y	y,y,y,y,y,y,y,y,y,y,y,y,y,y,y,y
+z	z,z,z,z,z,z,z,z,z,z,z,z,z,z,z,z
+SELECT SUBSTR(a,1,1), LENGTH(a), GROUP_CONCAT(b), COUNT(*) FROM t1 GROUP BY a;
+SUBSTR(a,1,1)	LENGTH(a)	GROUP_CONCAT(b)	COUNT(*)
+a	1000	a,a,a,a,a,a,a,a,a,a,a,a,a,a,a,a	16
+c	258	c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c	16
+c	1000	c,c,c,c,c,c,c,c,c,c,c,c,c,c,c,c	16
+f	966	f,f,f,f,f,f,f,f,f,f,f,f,f,f,f,f	16
+f	1000	f,f,f,f,f,f,f,f,f,f,f,f,f,f,f,f	16
+k	468	k,k,k,k,k,k,k,k,k,k,k,k,k,k,k,k	16
+o	648	o,o,o,o,o,o,o,o,o,o,o,o,o,o,o,o	16
+p	1000	p,p,p,p,p,p,p,p,p,p,p,p,p,p,p,p	16
+t	414	t,t,t,t,t,t,t,t,t,t,t,t,t,t,t,t	16
+t	932	t,t,t,t,t,t,t,t,t,t,t,t,t,t,t,t	16
+x	1	NULL	64
+y	746	y,y,y,y,y,y,y,y,y,y,y,y,y,y,y,y	16
+z	298	z,z,z,z,z,z,z,z,z,z,z,z,z,z,z,z	16
+SET @@sort_buffer_size= @save_sort_buffer_size;
+DROP TABLE t1;
+CREATE TABLE t1(a VARCHAR(1027), b INT);
+INSERT INTO t1 SELECT seq, seq from seq_1_to_34;
+SET @save_tmp_memory_table_size= @@tmp_memory_table_size;
+SET tmp_memory_table_size= 1056*2;
+SELECT COUNT(DISTINCT a) FROM t1;
+COUNT(DISTINCT a)
+34
+SET @@tmp_memory_table_size= @save_tmp_memory_table_size;
+DROP TABLE t1;
 # End of 10.5 tests
diff --git a/mysql-test/main/order_by.test b/mysql-test/main/order_by.test
index c78add674ba..be4a3e4253e 100644
--- a/mysql-test/main/order_by.test
+++ b/mysql-test/main/order_by.test
@@ -2465,4 +2465,40 @@ SELECT * FROM t1 ORDER BY CONVERT(AES_ENCRYPT(1,a), CHAR(4));
 --enable_warnings
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-22819:Wrong result or Assertion `ix > 0' failed in read_to_buffer upon select
+--echo # with GROUP BY and GROUP_CONCAT
+--echo #
+
+CREATE TABLE t1 (a VARCHAR(1000), b CHAR(1));
+
+INSERT INTO t1 VALUES
+    (REPEAT('a',1000),'a'),(REPEAT('t',932),'t'),('x',NULL),('x',NULL),
+    (REPEAT('z',298),'z'),(REPEAT('p',1000),'p'),(REPEAT('k',468),'k'),
+    (REPEAT('c',1000),'c'),(REPEAT('o',648),'o'),('x',NULL),('x',NULL),
+    (REPEAT('c',258),'c'),(REPEAT('t',414),'t'),(REPEAT('f',966),'f'),
+    (REPEAT('y',746),'y'),(REPEAT('f',1000),'f');
+
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+INSERT INTO t1 SELECT * FROM t1;
+
+SET @save_sort_buffer_size= @@sort_buffer_size;
+SET sort_buffer_size= 16384;
+SELECT LEFT(a,1), GROUP_CONCAT(b) FROM t1 GROUP BY a;
+SELECT SUBSTR(a,1,1), LENGTH(a), GROUP_CONCAT(b), COUNT(*) FROM t1 GROUP BY a;
+
+SET @@sort_buffer_size= @save_sort_buffer_size;
+DROP TABLE t1;
+
+CREATE TABLE t1(a VARCHAR(1027), b INT);
+INSERT INTO t1 SELECT seq, seq from seq_1_to_34;
+SET @save_tmp_memory_table_size= @@tmp_memory_table_size;
+SET tmp_memory_table_size= 1056*2;
+SELECT COUNT(DISTINCT a) FROM t1;
+SET @@tmp_memory_table_size= @save_tmp_memory_table_size;
+DROP TABLE t1;
+
+
 --echo # End of 10.5 tests
diff --git a/sql/filesort.cc b/sql/filesort.cc
index 6fa9b192655..ac43c96b0e0 100644
--- a/sql/filesort.cc
+++ b/sql/filesort.cc
@@ -1845,17 +1845,15 @@ bool merge_buffers(Sort_param *param, IO_CACHE *from_file,
                          offsetof(Merge_chunk,m_current_key), 0,
                           (queue_compare) cmp, first_cmp_arg, 0, 0)))
     DBUG_RETURN(1);                                /* purecov: inspected */
+  const size_t chunk_sz = (sort_buffer.size()/((uint) (Tb-Fb) +1));
   for (buffpek= Fb ; buffpek <= Tb ; buffpek++)
   {
-    buffpek->set_buffer(strpos,
-                        strpos + (sort_buffer.size()/((uint) (Tb-Fb) +1)));
-
+    buffpek->set_buffer(strpos, strpos + chunk_sz);
     buffpek->set_max_keys(maxcount);
     bytes_read= read_to_buffer(from_file, buffpek, param, packed_format);
     if (unlikely(bytes_read == (ulong) -1))
       goto err;					/* purecov: inspected */
-    strpos+= bytes_read;
-    buffpek->set_buffer_end(strpos);
+    strpos+= chunk_sz;
     // If less data in buffers than expected
     buffpek->set_max_keys(buffpek->mem_count());
     queue_insert(&queue, (uchar*) buffpek);
diff --git a/sql/uniques.cc b/sql/uniques.cc
index 60918ea534e..f5d85bbb5cc 100644
--- a/sql/uniques.cc
+++ b/sql/uniques.cc
@@ -735,6 +735,13 @@ bool Unique::merge(TABLE *table, uchar *buff, size_t buff_size,
   sort_param.cmp_context.key_compare= tree.compare;
   sort_param.cmp_context.key_compare_arg= tree.custom_arg;
 
+  /*
+    We need to remove the size allocated for the unique bufffer.
+    The sort_buffer_size is:
+      MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size;
+  */
+  buff_size-= full_size;
+
   /* Merge the buffers to one file, removing duplicates */
   if (merge_many_buff(&sort_param,
                       Bounds_checked_array<uchar>(buff, buff_size),
@@ -802,7 +809,8 @@ bool Unique::get(TABLE *table)
   /* Not enough memory; Save the result to file && free memory used by tree */
   if (flush())
     DBUG_RETURN(1);
-  size_t buff_sz= (max_in_memory_size / full_size + 1) * full_size;
+  size_t buff_sz= MY_MAX(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size;
+
   if (!(sort_buffer= (uchar*) my_malloc(key_memory_Unique_sort_buffer, buff_sz,
                                         MYF(MY_THREAD_SPECIFIC|MY_WME))))
     DBUG_RETURN(1);