MDEV-13458: Wrong result for aggregate function with distinct clause when the value for tmp_table_size is small

Fixed by making sure that the sort buffer would have atleast MERGEBUFF2 keys.
Also fixed MDEV-13457 by making sure that an empty tree is never dumped to the disk

4 files changed, 63 insertions, 5 deletions

diff --git a/mysql-test/r/count_distinct.result b/mysql-test/r/count_distinct.result
index d55a232c715..760b2710586 100644
--- a/mysql-test/r/count_distinct.result
+++ b/mysql-test/r/count_distinct.result
@@ -106,3 +106,28 @@ count(distinct user_id)
 17
 drop table t1;
 set @@tmp_table_size = default;
+create table t1 (
+a VARCHAR(1020),
+b int
+);
+insert into t1 values
+( 0 , 1 ),
+( 1 , 2 ),
+( 2 , 3 ),
+( 3 , 4 ),
+( 4 , 5 ),
+( 5 , 6 ),
+( 6 , 7 ),
+( 7 , 8 ),
+( 8 , 9 ),
+( 9 , 10 ),
+( 0 , 11 ),
+( 1 , 12 ),
+( 2 , 13 ),
+( 3 , 14 );
+set @@tmp_table_size=1024;
+select count(distinct a) from t1;
+count(distinct a)
+10
+drop table t1;
+set @@tmp_table_size = default;
diff --git a/mysql-test/t/count_distinct.test b/mysql-test/t/count_distinct.test
index a00574b6cba..86045e862e7 100644
--- a/mysql-test/t/count_distinct.test
+++ b/mysql-test/t/count_distinct.test
@@ -121,5 +121,34 @@ drop table t1;
 set @@tmp_table_size = default;
 
 #
+# MDEV-13457: Wrong result for aggregate function with distinct clause when the value for 
+# tmp_table_size is small
+#
+
+create table t1 (
+a VARCHAR(1020),
+b int
+);
+insert into t1 values
+( 0 , 1 ),
+( 1 , 2 ),
+( 2 , 3 ),
+( 3 , 4 ),
+( 4 , 5 ),
+( 5 , 6 ),
+( 6 , 7 ),
+( 7 , 8 ),
+( 8 , 9 ),
+( 9 , 10 ),
+( 0 , 11 ),
+( 1 , 12 ),
+( 2 , 13 ),
+( 3 , 14 );
+set @@tmp_table_size=1024;
+select count(distinct a) from t1;
+drop table t1;
+set @@tmp_table_size = default;
+
+#
 # End of 5.5 tests
 #
diff --git a/sql/sql_class.h b/sql/sql_class.h
index 5dd7cd18a5d..f4be996c9a9 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -3919,7 +3919,7 @@ public:
   {
     DBUG_ENTER("unique_add");
     DBUG_PRINT("info", ("tree %u - %lu", tree.elements_in_tree, max_elements));
-    if (!(tree.flag & TREE_ONLY_DUPS) && 
+    if (!(tree.flag & TREE_ONLY_DUPS) &&
         tree.elements_in_tree >= max_elements && flush())
       DBUG_RETURN(1);
     DBUG_RETURN(!tree_insert(&tree, ptr, 0, tree.custom_arg));
diff --git a/sql/uniques.cc b/sql/uniques.cc
index 8b7da7e6e52..89ab4682829 100644
--- a/sql/uniques.cc
+++ b/sql/uniques.cc
@@ -96,6 +96,9 @@ Unique::Unique(qsort_cmp2 comp_func, void * comp_func_fixed_arg,
   */
   max_elements= (ulong) (max_in_memory_size /
                          ALIGN_SIZE(sizeof(TREE_ELEMENT)+size));
+  if (!max_elements)
+    max_elements= 1;
+
   (void) open_cached_file(&file, mysql_tmpdir,TEMP_PREFIX, DISK_BUFFER_SIZE,
                           MYF(MY_WME));
 }
@@ -608,10 +611,11 @@ bool Unique::walk(TABLE *table, tree_walk_action action, void *walk_action_arg)
   if (flush_io_cache(&file) || reinit_io_cache(&file, READ_CACHE, 0L, 0, 0))
     return 1;
   /*
-    merge_buffer must fit at least MERGEBUFF2 keys, because
-    merge_index() can merge that many BUFFPEKs at once.
+    merge_buffer must fit at least MERGEBUFF2 + 1 keys, because
+    merge_index() can merge that many BUFFPEKs at once. The extra space for one key 
+    is needed when a piece of merge buffer is re-read, see merge_walk()
   */
-  size_t buff_sz= max(MERGEBUFF2, max_in_memory_size/full_size+1) * full_size;
+  size_t buff_sz= max(MERGEBUFF2+1, max_in_memory_size/full_size+1) * full_size;
   if (!(merge_buffer = (uchar *)my_malloc(buff_sz, MYF(MY_WME))))
     return 1;
   if (buff_sz < (ulong) (full_size * (file_ptrs.elements + 1)))
@@ -673,7 +677,7 @@ bool Unique::merge(TABLE *table, uchar *buff, bool without_last_merge)
    full_size;
   sort_param.min_dupl_count= min_dupl_count;
   sort_param.res_length= 0;
-  sort_param.keys= (uint) (max_in_memory_size / sort_param.sort_length);
+  sort_param.keys= (uint) max((max_in_memory_size / sort_param.sort_length), MERGEBUFF2);
   sort_param.not_killable= 1;
 
   sort_param.unique_buff= buff + (sort_param.keys * sort_param.sort_length);