MDEV-11162 Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint)

When JOIN::destroy() is called for a JOIN object that has
   - join->tmp_join != NULL
   - also has join->table[0]->sort

then the latter was not cleaned up.
This could cause a memory leak and/or asserts in the subsequent queries.

Fixed by adding a cleanup call.

3 files changed, 18 insertions, 0 deletions

diff --git a/mysql-test/r/group_by_innodb.result b/mysql-test/r/group_by_innodb.result
index 381e0d7493c..803efb8fd5c 100644
--- a/mysql-test/r/group_by_innodb.result
+++ b/mysql-test/r/group_by_innodb.result
@@ -123,4 +123,11 @@ id	xtext	optionen
 2	number	22,25
 1	select	Kabel mit Stecker 5-polig,Kabel ohne Stecker
 DROP TABLE t1, t2;
+CREATE TABLE t1 (i INT) ENGINE=InnoDB;
+SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i );
+( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i )
+NULL
+SELECT i FROM t1 order by i LIMIT 1;
+i
+DROP TABLE t1;
 # End of tests
diff --git a/mysql-test/t/group_by_innodb.test b/mysql-test/t/group_by_innodb.test
index e072a94fada..074df222294 100644
--- a/mysql-test/t/group_by_innodb.test
+++ b/mysql-test/t/group_by_innodb.test
@@ -125,4 +125,14 @@ ORDER BY id DESC;
 
 DROP TABLE t1, t2;
 
+--echo #
+--echo # MDEV-11162: Assertion `num_records == m_idx_array.size()' failed in Filesort_buffer::alloc_sort_buffer(uint, uint)
+--echo #
+
+CREATE TABLE t1 (i INT) ENGINE=InnoDB;
+SELECT ( SELECT DISTINCT GROUP_CONCAT(SLEEP(0)) FROM t1 GROUP BY i );
+SELECT i FROM t1 order by i LIMIT 1;
+DROP TABLE t1;
+
 --echo # End of tests
+
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 2db9a2b8482..f345d3c9687 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -3144,6 +3144,7 @@ JOIN::destroy()
     */
     tmp_table_param.cleanup();
     tmp_join->tmp_table_param.copy_field= 0;
+    cleanup(1);
     DBUG_RETURN(tmp_join->destroy());
   }
   cond_equal= 0;