MDEV-22019: Sig 11 in next_breadth_first_tab | max_sort_length setting + double GROUP BY leads to crash

No need to create a temp table for aggregation if we have encountered some error.

4 files changed, 40 insertions, 6 deletions

diff --git a/mysql-test/r/group_by.result b/mysql-test/r/group_by.result
index cbce6340c47..c996627486c 100644
--- a/mysql-test/r/group_by.result
+++ b/mysql-test/r/group_by.result
@@ -2861,3 +2861,17 @@ SELECT 1 FROM t1
 GROUP BY REPEAT((BINARY pk), v1), AES_DECRYPT((@A := i1), 20852) WITH ROLLUP HAVING LOAD_FILE('a') ;
 1
 drop table t1;
+#
+# MDEV-22019: Sig 11 in next_breadth_first_tab | max_sort_length setting + double
+# GROUP BY leads to crash
+#
+CALL mtr.add_suppression("Out of sort memory");
+CALL mtr.add_suppression("Sort aborted");
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=2000000;
+SELECT * FROM information_schema.tables t JOIN information_schema.columns c
+ON t.table_schema=c.table_schema
+WHERE c.table_schema=(SELECT COUNT(*) FROM INFORMATION_SCHEMA.columns GROUP BY column_type)
+GROUP BY t.table_name;
+ERROR HY001: Out of sort memory, consider increasing server sort buffer size
+SET max_sort_length= @save_max_sort_length;
diff --git a/mysql-test/t/group_by.test b/mysql-test/t/group_by.test
index 0436cd16e7c..324b41ce23c 100644
--- a/mysql-test/t/group_by.test
+++ b/mysql-test/t/group_by.test
@@ -1979,3 +1979,20 @@ GROUP BY REPEAT((BINARY pk), v1), AES_DECRYPT((@A := i1), 20852) WITH ROLLUP HAV
 SELECT 1 FROM t1
 GROUP BY REPEAT((BINARY pk), v1), AES_DECRYPT((@A := i1), 20852) WITH ROLLUP HAVING LOAD_FILE('a') ;
 drop table t1;
+
+--echo #
+--echo # MDEV-22019: Sig 11 in next_breadth_first_tab | max_sort_length setting + double
+--echo # GROUP BY leads to crash
+--echo #
+
+
+CALL mtr.add_suppression("Out of sort memory");
+CALL mtr.add_suppression("Sort aborted");
+SET @save_max_sort_length= @@max_sort_length;
+SET max_sort_length=2000000;
+--error ER_OUT_OF_SORTMEMORY
+SELECT * FROM information_schema.tables t JOIN information_schema.columns c
+ON t.table_schema=c.table_schema
+WHERE c.table_schema=(SELECT COUNT(*) FROM INFORMATION_SCHEMA.columns GROUP BY column_type)
+GROUP BY t.table_name;
+SET max_sort_length= @save_max_sort_length;
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 97002ef071f..c601946cfa0 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -2210,8 +2210,12 @@ JOIN::optimize_inner()
     having_is_correlated= MY_TEST(having->used_tables() & OUTER_REF_TABLE_BIT);
   tmp_having= having;
 
-  if ((select_lex->options & OPTION_SCHEMA_TABLE))
-    optimize_schema_tables_reads(this);
+  if ((select_lex->options & OPTION_SCHEMA_TABLE) &&
+       optimize_schema_tables_reads(this))
+    DBUG_RETURN(TRUE);
+
+  if (unlikely(thd->is_error()))
+    DBUG_RETURN(TRUE);
 
   /*
     The loose index scan access method guarantees that all grouping or
diff --git a/sql/sql_show.cc b/sql/sql_show.cc
index 78ca1d09e7a..4e37b53b87e 100644
--- a/sql/sql_show.cc
+++ b/sql/sql_show.cc
@@ -8299,7 +8299,6 @@ end:
 bool optimize_schema_tables_reads(JOIN *join)
 {
   THD *thd= join->thd;
-  bool result= 0;
   DBUG_ENTER("optimize_schema_tables_reads");
 
   JOIN_TAB *tab;
@@ -8334,11 +8333,11 @@ bool optimize_schema_tables_reads(JOIN *join)
         */
         cond= tab->cache_select->cond;
       }
-
-      optimize_for_get_all_tables(thd, table_list, cond);
+      if (optimize_for_get_all_tables(thd, table_list, cond))
+        DBUG_RETURN(TRUE);   // Handle OOM
     }
   }
-  DBUG_RETURN(result);
+  DBUG_RETURN(FALSE);
 }