MDEV-23809: Server crash in JOIN_CACHE::free or ...

The problem was caused by use of COLLATION(AVG('x')). This is an
item whose value is a constant.
Name Resolution code called convert_const_to_int() which removed AVG('x').
However, the item representing COLLATION(...) still had with_sum_func=1.

This inconsistent state confused the code that handles grouping and
DISTINCT: JOIN::get_best_combination() decided to use one temporary
table and allocated one JOIN_TAB for it, but then
JOIN::make_aggr_tables_info() attempted to use two and made writes
beyond the end of the JOIN::join_tab array.

The fix:
- Do not replace constant expressions which contain aggregate functions.
- Add JOIN::dbug_join_tab_array_size to catch attempts to use more
  JOIN_TAB objects than we've allocated.

5 files changed, 69 insertions, 1 deletions

diff --git a/mysql-test/main/func_group.result b/mysql-test/main/func_group.result
index 37d8e15e52d..d5ac57d5443 100644
--- a/mysql-test/main/func_group.result
+++ b/mysql-test/main/func_group.result
@@ -2523,5 +2523,30 @@ DROP TABLE t2;
 DROP VIEW v1;
 DROP TABLE t1;
 #
+# MDEV-23809: Server crash in JOIN_CACHE::free or ...
+#
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+SELECT DISTINCT CASE CONVERT(EXPORT_SET(0, COLLATION(BENCHMARK(1, BIT_OR(0))),0),TIME) WHEN a THEN 1 END AS f FROM t1;
+f
+NULL
+Warnings:
+Warning	1292	Truncated incorrect time value: '0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0'
+DROP TABLE t1;
+CREATE TABLE t1 (a VARCHAR(8) NULL, b BIGINT);
+INSERT INTO t1 (a,b) VALUES (NULL,NULL),('foo',NULL);
+SELECT DISTINCT STRCMP((b > COLLATION(STDDEV_SAMP(15750))), a) AS f FROM t1;
+f
+NULL
+DROP TABLE t1;
+CREATE TABLE t1 (a BIGINT) AS SELECT 1 AS v3 UNION SELECT FALSE ;
+SELECT DISTINCT a IN ( COLLATION (AVG ('x'))) FROM t1 ;
+a IN ( COLLATION (AVG ('x')))
+NULL
+Warnings:
+Warning	1292	Truncated incorrect DOUBLE value: 'x'
+Warning	1292	Truncated incorrect DOUBLE value: 'x'
+DROP TABLE t1;
+#
 # End of 10.3 tests
 #
diff --git a/mysql-test/main/func_group.test b/mysql-test/main/func_group.test
index fb2106ac3ae..f91aee3e577 100644
--- a/mysql-test/main/func_group.test
+++ b/mysql-test/main/func_group.test
@@ -1757,5 +1757,24 @@ DROP VIEW v1;
 DROP TABLE t1;
 
 --echo #
+--echo # MDEV-23809: Server crash in JOIN_CACHE::free or ...
+--echo #
+
+CREATE TABLE t1 (a INT);
+INSERT INTO t1 VALUES (1),(2);
+SELECT DISTINCT CASE CONVERT(EXPORT_SET(0, COLLATION(BENCHMARK(1, BIT_OR(0))),0),TIME) WHEN a THEN 1 END AS f FROM t1;
+DROP TABLE t1;
+
+
+CREATE TABLE t1 (a VARCHAR(8) NULL, b BIGINT);
+INSERT INTO t1 (a,b) VALUES (NULL,NULL),('foo',NULL);
+SELECT DISTINCT STRCMP((b > COLLATION(STDDEV_SAMP(15750))), a) AS f FROM t1;
+DROP TABLE t1;
+
+CREATE TABLE t1 (a BIGINT) AS SELECT 1 AS v3 UNION SELECT FALSE ;
+SELECT DISTINCT a IN ( COLLATION (AVG ('x'))) FROM t1 ;
+DROP TABLE t1;
+
+--echo #
 --echo # End of 10.3 tests
 --echo #
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index a3c0d4d95df..fe6b8feb4de 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -340,7 +340,18 @@ static bool convert_const_to_int(THD *thd, Item_field *field_item,
       field_item->field_type() != MYSQL_TYPE_YEAR)
     return 1;
 
-  if ((*item)->const_item() && !(*item)->is_expensive())
+  /*
+    Replace (*item) with its value if the item can be computed.
+
+    Do not replace items that contain aggregate functions:
+    There can be such items that are constants, e.g. COLLATION(AVG(123)),
+    but this function is called at Name Resolution phase.
+    Removing aggregate functions may confuse query plan generation code, e.g.
+    the optimizer might conclude that the query doesn't need to do grouping
+    at all.
+  */
+  if ((*item)->const_item() && !(*item)->is_expensive() &&
+      !(*item)->with_sum_func)
   {
     TABLE *table= field->table;
     Sql_mode_save sql_mode(thd);
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 33fb19966b2..a865c75184f 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -1596,6 +1596,9 @@ JOIN::optimize_inner()
   DEBUG_SYNC(thd, "before_join_optimize");
 
   THD_STAGE_INFO(thd, stage_optimizing);
+#ifndef DBUG_OFF
+  dbug_join_tab_array_size= 0;
+#endif
 
   set_allowed_join_cache_types();
   need_distinct= TRUE;
@@ -2740,6 +2743,9 @@ setup_subq_exit:
     {
       if (!(join_tab= (JOIN_TAB*) thd->alloc(sizeof(JOIN_TAB))))
         DBUG_RETURN(1);
+#ifndef DBUG_OFF
+      dbug_join_tab_array_size= 1;
+#endif
       need_tmp= 1;
     }
     if (make_aggr_tables_info())
@@ -3044,6 +3050,7 @@ bool JOIN::make_aggr_tables_info()
   {
     aggr_tables++;
     curr_tab= join_tab + exec_join_tab_cnt();
+    DBUG_ASSERT(curr_tab - join_tab < dbug_join_tab_array_size);
     bzero((void*)curr_tab, sizeof(JOIN_TAB));
     curr_tab->ref.key= -1;
     if (only_const_tables())
@@ -3172,6 +3179,7 @@ bool JOIN::make_aggr_tables_info()
 
       curr_tab++;
       aggr_tables++;
+      DBUG_ASSERT(curr_tab - join_tab < dbug_join_tab_array_size);
       bzero((void*)curr_tab, sizeof(JOIN_TAB));
       curr_tab->ref.key= -1;
 
@@ -9781,6 +9789,10 @@ bool JOIN::get_best_combination()
 
   fix_semijoin_strategies_for_picked_join_order(this);
 
+#ifndef DBUG_OFF
+  dbug_join_tab_array_size= top_join_tab_count + aggr_tables;
+#endif
+
   if (inject_splitting_cond_for_all_tables_with_split_opt())
     DBUG_RETURN(TRUE);
    
diff --git a/sql/sql_select.h b/sql/sql_select.h
index c8ff8b24385..fe6a640003f 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1265,6 +1265,7 @@ public:
 
 #ifndef DBUG_OFF
   void dbug_verify_sj_inner_tables(uint n_positions) const;
+  uint dbug_join_tab_array_size;
 #endif
 
   /* We also maintain a stack of join optimization states in * join->positions[] */