MDEV-10776: Server crash on query

Exclude untouched in prepare phese subqueries from the select/unit tree
because they became unreachable by execution.

8 files changed, 106 insertions, 5 deletions

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 16ab4cf9d82..0a599a64f80 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7094,3 +7094,14 @@ a
 0
 DROP TABLE t1;
 SET SESSION big_tables=0;
+#
+# MDEV-10776: Server crash on query
+#
+create table t1 (field1 int);
+insert into t1 values (1);
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+round((select 1 from t1 limit 1))
+1
+drop table t1;
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index aed4fb39a84..b819b1e4ef9 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7091,6 +7091,17 @@ a
 0
 DROP TABLE t1;
 SET SESSION big_tables=0;
+#
+# MDEV-10776: Server crash on query
+#
+create table t1 (field1 int);
+insert into t1 values (1);
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+round((select 1 from t1 limit 1))
+1
+drop table t1;
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index e1f08537788..e1001a51658 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7089,4 +7089,15 @@ a
 0
 DROP TABLE t1;
 SET SESSION big_tables=0;
+#
+# MDEV-10776: Server crash on query
+#
+create table t1 (field1 int);
+insert into t1 values (1);
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+round((select 1 from t1 limit 1))
+1
+drop table t1;
 set @optimizer_switch_for_subselect_test=null;
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index b5fa7b1dd8f..e175e7e0072 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7100,6 +7100,17 @@ a
 0
 DROP TABLE t1;
 SET SESSION big_tables=0;
+#
+# MDEV-10776: Server crash on query
+#
+create table t1 (field1 int);
+insert into t1 values (1);
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+round((select 1 from t1 limit 1))
+1
+drop table t1;
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index b815559433c..a211d498762 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7089,5 +7089,16 @@ a
 0
 DROP TABLE t1;
 SET SESSION big_tables=0;
+#
+# MDEV-10776: Server crash on query
+#
+create table t1 (field1 int);
+insert into t1 values (1);
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+round((select 1 from t1 limit 1))
+1
+drop table t1;
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index b802761aff3..77b6c6c5582 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -5974,3 +5974,17 @@ INSERT INTO t1 VALUES(0),(0),(0);
 SELECT * FROM t1 WHERE a IN(SELECT MIN(a) FROM t1);
 DROP TABLE t1;
 SET SESSION big_tables=0;
+
+
+--echo #
+--echo # MDEV-10776: Server crash on query
+--echo #
+create table t1 (field1 int);
+
+insert into t1 values (1);
+
+select round((select 1 from t1 limit 1))
+from t1
+group by round((select 1 from t1 limit 1));
+
+drop table t1;
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 3727711a395..6427b0ecae4 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -209,6 +209,7 @@ bool
 Item_subselect::select_transformer(JOIN *join)
 {
   DBUG_ENTER("Item_subselect::select_transformer");
+  DBUG_ASSERT(thd == join->thd);
   DBUG_RETURN(false);
 }
 
@@ -579,7 +580,7 @@ bool Item_subselect::is_expensive()
 
     examined_rows+= cur_join->get_examined_rows();
   }
-
+  // here we are sure that subquery is optimized so thd is set
   return (examined_rows > thd->variables.expensive_subquery_limit);
 }
 
@@ -643,6 +644,7 @@ bool Item_subselect::exec()
   subselect_engine *org_engine= engine;
 
   DBUG_ENTER("Item_subselect::exec");
+  DBUG_ASSERT(fixed);
 
   /*
     Do not execute subselect in case of a fatal error
@@ -688,6 +690,7 @@ int Item_in_subselect::optimize(double *out_rows, double *cost)
 {
   int res;
   DBUG_ENTER("Item_in_subselect::optimize");
+  DBUG_ASSERT(fixed);
   SELECT_LEX *save_select= thd->lex->current_select;
   JOIN *join= unit->first_select()->join;
 
@@ -802,6 +805,7 @@ bool Item_in_subselect::expr_cache_is_needed(THD *thd)
 bool Item_in_subselect::exec()
 {
   DBUG_ENTER("Item_in_subselect::exec");
+  DBUG_ASSERT(fixed);
   /*
     Initialize the cache of the left predicate operand. This has to be done as
     late as now, because Cached_item directly contains a resolved field (not
@@ -856,6 +860,7 @@ table_map Item_subselect::used_tables() const
 
 bool Item_subselect::const_item() const
 {
+  DBUG_ASSERT(thd);
   return (thd->lex->context_analysis_only ?
           FALSE :
           forced_const || const_item_cache);
@@ -1049,10 +1054,11 @@ Item_singlerow_subselect::select_transformer(JOIN *join)
   DBUG_ENTER("Item_singlerow_subselect::select_transformer");
   if (changed)
     DBUG_RETURN(false);
+  DBUG_ASSERT(join->thd == thd);
 
   SELECT_LEX *select_lex= join->select_lex;
   Query_arena *arena= thd->stmt_arena;
- 
+
   if (!select_lex->master_unit()->is_union() &&
       !select_lex->table_list.elements &&
       select_lex->item_list.elements == 1 &&
@@ -1717,6 +1723,7 @@ Item_in_subselect::single_value_transformer(JOIN *join)
 {
   SELECT_LEX *select_lex= join->select_lex;
   DBUG_ENTER("Item_in_subselect::single_value_transformer");
+  DBUG_ASSERT(thd == join->thd);
 
   /*
     Check that the right part of the subselect contains no more than one
@@ -1829,9 +1836,9 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
   if (!test_strategy(SUBS_MAXMIN_INJECTED | SUBS_MAXMIN_ENGINE))
     DBUG_RETURN(false);
   Item **place= optimizer->arguments() + 1;
-  THD *thd= join->thd;
   SELECT_LEX *select_lex= join->select_lex;
   Item *subs;
+  DBUG_ASSERT(thd == join->thd);
 
   /*
   */
@@ -1938,6 +1945,7 @@ bool Item_allany_subselect::transform_into_max_min(JOIN *join)
 bool Item_in_subselect::fix_having(Item *having, SELECT_LEX *select_lex)
 {
   bool fix_res= 0;
+  DBUG_ASSERT(thd);
   if (!having->fixed)
   {
     select_lex->having_fix_field= 1;
@@ -2000,6 +2008,7 @@ Item_in_subselect::create_single_in_to_exists_cond(JOIN *join,
                                                    Item **having_item)
 {
   SELECT_LEX *select_lex= join->select_lex;
+  DBUG_ASSERT(thd == join->thd);
   /*
     The non-transformed HAVING clause of 'join' may be stored in two ways
     during JOIN::optimize: this->tmp_having= this->having; this->having= 0;
@@ -2136,6 +2145,7 @@ Item_in_subselect::row_value_transformer(JOIN *join)
   uint cols_num= left_expr->cols();
 
   DBUG_ENTER("Item_in_subselect::row_value_transformer");
+  DBUG_ASSERT(thd == join->thd);
 
   // psergey: duplicated_subselect_card_check
   if (select_lex->item_list.elements != cols_num)
@@ -2248,6 +2258,7 @@ Item_in_subselect::create_row_in_to_exists_cond(JOIN * join,
                         !select_lex->table_list.elements);
 
   DBUG_ENTER("Item_in_subselect::create_row_in_to_exists_cond");
+  DBUG_ASSERT(thd == join->thd);
 
   *where_item= NULL;
   *having_item= NULL;
@@ -2473,6 +2484,7 @@ bool Item_in_subselect::inject_in_to_exists_cond(JOIN *join_arg)
   Item *having_item= join_arg->in_to_exists_having;
 
   DBUG_ENTER("Item_in_subselect::inject_in_to_exists_cond");
+  DBUG_ASSERT(thd == join_arg->thd);
 
   if (where_item)
   {
@@ -2561,6 +2573,7 @@ Item_in_subselect::select_in_like_transformer(JOIN *join)
   bool result;
 
   DBUG_ENTER("Item_in_subselect::select_in_like_transformer");
+  DBUG_ASSERT(thd == join->thd);
 
   /*
     IN/SOME/ALL/ANY subqueries aren't support LIMIT clause. Without it
@@ -2762,6 +2775,7 @@ bool Item_in_subselect::setup_mat_engine()
   subselect_single_select_engine *select_engine;
 
   DBUG_ENTER("Item_in_subselect::setup_mat_engine");
+  DBUG_ASSERT(thd);
 
   /*
     The select_engine (that executes transformed IN=>EXISTS subselects) is
@@ -2800,6 +2814,7 @@ bool Item_in_subselect::setup_mat_engine()
 bool Item_in_subselect::init_left_expr_cache()
 {
   JOIN *outer_join;
+  DBUG_ASSERT(thd);
 
   outer_join= unit->outer_select()->join;
   /*
@@ -2826,6 +2841,7 @@ bool Item_in_subselect::init_left_expr_cache()
 
 bool Item_in_subselect::init_cond_guards()
 {
+  DBUG_ASSERT(thd);
   uint cols_num= left_expr->cols();
   if (!abort_on_null && left_expr->maybe_null && !pushed_cond_guards)
   {
diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc
index 2462f0fea17..f2e7b4f7c3a 100644
--- a/sql/sql_lex.cc
+++ b/sql/sql_lex.cc
@@ -3486,12 +3486,28 @@ bool st_select_lex::add_index_hint (THD *thd, char *str, uint length)
 
 bool st_select_lex::optimize_unflattened_subqueries(bool const_only)
 {
-  for (SELECT_LEX_UNIT *un= first_inner_unit(); un; un= un->next_unit())
+  SELECT_LEX_UNIT *next_unit= NULL;
+  for (SELECT_LEX_UNIT *un= first_inner_unit();
+       un;
+       un= next_unit ? next_unit : un->next_unit())
   {
     Item_subselect *subquery_predicate= un->item;
-    
+    next_unit= NULL;
+
     if (subquery_predicate)
     {
+      if (!subquery_predicate->fixed)
+      {
+	/*
+	 This subquery was excluded as part of some expression so it is
+	 invisible from all prepared expression.
+       */
+	next_unit= un->next_unit();
+	un->exclude_level();
+	if (next_unit)
+	  continue;
+	break;
+      }
       if (subquery_predicate->substype() == Item_subselect::IN_SUBS)
       {
         Item_in_subselect *in_subs= (Item_in_subselect*) subquery_predicate;