MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops

Do not set 'optimized' flag until whole optimization procedure is finished.

10 files changed, 155 insertions, 34 deletions

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 6a531997d79..428cf89c36c 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7157,5 +7157,21 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
diff --git a/mysql-test/r/subselect_no_exists_to_in.result b/mysql-test/r/subselect_no_exists_to_in.result
index aa6843409c0..b24edd438b2 100644
--- a/mysql-test/r/subselect_no_exists_to_in.result
+++ b/mysql-test/r/subselect_no_exists_to_in.result
@@ -7157,8 +7157,24 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%exists_to_in=off%';
 @@optimizer_switch like '%exists_to_in=off%'
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index 754aec1db20..23b8ade7ef5 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7150,8 +7150,24 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index e05dd4d140d..2907fd3f4b3 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7148,6 +7148,22 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
 set @optimizer_switch_for_subselect_test=null;
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index 71ade62b423..08394bc6332 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7163,8 +7163,24 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index 43d191b1225..9b1d4d24031 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7148,7 +7148,23 @@ INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
 f
 foo
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
+f
+foo
 drop table t1;
 SET NAMES default;
+#
+# MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+#
+SET NAMES utf8;
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+f1	f2	f3
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+f1	f2	f3
+DROP TABLE t1, t2;
+SET NAMES default;
+End of 10.1 tests
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index 3599b523d91..dba2154ef73 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -6009,5 +6009,22 @@ SET NAMES utf8;
 CREATE TABLE t1 (f VARCHAR(8)) ENGINE=MyISAM;
 INSERT INTO t1 VALUES ('foo');
 SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar' UNION SELECT 'baz' );
+SELECT f FROM t1 WHERE f > ALL ( SELECT 'bar');
 drop table t1;
 SET NAMES default;
+
+--echo #
+--echo # MDEV-10045: Server crashes in Time_and_counter_tracker::incr_loops
+--echo #
+SET NAMES utf8;
+
+CREATE TABLE t1 (f1 VARCHAR(3), f2 INT UNSIGNED) ENGINE=MyISAM;
+CREATE TABLE t2 (f3 INT) ENGINE=MyISAM;
+
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo' UNION SELECT 'bar' );
+SELECT * FROM t1, t2 WHERE f3 = f2 AND f1 > ANY ( SELECT 'foo');
+
+DROP TABLE t1, t2;
+SET NAMES default;
+
+--echo End of 10.1 tests
diff --git a/sql/item_subselect.cc b/sql/item_subselect.cc
index 301856ea3b8..2999e01d166 100644
--- a/sql/item_subselect.cc
+++ b/sql/item_subselect.cc
@@ -560,6 +560,21 @@ void Item_subselect::recalc_used_tables(st_select_lex *new_parent,
 bool Item_subselect::is_expensive()
 {
   double examined_rows= 0;
+  bool all_are_simple= true;
+
+  /* check extremely simple select */
+  if (!unit->first_select()->next_select()) // no union
+  {
+    /*
+      such single selects works even without optimization because
+      can not makes loops
+    */
+    SELECT_LEX *sl= unit->first_select();
+    JOIN *join = sl->join;
+    if (join && !join->tables_list && !sl->first_inner_unit())
+      return false;
+  }
+
 
   for (SELECT_LEX *sl= unit->first_select(); sl; sl= sl->next_select())
   {
@@ -569,23 +584,27 @@ bool Item_subselect::is_expensive()
     if (!cur_join)
       return true;
 
-    /* very simple subquery */
-    if (!cur_join->tables_list && !sl->first_inner_unit())
-      return false;
-
     /*
       If the subquery is not optimised or in the process of optimization
       it supposed to be expensive
     */
-    if (!cur_join->optimized)
+    if (cur_join->optimization_state != JOIN::OPTIMIZATION_DONE)
       return true;
 
+    if (!cur_join->tables_list && !sl->first_inner_unit())
+      continue;
+
     /*
       Subqueries whose result is known after optimization are not expensive.
       Such subqueries have all tables optimized away, thus have no join plan.
     */
     if ((cur_join->zero_result_cause || !cur_join->tables_list))
-      return false;
+      continue;
+
+    /*
+      This is not simple SELECT in union so we can not go by simple condition
+    */
+    all_are_simple= false;
 
     /*
       If a subquery is not optimized we cannot estimate its cost. A subquery is
@@ -606,7 +625,8 @@ bool Item_subselect::is_expensive()
     examined_rows+= cur_join->get_examined_rows();
   }
 
-  return (examined_rows > thd->variables.expensive_subquery_limit);
+  return !all_are_simple &&
+    (examined_rows > thd->variables.expensive_subquery_limit);
 }
 
 
@@ -3672,7 +3692,7 @@ int subselect_single_select_engine::exec()
   SELECT_LEX *save_select= thd->lex->current_select;
   thd->lex->current_select= select_lex;
 
-  if (!join->optimized)
+  if (join->optimization_state == JOIN::NOT_OPTIMIZED)
   {
     SELECT_LEX_UNIT *unit= select_lex->master_unit();
 
@@ -5321,7 +5341,8 @@ int subselect_hash_sj_engine::exec()
   */
   thd->lex->current_select= materialize_engine->select_lex;
   /* The subquery should be optimized, and materialized only once. */
-  DBUG_ASSERT(materialize_join->optimized && !is_materialized);
+  DBUG_ASSERT(materialize_join->optimization_state == JOIN::OPTIMIZATION_DONE &&
+              !is_materialized);
   materialize_join->exec();
   if ((res= MY_TEST(materialize_join->error || thd->is_fatal_error ||
                     thd->is_error())))
diff --git a/sql/sql_select.cc b/sql/sql_select.cc
index 387e0403f96..96ac6f43c45 100644
--- a/sql/sql_select.cc
+++ b/sql/sql_select.cc
@@ -694,7 +694,7 @@ JOIN::prepare(Item ***rref_pointer_array,
   DBUG_ENTER("JOIN::prepare");
 
   // to prevent double initialization on EXPLAIN
-  if (optimized)
+  if (optimization_state != JOIN::NOT_OPTIMIZED)
     DBUG_RETURN(0);
 
   conds= conds_init;
@@ -1032,24 +1032,13 @@ err:
 
 int JOIN::optimize()
 {
-  bool was_optimized= optimized;
+  // to prevent double initialization on EXPLAIN
+  if (optimization_state != JOIN::NOT_OPTIMIZED)
+    return FALSE;
+  optimization_state= JOIN::OPTIMIZATION_IN_PROGRESS;
+
   int res= optimize_inner();
-  /*
-    If we're inside a non-correlated subquery, this function may be 
-    called for the second time after the subquery has been executed
-    and deleted. The second call will not produce a valid query plan, it will
-    short-circuit because optimized==TRUE.
-
-    "was_optimized != optimized" is here to handle this case:
-      - first optimization starts, gets an error (from a const. cheap
-        subquery), returns 1
-      - another JOIN::optimize() call made, and now join->optimize() will
-        return 0, even though we never had a query plan.
-
-    Can have QEP_NOT_PRESENT_YET for degenerate queries (for example,
-    SELECT * FROM tbl LIMIT 0)
-  */
-  if (was_optimized != optimized && !res && have_query_plan != QEP_DELETED)
+  if (!res && have_query_plan != QEP_DELETED)
   {
     create_explain_query_if_not_exists(thd->lex, thd->mem_root);
     have_query_plan= QEP_AVAILABLE;
@@ -1058,6 +1047,7 @@ int JOIN::optimize()
                       !skip_sort_order && !no_order && (order || group_list),
                       select_distinct);
   }
+  optimization_state= JOIN::OPTIMIZATION_DONE;
   return res;
 }
 
@@ -1083,10 +1073,6 @@ JOIN::optimize_inner()
   DBUG_ENTER("JOIN::optimize");
 
   do_send_rows = (unit->select_limit_cnt) ? 1 : 0;
-  // to prevent double initialization on EXPLAIN
-  if (optimized)
-    DBUG_RETURN(0);
-  optimized= 1;
   DEBUG_SYNC(thd, "before_join_optimize");
 
   THD_STAGE_INFO(thd, stage_optimizing);
@@ -2060,7 +2046,7 @@ int JOIN::init_execution()
 {
   DBUG_ENTER("JOIN::init_execution");
 
-  DBUG_ASSERT(optimized);
+  DBUG_ASSERT(optimization_state == JOIN::OPTIMIZATION_DONE);
   DBUG_ASSERT(!(select_options & SELECT_DESCRIBE));
   initialized= true;
 
diff --git a/sql/sql_select.h b/sql/sql_select.h
index 89ee63e87b0..dfa96f1c81c 100644
--- a/sql/sql_select.h
+++ b/sql/sql_select.h
@@ -1290,7 +1290,8 @@ public:
   enum join_optimization_state { NOT_OPTIMIZED=0,
                                  OPTIMIZATION_IN_PROGRESS=1,
                                  OPTIMIZATION_DONE=2};
-  bool optimized; ///< flag to avoid double optimization in EXPLAIN
+  // state of JOIN optimization
+  enum join_optimization_state optimization_state;
   bool initialized; ///< flag to avoid double init_execution calls
 
   Explain_select *explain;
@@ -1378,7 +1379,7 @@ public:
     ref_pointer_array= items0= items1= items2= items3= 0;
     ref_pointer_array_size= 0;
     zero_result_cause= 0;
-    optimized= 0;
+    optimization_state= JOIN::NOT_OPTIMIZED;
     have_query_plan= QEP_NOT_PRESENT_YET;
     initialized= 0;
     cleaned= 0;