MDEV-7565: Server crash with Signal 6 (part 2)

Preparation of subselect moved earlier (before checks which needs it prepared).

7 files changed, 78 insertions, 6 deletions

diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result
index 4afde93925d..8eac09a2a17 100644
--- a/mysql-test/r/subselect.result
+++ b/mysql-test/r/subselect.result
@@ -7015,3 +7015,15 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result
index 9df216da3cb..aed0e6a7e30 100644
--- a/mysql-test/r/subselect_no_mat.result
+++ b/mysql-test/r/subselect_no_mat.result
@@ -7012,6 +7012,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%materialization=on%';
 @@optimizer_switch like '%materialization=on%'
diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result
index 7b93f277274..07c95851828 100644
--- a/mysql-test/r/subselect_no_opts.result
+++ b/mysql-test/r/subselect_no_opts.result
@@ -7010,4 +7010,16 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;
diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result
index b6d3a89ea7a..9a49fedb093 100644
--- a/mysql-test/r/subselect_no_scache.result
+++ b/mysql-test/r/subselect_no_scache.result
@@ -7021,6 +7021,18 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set optimizer_switch=default;
 select @@optimizer_switch like '%subquery_cache=on%';
 @@optimizer_switch like '%subquery_cache=on%'
diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result
index d51d211e71d..658888d45f5 100644
--- a/mysql-test/r/subselect_no_semijoin.result
+++ b/mysql-test/r/subselect_no_semijoin.result
@@ -7010,5 +7010,17 @@ select exists(select 1 from t1 group by `c` in (select `c` from t1));
 exists(select 1 from t1 group by `c` in (select `c` from t1))
 0
 drop table t1;
+#
+# MDEV-7565: Server crash with Signal 6 (part 2)
+#
+Select 
+(Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+ControlRev
+NULL
 set @optimizer_switch_for_subselect_test=null;
 set @join_cache_level_for_subselect_test=NULL;
diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test
index d1c3774947a..dd9603ca5d4 100644
--- a/mysql-test/t/subselect.test
+++ b/mysql-test/t/subselect.test
@@ -5890,3 +5890,15 @@ DROP TABLE t1,t2;
 create table t1 (c int);
 select exists(select 1 from t1 group by `c` in (select `c` from t1));
 drop table t1;
+
+--echo #
+--echo # MDEV-7565: Server crash with Signal 6 (part 2)
+--echo #
+Select 
+  (Select Sum(`TestCase`.Revenue) From mysql.slow_log E           
+    Where TestCase.TemplateID not in (Select 1 from mysql.slow_log where 2=2)
+  ) As `ControlRev`
+From 
+(Select  3 as Revenue, 4 as TemplateID) As `TestCase` 
+Group By  TestCase.Revenue, TestCase.TemplateID;
+
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index ebf640c2987..a8e1ff97b82 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -703,6 +703,12 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
       if (!optimizer_flag(thd, OPTIMIZER_SWITCH_IN_TO_EXISTS) &&
           !optimizer_flag(thd, OPTIMIZER_SWITCH_MATERIALIZATION))
         my_error(ER_ILLEGAL_SUBQUERY_OPTIMIZER_SWITCHES, MYF(0));
+      /*
+        Transform each subquery predicate according to its overloaded
+        transformer.
+      */
+      if (subselect->select_transformer(join))
+        DBUG_RETURN(-1);
 
       /*
         If the subquery predicate is IN/=ANY, analyse and set all possible
@@ -754,12 +760,6 @@ int check_and_do_in_subquery_rewrites(JOIN *join)
         allany_subs->add_strategy(strategy);
       }
 
-      /*
-        Transform each subquery predicate according to its overloaded
-        transformer.
-      */
-      if (subselect->select_transformer(join))
-        DBUG_RETURN(-1);
     }
   }
   DBUG_RETURN(0);