MDEV-18339: ASAN heap-buffer-overflow in Item_exists_subselect::is_top_level_itembb-10.4-MDEV-18339

Right argument of Item_in_optimizer can not be cast to Item_in_subselect in invisible mode.

3 files changed, 77 insertions, 4 deletions

diff --git a/mysql-test/main/subselect_innodb.result b/mysql-test/main/subselect_innodb.result
index 0eb40c9be00..64e67c1dfc1 100644
--- a/mysql-test/main/subselect_innodb.result
+++ b/mysql-test/main/subselect_innodb.result
@@ -616,3 +616,38 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
 Warnings:
 Note	1003	select `test`.`t1`.`f1` AS `f1`,`test`.`t2`.`f2` AS `f2`,`test`.`t3`.`f3` AS `f3` from `test`.`t1` join `test`.`t2` semi join (`test`.`t4`) join `test`.`t3` where `test`.`t4`.`f4` = 1 and `test`.`t1`.`f1` >= `test`.`t2`.`f2`
 DROP TABLE t1,t2,t3,t4;
+#
+# MDEV-18339: ASAN heap-buffer-overflow in
+# Item_exists_subselect::is_top_level_item
+#
+CREATE TABLE t1 ( pk int PRIMARY KEY , iiiiiiiiiiiii int , col_int1111 int, col_date_nokey date , col_time_key time, col_time_nokey time , col_datetime_key time, col_datetime_nokey time , ccccccccccccccc varchar(1), vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+CREATE TABLE t2 ( iiiiiiiiiiiii int , vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+CREATE TABLE t3 ( pk int PRIMARY KEY) engine=innodb;
+CREATE TABLE t4 ( iiiiiiiiiiiii int , vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+select * from
+(select distinct
+(select count(t111111111.`ccccccccccccccc`) from t1 as t111111111
+where (exists(select distinct t22222222222.`iiiiiiiiiiiii` from t2 as t22222222222 where t22222222222.`vvvvvvvvvvvvvvvvv` < t111111111.`vvvvvvvvvvvvvvvvv`)
+or t111111111.`ccccccccccccccc` != t111111111.`vvvvvvvvvvvvvvvvv`)
+) as field1
+from
+(select t1_______2.*
+from (t1 as t1_______1 join t1 as t1_______2
+on (t1_______2.`vvvvvvvvvvvvvvvvv` = t1_______1.`ccccccccccccccc`
+             and t1_______1.`iiiiiiiiiiiii` !=
+(select sum(t44444444444.`iiiiiiiiiiiii`)
+from (t4 as t44444444444 join t3 as t33333333333
+on (t33333333333.`pk` = t44444444444.`iiiiiiiiiiiii`))
+where t44444444444.`vvvvvvvvvvvvvvvvv` > 'x')
+)
+)
+) as alias1
+straight_join
+t2 as alias2
+on (alias2.`iiiiiiiiiiiii` = alias1.`iiiiiiiiiiiii`)
+where ((select 9 from dual) is null)
+and alias1.`pk` in (32, 129, 87, 51, 58, 152, 241, 37, 55, 237, 166)
+group by field1 /*        111  
+111111111 */  ) as derived_aaaaa  /* comment11111111111111111111111111 */;
+field1
+# End of 10.4 tests
diff --git a/mysql-test/main/subselect_innodb.test b/mysql-test/main/subselect_innodb.test
index 544bcd994ed..90d3b07c1ad 100644
--- a/mysql-test/main/subselect_innodb.test
+++ b/mysql-test/main/subselect_innodb.test
@@ -611,3 +611,41 @@ FROM t1
 
 DROP TABLE t1,t2,t3,t4;
 
+--echo #
+--echo # MDEV-18339: ASAN heap-buffer-overflow in
+--echo # Item_exists_subselect::is_top_level_item
+--echo #
+
+CREATE TABLE t1 ( pk int PRIMARY KEY , iiiiiiiiiiiii int , col_int1111 int, col_date_nokey date , col_time_key time, col_time_nokey time , col_datetime_key time, col_datetime_nokey time , ccccccccccccccc varchar(1), vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+
+CREATE TABLE t2 ( iiiiiiiiiiiii int , vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+CREATE TABLE t3 ( pk int PRIMARY KEY) engine=innodb;
+CREATE TABLE t4 ( iiiiiiiiiiiii int , vvvvvvvvvvvvvvvvv varchar(1)) engine=innodb;
+
+select * from
+(select distinct
+    (select count(t111111111.`ccccccccccccccc`) from t1 as t111111111
+    where (exists(select distinct t22222222222.`iiiiiiiiiiiii` from t2 as t22222222222 where t22222222222.`vvvvvvvvvvvvvvvvv` < t111111111.`vvvvvvvvvvvvvvvvv`)
+          or t111111111.`ccccccccccccccc` != t111111111.`vvvvvvvvvvvvvvvvv`)
+    ) as field1
+from
+    (select t1_______2.*
+    from (t1 as t1_______1 join t1 as t1_______2
+            on (t1_______2.`vvvvvvvvvvvvvvvvv` = t1_______1.`ccccccccccccccc`
+             and t1_______1.`iiiiiiiiiiiii` !=
+                (select sum(t44444444444.`iiiiiiiiiiiii`)
+                from (t4 as t44444444444 join t3 as t33333333333
+                    on (t33333333333.`pk` = t44444444444.`iiiiiiiiiiiii`))
+                where t44444444444.`vvvvvvvvvvvvvvvvv` > 'x')
+                )
+        )
+    ) as alias1
+straight_join
+    t2 as alias2
+on (alias2.`iiiiiiiiiiiii` = alias1.`iiiiiiiiiiiii`)
+where ((select 9 from dual) is null)
+and alias1.`pk` in (32, 129, 87, 51, 58, 152, 241, 37, 55, 237, 166)
+group by field1 /*        111  
+111111111 */  ) as derived_aaaaa  /* comment11111111111111111111111111 */;
+
+--echo # End of 10.4 tests
diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index 55a06254917..ffb7b60e4de 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -1182,6 +1182,8 @@ longlong Item_func_truth::val_int()
 
 bool Item_in_optimizer::is_top_level_item()
 {
+  if (invisible_mode())
+    return FALSE;
   return ((Item_in_subselect *)args[1])->is_top_level_item();
 }
 
@@ -1237,8 +1239,7 @@ void Item_in_optimizer::print(String *str, enum_query_type query_type)
 
 void Item_in_optimizer::restore_first_argument()
 {
-  if (args[1]->type() == Item::SUBSELECT_ITEM &&
-      ((Item_subselect *)args[1])->is_in_predicate())
+  if (!invisible_mode())
   {
     args[0]= ((Item_in_subselect *)args[1])->left_expr;
   }
@@ -1255,8 +1256,7 @@ bool Item_in_optimizer::fix_left(THD *thd)
     it is args[0].
   */
   Item **ref0= args;
-  if (args[1]->type() == Item::SUBSELECT_ITEM &&
-      ((Item_subselect *)args[1])->is_in_predicate())
+  if (!invisible_mode())
   {
     /*
        left_expr->fix_fields() may cause left_expr to be substituted for