BUG#878753: Assertion '0' failed in replace_where_subcondition with derived_merge

- Remove the assert in replace_where_subcondition (the patch has explanation why)

4 files changed, 82 insertions, 4 deletions

diff --git a/mysql-test/r/subselect_sj.result b/mysql-test/r/subselect_sj.result
index 351515f8d40..99d8e9d2d77 100644
--- a/mysql-test/r/subselect_sj.result
+++ b/mysql-test/r/subselect_sj.result
@@ -1994,4 +1994,28 @@ a
 20
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@tmp_877288;
+#
+# BUG#878753: Assertion '0' failed in replace_where_subcondition with derived_merge
+#
+set @tmp878753= @@optimizer_switch;
+set optimizer_switch= 'semijoin=on,derived_merge=on';
+CREATE TABLE t1 (b int(11)) ;
+CREATE TABLE t2 (c int, b int, d varchar(52) NOT NULL) ;
+CREATE TABLE t3 (b int(11)) ;
+PREPARE st1 FROM '
+        SELECT * FROM t1
+        JOIN (
+                SELECT t2.* FROM t2
+                WHERE t2.d <> "a"
+                AND t2.c IN (
+                        SELECT t3.b
+                        FROM t3
+                )
+        ) AS alias2
+        ON ( alias2.b = t1.b );
+';
+EXECUTE st1;
+b	c	b	d
+DROP TABLE t1,t2,t3;
+set optimizer_switch=@tmp878753;
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/mysql-test/r/subselect_sj_jcl6.result b/mysql-test/r/subselect_sj_jcl6.result
index 0931a4d743a..2d1da2ea953 100644
--- a/mysql-test/r/subselect_sj_jcl6.result
+++ b/mysql-test/r/subselect_sj_jcl6.result
@@ -2005,6 +2005,30 @@ a
 20
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@tmp_877288;
+#
+# BUG#878753: Assertion '0' failed in replace_where_subcondition with derived_merge
+#
+set @tmp878753= @@optimizer_switch;
+set optimizer_switch= 'semijoin=on,derived_merge=on';
+CREATE TABLE t1 (b int(11)) ;
+CREATE TABLE t2 (c int, b int, d varchar(52) NOT NULL) ;
+CREATE TABLE t3 (b int(11)) ;
+PREPARE st1 FROM '
+        SELECT * FROM t1
+        JOIN (
+                SELECT t2.* FROM t2
+                WHERE t2.d <> "a"
+                AND t2.c IN (
+                        SELECT t3.b
+                        FROM t3
+                )
+        ) AS alias2
+        ON ( alias2.b = t1.b );
+';
+EXECUTE st1;
+b	c	b	d
+DROP TABLE t1,t2,t3;
+set optimizer_switch=@tmp878753;
 set optimizer_switch=@subselect_sj_tmp;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off
diff --git a/mysql-test/t/subselect_sj.test b/mysql-test/t/subselect_sj.test
index 385bf35753a..9d618a8590e 100644
--- a/mysql-test/t/subselect_sj.test
+++ b/mysql-test/t/subselect_sj.test
@@ -1827,5 +1827,30 @@ SELECT * FROM t1 WHERE (a) IN (SELECT a FROM t2 JOIN t3 ON b = a);
 DROP TABLE t1,t2,t3;
 set optimizer_switch=@tmp_877288;
 
+--echo #
+--echo # BUG#878753: Assertion '0' failed in replace_where_subcondition with derived_merge
+--echo #
+set @tmp878753= @@optimizer_switch;
+set optimizer_switch= 'semijoin=on,derived_merge=on';
+CREATE TABLE t1 (b int(11)) ;
+CREATE TABLE t2 (c int, b int, d varchar(52) NOT NULL) ;
+CREATE TABLE t3 (b int(11)) ;
+
+PREPARE st1 FROM '
+        SELECT * FROM t1
+        JOIN (
+                SELECT t2.* FROM t2
+                WHERE t2.d <> "a"
+                AND t2.c IN (
+                        SELECT t3.b
+                        FROM t3
+                )
+        ) AS alias2
+        ON ( alias2.b = t1.b );
+';
+EXECUTE st1;
+DROP TABLE t1,t2,t3;
+set optimizer_switch=@tmp878753;
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index 416dcadc67f..aa0a8152bbb 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -1026,7 +1026,6 @@ static bool replace_where_subcondition(JOIN *join, Item **expr,
                                        Item *old_cond, Item *new_cond,
                                        bool do_fix_fields)
 {
-  //Item **expr= (emb_nest == (TABLE_LIST*)1)? &join->conds : &emb_nest->on_expr;
   if (*expr == old_cond)
   {
     *expr= new_cond;
@@ -1050,9 +1049,15 @@ static bool replace_where_subcondition(JOIN *join, Item **expr,
       }
     }
   }
-  // If we came here it means there were an error during prerequisites check.
-  DBUG_ASSERT(0);
-  return TRUE;
+  /* 
+    We can come to here when 
+     - we're doing replace operations on both on_expr and prep_on_expr
+     - on_expr is the same as prep_on_expr, or they share a sub-tree 
+       (so, when we do replace in on_expr, we replace in prep_on_expr, too,
+        and when we try doing a replace in prep_on_expr, the item we wanted 
+        to replace there has already been replaced)
+  */
+  return FALSE;
 }
 
 static int subq_sj_candidate_cmp(Item_in_subselect* el1, Item_in_subselect* el2,