MDEV-7911: crash in Item_cond::eval_not_null_tables

convert_subq_to_sj() must check the results of in_equality->fix_fields()
call. It can fail in a meaningful way when e.g. we're trying to compare
columns with incompatible collations.

4 files changed, 56 insertions, 9 deletions

diff --git a/mysql-test/r/subselect_sj.result b/mysql-test/r/subselect_sj.result
index 98905f10018..21acd463b59 100644
--- a/mysql-test/r/subselect_sj.result
+++ b/mysql-test/r/subselect_sj.result
@@ -2986,4 +2986,13 @@ pk1	i1	i2	c2	pk3	i3	c3
 SET join_cache_level=@tmp_mdev5059;
 set optimizer_switch=@tmp_os_mdev5059;
 DROP TABLE t1,t2,t3,t4;
+#
+# MDEV-7911: crash in Item_cond::eval_not_null_tables
+#
+create table t1(a int);
+insert into t1 values(1),(2),(3),(null);
+explain
+select 1 from t1 where _cp932 "1" in (select '1' from t1);
+ERROR HY000: Illegal mix of collations (cp932_japanese_ci,COERCIBLE) and (latin1_swedish_ci,COERCIBLE) for operation '='
+drop table t1;
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/mysql-test/r/subselect_sj_jcl6.result b/mysql-test/r/subselect_sj_jcl6.result
index f4d874aa8f2..d9aa187e717 100644
--- a/mysql-test/r/subselect_sj_jcl6.result
+++ b/mysql-test/r/subselect_sj_jcl6.result
@@ -3000,6 +3000,15 @@ pk1	i1	i2	c2	pk3	i3	c3
 SET join_cache_level=@tmp_mdev5059;
 set optimizer_switch=@tmp_os_mdev5059;
 DROP TABLE t1,t2,t3,t4;
+#
+# MDEV-7911: crash in Item_cond::eval_not_null_tables
+#
+create table t1(a int);
+insert into t1 values(1),(2),(3),(null);
+explain
+select 1 from t1 where _cp932 "1" in (select '1' from t1);
+ERROR HY000: Illegal mix of collations (cp932_japanese_ci,COERCIBLE) and (latin1_swedish_ci,COERCIBLE) for operation '='
+drop table t1;
 set optimizer_switch=@subselect_sj_tmp;
 #
 # BUG#49129: Wrong result with IN-subquery with join_cache_level=6 and firstmatch=off
diff --git a/mysql-test/t/subselect_sj.test b/mysql-test/t/subselect_sj.test
index 1b0076fd1e6..9d7ea17bbe2 100644
--- a/mysql-test/t/subselect_sj.test
+++ b/mysql-test/t/subselect_sj.test
@@ -2689,5 +2689,16 @@ SET join_cache_level=@tmp_mdev5059;
 set optimizer_switch=@tmp_os_mdev5059;
 DROP TABLE t1,t2,t3,t4;
 
+--echo #
+--echo # MDEV-7911: crash in Item_cond::eval_not_null_tables
+--echo #
+
+create table t1(a int);
+insert into t1 values(1),(2),(3),(null);
+--error ER_CANT_AGGREGATE_2COLLATIONS
+explain
+select 1 from t1 where _cp932 "1" in (select '1' from t1);
+drop table t1;
+
 # The following command must be the last one the file 
 set optimizer_switch=@subselect_sj_tmp;
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index ca880e8fdaa..ebf640c2987 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -1610,9 +1610,20 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
       sj_nest->sj_on_expr= and_items(sj_nest->sj_on_expr, item_eq);
     }
   }
-  /* Fix the created equality and AND */
-  if (!sj_nest->sj_on_expr->fixed)
-    sj_nest->sj_on_expr->fix_fields(parent_join->thd, &sj_nest->sj_on_expr);
+  /*
+    Fix the created equality and AND
+
+    Note that fix_fields() can actually fail in a meaningful way here. One
+    example is when the IN-equality is not valid, because it compares columns
+    with incompatible collations. (One can argue it would be more appropriate
+    to check for this at name resolution stage, but as a legacy of IN->EXISTS
+    we have in here).
+  */
+  if (!sj_nest->sj_on_expr->fixed &&
+      sj_nest->sj_on_expr->fix_fields(parent_join->thd, &sj_nest->sj_on_expr))
+  {
+    DBUG_RETURN(TRUE);
+  }
 
   /*
     Walk through sj nest's WHERE and ON expressions and call
@@ -1631,12 +1642,15 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
   /* Inject sj_on_expr into the parent's WHERE or ON */
   if (emb_tbl_nest)
   {
-    emb_tbl_nest->on_expr= and_items(emb_tbl_nest->on_expr, 
+    emb_tbl_nest->on_expr= and_items(emb_tbl_nest->on_expr,
                                      sj_nest->sj_on_expr);
     emb_tbl_nest->on_expr->top_level_item();
-    if (!emb_tbl_nest->on_expr->fixed)
-      emb_tbl_nest->on_expr->fix_fields(parent_join->thd,
-                                        &emb_tbl_nest->on_expr);
+    if (!emb_tbl_nest->on_expr->fixed &&
+         emb_tbl_nest->on_expr->fix_fields(parent_join->thd,
+                                           &emb_tbl_nest->on_expr))
+    {
+      DBUG_RETURN(TRUE);
+    }
   }
   else
   {
@@ -1649,8 +1663,12 @@ static bool convert_subq_to_sj(JOIN *parent_join, Item_in_subselect *subq_pred)
     */
     save_lex= thd->lex->current_select;
     thd->lex->current_select=parent_join->select_lex;
-    if (!parent_join->conds->fixed)
-      parent_join->conds->fix_fields(parent_join->thd, &parent_join->conds);
+    if (!parent_join->conds->fixed &&
+         parent_join->conds->fix_fields(parent_join->thd,
+                                        &parent_join->conds))
+    {
+      DBUG_RETURN(1);
+    }
     thd->lex->current_select=save_lex;
     parent_join->select_lex->where= parent_join->conds;
   }