MDEV-20900: IN predicate to IN subquery conversion causes performance regression

Disable the IN predicate to IN subquery conversion when the types on the left and
right hand side of the IN predicate are not of comparable type.

3 files changed, 62 insertions, 2 deletions

diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index 24a9991640a..29c32be1c80 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -2418,6 +2418,7 @@ public:
   bool to_be_transformed_into_in_subq(THD *thd);
   bool create_value_list_for_tvc(THD *thd, List< List<Item> > *values);
   Item *in_predicate_to_in_subs_transformer(THD *thd, uchar *arg);
+  uint32 max_length_of_left_expr();
 };
 
 class cmp_item_row :public cmp_item
diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc
index a4ee27950be..50f7efd6d65 100644
--- a/sql/opt_subselect.cc
+++ b/sql/opt_subselect.cc
@@ -829,7 +829,12 @@ bool subquery_types_allow_materialization(Item_in_subselect *in_subs)
 
   in_subs->types_allow_materialization= FALSE;  // Assign default values
   in_subs->sjm_scan_allowed= FALSE;
-  
+
+  /*
+    The checks here must be kept in sync with the one in
+    Item_func_in::in_predicate_to_in_subs_transformer().
+  */
+
   bool all_are_fields= TRUE;
   uint32 total_key_length = 0;
   for (uint i= 0; i < elements; i++)
diff --git a/sql/sql_tvc.cc b/sql/sql_tvc.cc
index 816c6fe1089..b4538248e07 100644
--- a/sql/sql_tvc.cc
+++ b/sql/sql_tvc.cc
@@ -796,6 +796,38 @@ bool Item_subselect::wrap_tvc_into_select(THD *thd, st_select_lex *tvc_sl)
 }
 
 
+/*
+  @brief
+   Check whether the items are of comparable type or not
+
+  @details
+    This check are done because materialization is not performed
+    if the left expr and right expr are of the same types.
+    @see subquery_types_allow_materialization()
+
+  @retval
+   0 comparable
+   1 not comparable
+*/
+
+static bool cmp_row_types(Item* item1, Item* item2)
+{
+  uint n= item1->cols();
+  if (item2->check_cols(n))
+    return true;
+
+  for (uint i=0; i < n; i++)
+  {
+    Item *inner= item1->element_index(i);
+    Item *outer= item2->element_index(i);
+    if (!inner->type_handler()->subquery_type_allows_materialization(inner,
+                                                                     outer))
+      return true;
+  }
+  return false;
+}
+
+
 /**
   @brief
     Transform IN predicate into IN subquery
@@ -840,10 +872,22 @@ Item *Item_func_in::in_predicate_to_in_subs_transformer(THD *thd,
   /* SELECT_LEX object where the transformation is performed */
   SELECT_LEX *parent_select= lex->current_select;
   uint8 save_derived_tables= lex->derived_tables;
+
+  /*
+    Make sure that create_tmp_table will not fail due to too long keys.
+    Here the strategy would mainly use materialization, so we need to make
+    sure that the materialized table can be created.
+
+    The checks here are the same as in subquery_type_allows_materialization()
+  */
+  uint32 length= max_length_of_left_expr();
+  if (!length  || length > tmp_table_max_key_length() ||
+      args[0]->cols() > tmp_table_max_key_parts())
+    return this;
   
   for (uint i=1; i < arg_count; i++)
   {
-    if (!args[i]->const_item())
+    if (!args[i]->const_item() || cmp_row_types(args[0], args[i]))
       return this;
   }
 
@@ -948,6 +992,16 @@ err:
 }
 
 
+uint32 Item_func_in::max_length_of_left_expr()
+{
+  uint n= args[0]->cols();
+  uint32 length= 0;
+  for (uint i=0; i < n; i++)
+    length+= args[0]->element_index(i)->max_length;
+  return length;
+}
+
+
 /**
   @brief
     Check if this IN-predicate can be transformed in IN-subquery