Fixed bug #28076: inconsistent binary/varbinary comparison.

After adding an index the <VARBINARY> IN (SELECT <BINARY> ...)
clause returned a wrong result: the VARBINARY value was illegally padded
with zero bytes to the length of the BINARY column for the index search.
(<VARBINARY>, ...) IN (SELECT <BINARY>, ... ) clauses are affected too.


sql/item.cc:
  Fixed bug #28076.
  The Item_cache_str::save_in_field method has been overloaded
  to check cached values for an illegal padding before the saving
  into a field.
sql/item.h:
  Fixed bug #28076.
  The Item_cache_str::is_varbinary flag has been added and the
  Item_cache_str::save_in_field method has been overloaded to prevent
  cached values from an illegal padding when saving in fields.
  The signature of the Item_cache::get_cache method has been
  changed to accept pointers to Item instead of Item_result
  values.
sql/item_cmpfunc.cc:
  Fixed bug #28076.
  The Item_in_optimizer::fix_left method has been modified to
  to call Item_cache::get_cache in a new manner.
sql/item_subselect.cc:
  Fixed bug #28076.
  The subselect_indexsubquery_engine::exec method has been
  modified to take into account field conversion errors
  (copy&paste from subselect_uniquesubquery_engine::exec).
sql/sp_rcontext.cc:
  Fixed bug #28076.
  The sp_rcontext::create_case_expr_holder method has been
  modified to call Item_cache::get_cache in a new manner.
sql/sp_rcontext.h:
  Fixed bug #28076.
  The sp_rcontext::create_case_expr_holder method signature
  has been modified to pass Item pointers to the
  Item_cache::get_cache method.
sql/sql_class.cc:
  Fixed bug #28076.
  The select_max_min_finder_subselect::send_data method has been
  modified to call Item_cache::get_cache in a new manner.
mysql-test/t/subselect.test:
  Added test case for bug #28076.
mysql-test/r/subselect.result:
  Added test case for bug #28076.

1 files changed, 12 insertions, 4 deletions

diff --git a/sql/item.cc b/sql/item.cc
index dc3b1fc6b79..af3b1566632 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -6259,9 +6259,9 @@ bool field_is_equal_to_item(Field *field,Item *item)
   return result == field->val_real();
 }
 
-Item_cache* Item_cache::get_cache(Item_result type)
+Item_cache* Item_cache::get_cache(const Item *item)
 {
-  switch (type) {
+  switch (item->result_type()) {
   case INT_RESULT:
     return new Item_cache_int();
   case REAL_RESULT:
@@ -6269,7 +6269,7 @@ Item_cache* Item_cache::get_cache(Item_result type)
   case DECIMAL_RESULT:
     return new Item_cache_decimal();
   case STRING_RESULT:
-    return new Item_cache_str();
+    return new Item_cache_str(item);
   case ROW_RESULT:
     return new Item_cache_row();
   default:
@@ -6447,6 +6447,14 @@ my_decimal *Item_cache_str::val_decimal(my_decimal *decimal_val)
 }
 
 
+int Item_cache_str::save_in_field(Field *field, bool no_conversions)
+{
+  int res= Item_cache::save_in_field(field, no_conversions);
+  return (is_varbinary && field->type() == MYSQL_TYPE_STRING &&
+          value->length() < field->field_length) ? 1 : res;
+}
+
+
 bool Item_cache_row::allocate(uint num)
 {
   item_count= num;
@@ -6465,7 +6473,7 @@ bool Item_cache_row::setup(Item * item)
   {
     Item *el= item->element_index(i);
     Item_cache *tmp;
-    if (!(tmp= values[i]= Item_cache::get_cache(el->result_type())))
+    if (!(tmp= values[i]= Item_cache::get_cache(el)))
       return 1;
     tmp->setup(el);
   }