Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions

- Honor unsigned_flag in the corresponding functions
- Use compare_int_signed_unsigned()/compare_int_unsigned_signed() instead of explicit comparison in GREATEST() and LEAST()


mysql-test/r/case.result:
  Added test case for bug #20924
mysql-test/r/func_if.result:
  Added test case for bug #20924
mysql-test/r/func_test.result:
  Added test case for bug #20924
mysql-test/r/user_var.result:
  Added test case for bug #20924
mysql-test/t/case.test:
  Added test case for bug #20924
mysql-test/t/func_if.test:
  Added test case for bug #20924
mysql-test/t/func_test.test:
  Added test case for bug #20924
mysql-test/t/user_var.test:
  Added test case for bug #20924
sql/item_cmpfunc.cc:
  Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions
  
  - Moved some code out of Arg_comparator to external functions to be reused in Item_func_min_max
  - Fixed IFNULL(), IF(), CASE() and COALESCE()
sql/item_cmpfunc.h:
  Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions
  
  - Moved some code out of Arg_comparator to external functions to be reused in Item_func_min_max
sql/item_func.cc:
  Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions
  
  Fixed LEAST(), GREATEST() and "SET @a=..." parts
sql/item_func.h:
  Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions
  
  Fixed "SET @a=..." part
sql/sql_class.h:
  Bug #20924: CAST(expr as UNSIGNED) returns SIGNED value when used in various functions
  
  Fixed "SET @a=..." part

5 files changed, 55 insertions, 20 deletions

diff --git a/sql/item_cmpfunc.cc b/sql/item_cmpfunc.cc
index f14efc7187b..cf8d0c39d58 100644
--- a/sql/item_cmpfunc.cc
+++ b/sql/item_cmpfunc.cc
@@ -619,11 +619,7 @@ int Arg_comparator::compare_int_signed_unsigned()
     if (!(*b)->null_value)
     {
       owner->null_value= 0;
-      if (sval1 < 0 || (ulonglong)sval1 < uval2)
-        return -1;
-      if ((ulonglong)sval1 == uval2)
-        return 0;
-      return 1;
+      return ::compare_int_signed_unsigned(sval1, uval2);
     }
   }
   owner->null_value= 1;
@@ -644,13 +640,7 @@ int Arg_comparator::compare_int_unsigned_signed()
     if (!(*b)->null_value)
     {
       owner->null_value= 0;
-      if (sval2 < 0)
-        return 1;
-      if (uval1 < (ulonglong)sval2)
-        return -1;
-      if (uval1 == (ulonglong)sval2)
-        return 0;
-      return 1;
+      return ::compare_int_unsigned_signed(uval1, sval2);
     }
   }
   owner->null_value= 1;
@@ -1162,11 +1152,13 @@ Item_func_ifnull::val_int()
   if (!args[0]->null_value)
   {
     null_value=0;
+    unsigned_flag= args[0]->unsigned_flag;
     return value;
   }
   value=args[1]->val_int();
   if ((null_value=args[1]->null_value))
     return 0;
+  unsigned_flag= args[1]->unsigned_flag;
   return value;
 }
 
@@ -1286,6 +1278,7 @@ Item_func_if::val_int()
   Item *arg= args[0]->val_int() ? args[1] : args[2];
   longlong value=arg->val_int();
   null_value=arg->null_value;
+  unsigned_flag= arg->unsigned_flag;
   return value;
 }
 
@@ -1492,6 +1485,7 @@ longlong Item_func_case::val_int()
   }
   res=item->val_int();
   null_value=item->null_value;
+  unsigned_flag= item->unsigned_flag;
   return res;
 }
 
@@ -1623,7 +1617,10 @@ longlong Item_func_coalesce::val_int()
   {
     longlong res=args[i]->val_int();
     if (!args[i]->null_value)
+    {
+      unsigned_flag= args[i]->unsigned_flag;  
       return res;
+    }
   }
   null_value=1;
   return 0;
diff --git a/sql/item_cmpfunc.h b/sql/item_cmpfunc.h
index 73abe208d9e..513260205c2 100644
--- a/sql/item_cmpfunc.h
+++ b/sql/item_cmpfunc.h
@@ -1077,3 +1077,17 @@ inline Item *and_conds(Item *a, Item *b)
 }
 
 Item *and_expressions(Item *a, Item *b, Item **org_item);
+
+inline int compare_int_signed_unsigned(longlong sval, ulonglong uval)
+{
+  if (sval < 0 || (ulonglong)sval < uval)
+    return -1;
+  if ((ulonglong)sval == uval)
+    return 0;
+  return 1;
+}
+
+inline int compare_int_unsigned_signed(ulonglong uval, longlong sval)
+{
+  return -compare_int_signed_unsigned(sval, uval);
+}
diff --git a/sql/item_func.cc b/sql/item_func.cc
index 91ccef6511f..85865ad6fb6 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1235,19 +1235,35 @@ longlong Item_func_min_max::val_int()
 {
   DBUG_ASSERT(fixed == 1);
   longlong value=0;
+  my_bool arg_unsigned_flag;
+  my_bool cmp;
   null_value=1;
   for (uint i=0; i < arg_count ; i++)
   {
+    longlong tmp= args[i]->val_int();
+    arg_unsigned_flag= args[i]->unsigned_flag;
     if (null_value)
     {
-      value=args[i]->val_int();
+      value= tmp;
       null_value=args[i]->null_value;
+      unsigned_flag= arg_unsigned_flag;
     }
     else
     {
-      longlong tmp=args[i]->val_int();
-      if (!args[i]->null_value && (tmp < value ? cmp_sign : -cmp_sign) > 0)
-	value=tmp;
+      if (args[i]->null_value)
+        continue;
+      if (unsigned_flag && arg_unsigned_flag ||
+          (!unsigned_flag && !arg_unsigned_flag))
+        cmp= tmp < value;
+      else if (unsigned_flag)
+        cmp= compare_int_signed_unsigned(tmp, value) < 0;
+      else
+        cmp= compare_int_unsigned_signed(tmp, value) < 0;
+      if ((cmp ? cmp_sign : -cmp_sign) > 0)
+      {
+        value= tmp;
+        unsigned_flag= arg_unsigned_flag;
+      }
     }
   }
   return value;
@@ -2313,6 +2329,7 @@ static user_var_entry *get_variable(HASH *hash, LEX_STRING &name,
     entry->length=0;
     entry->update_query_id=0;
     entry->collation.set(NULL, DERIVATION_IMPLICIT);
+    entry->unsigned_flag= 0;
     /*
       If we are here, we were called from a SET or a query which sets a
       variable. Imagine it is this:
@@ -2390,7 +2407,7 @@ Item_func_set_user_var::fix_length_and_dec()
 bool Item_func_set_user_var::update_hash(void *ptr, uint length,
 					 Item_result type,
 					 CHARSET_INFO *cs,
-					 Derivation dv)
+					 Derivation dv, bool unsigned_arg)
 {
   if ((null_value=args[0]->null_value))
   {
@@ -2437,6 +2454,7 @@ bool Item_func_set_user_var::update_hash(void *ptr, uint length,
     entry->length= length;
     entry->type=type;
     entry->collation.set(cs, dv);
+    entry->unsigned_flag= unsigned_arg;
   }
   return 0;
 
@@ -2507,7 +2525,10 @@ String *user_var_entry::val_str(my_bool *null_value, String *str,
     str->set(*(double*) value, decimals, &my_charset_bin);
     break;
   case INT_RESULT:
-    str->set(*(longlong*) value, &my_charset_bin);
+      if (!unsigned_flag)
+        str->set(*(longlong*) value, &my_charset_bin);
+      else
+        str->set(*(ulonglong*) value, &my_charset_bin);
     break;
   case STRING_RESULT:
     if (str->copy(value, length, collation.collation))
@@ -2548,6 +2569,7 @@ Item_func_set_user_var::check()
   case INT_RESULT:
   {
     save_result.vint= args[0]->val_int();
+    unsigned_flag= args[0]->unsigned_flag;
     break;
   }
   case STRING_RESULT:
@@ -2598,7 +2620,8 @@ Item_func_set_user_var::update()
   case INT_RESULT:
   {
     res= update_hash((void*) &save_result.vint, sizeof(save_result.vint),
-		     INT_RESULT, &my_charset_bin, DERIVATION_IMPLICIT);
+         INT_RESULT, &my_charset_bin, DERIVATION_IMPLICIT,
+         unsigned_flag);
     break;
   }
   case STRING_RESULT:
diff --git a/sql/item_func.h b/sql/item_func.h
index f4a1258a02c..d9e1396fde6 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -962,7 +962,7 @@ public:
   longlong val_int();
   String *val_str(String *str);
   bool update_hash(void *ptr, uint length, enum Item_result type, 
-  		   CHARSET_INFO *cs, Derivation dv);
+  		   CHARSET_INFO *cs, Derivation dv, bool unsigned_arg= 0);
   bool check();
   bool update();
   enum Item_result result_type () const { return cached_result_type; }
diff --git a/sql/sql_class.h b/sql/sql_class.h
index f1cf9c7b3e2..2db0077c0b4 100644
--- a/sql/sql_class.h
+++ b/sql/sql_class.h
@@ -1460,6 +1460,7 @@ class user_var_entry
   char *value;
   ulong length, update_query_id, used_query_id;
   Item_result type;
+  bool unsigned_flag;
 
   double val(my_bool *null_value);
   longlong val_int(my_bool *null_value);