MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED

5 files changed, 48 insertions, 0 deletions

diff --git a/sql/item.cc b/sql/item.cc
index 243c10ddcbb..23f816fc8e6 100644
--- a/sql/item.cc
+++ b/sql/item.cc
@@ -6718,6 +6718,22 @@ void Item_hex_hybrid::print(String *str, enum_query_type query_type)
 }
 
 
+uint Item_hex_hybrid::decimal_precision() const
+{
+  switch (max_length) {// HEX                                 DEC
+  case 0:              // ----                                ---
+  case 1: return 3;    // 0xFF                                255
+  case 2: return 5;    // 0xFFFF                            65535
+  case 3: return 8;    // 0xFFFFFF                       16777215
+  case 4: return 10;   // 0xFFFFFFFF                   4294967295
+  case 5: return 13;   // 0xFFFFFFFFFF              1099511627775
+  case 6: return 15;   // 0xFFFFFFFFFFFF          281474976710655
+  case 7: return 17;   // 0xFFFFFFFFFFFFFF      72057594037927935
+  }
+  return 20;           // 0xFFFFFFFFFFFFFFFF 18446744073709551615
+}
+
+
 void Item_hex_string::print(String *str, enum_query_type query_type)
 {
   str->append("X'");
diff --git a/sql/item.h b/sql/item.h
index f9a90f719c9..b3ed31d5bb1 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -3810,6 +3810,7 @@ public:
   Item_hex_hybrid(THD *thd): Item_hex_constant(thd) {}
   Item_hex_hybrid(THD *thd, const char *str, uint str_length):
     Item_hex_constant(thd, str, str_length) {}
+  uint decimal_precision() const;
   double val_real()
   { 
     DBUG_ASSERT(fixed == 1); 
diff --git a/sql/item_func.h b/sql/item_func.h
index 860a4fdcc7d..c644c1abf1b 100644
--- a/sql/item_func.h
+++ b/sql/item_func.h
@@ -852,6 +852,19 @@ public:
     set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
     fix_char_length(char_length);
   }
+  void fix_length_and_dec_string()
+  {
+    /*
+      For strings, use decimal_int_part() instead of max_char_length().
+      This is important for Item_hex_hybrid:
+        SELECT CAST(0x1FFFFFFFF AS SIGNED);
+      Length is 5, decimal_int_part() is 13.
+    */
+    uint32 char_length= MY_MIN(args[0]->decimal_int_part(),
+                               MY_INT64_NUM_DECIMAL_DIGITS);
+    set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
+    fix_char_length(char_length);
+  }
   void fix_length_and_dec()
   {
     args[0]->type_handler()->Item_func_signed_fix_length_and_dec(this);
diff --git a/sql/sql_type.cc b/sql/sql_type.cc
index 5477cdb87b7..7f560e9ff26 100644
--- a/sql/sql_type.cc
+++ b/sql/sql_type.cc
@@ -4168,6 +4168,22 @@ bool Type_handler::
 }
 
 
+bool Type_handler_string_result::
+       Item_func_signed_fix_length_and_dec(Item_func_signed *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
+bool Type_handler_string_result::
+       Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
 bool Type_handler::
        Item_double_typecast_fix_length_and_dec(Item_double_typecast *item) const
 {
diff --git a/sql/sql_type.h b/sql/sql_type.h
index 9c04593c7f4..374f250f443 100644
--- a/sql/sql_type.h
+++ b/sql/sql_type.h
@@ -1615,6 +1615,8 @@ public:
   bool Item_sum_sum_fix_length_and_dec(Item_sum_sum *) const;
   bool Item_sum_avg_fix_length_and_dec(Item_sum_avg *) const;
   bool Item_sum_variance_fix_length_and_dec(Item_sum_variance *) const;
+  bool Item_func_signed_fix_length_and_dec(Item_func_signed *item) const;
+  bool Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const;
   longlong Item_val_int_signed_typecast(Item *item) const;
   longlong Item_val_int_unsigned_typecast(Item *item) const;
   String *Item_func_hex_val_str_ascii(Item_func_hex *item, String *str) const;