MDEV-23320 Hex hybrid constants 0xHHHH work badly in rounding functions

- Type_handler_hex_hybrid did not override
  Type_handler_string_result::Item_func_round_fix_length_and_dec(),
  so the result type of ROUND(0xFFFFFFFFFFFFFFFF) was erroneously
  calculated ad DOUBLE with a wrong length.
  Overriding Item_func_round_fix_length_and_dec(), to calculated
  the result type as INT/BIGINT.

  Also, fixing Item_func_round::fix_arg_int() to use
  args[0]->decimal_precision() instead of args[0]->max_length
  when calculating this->max_length, to get a correct result
  for hex hybrids.

- Type_handler_hex_hybrid::Item_func_int_val_fix_length_and_dec()
  called item->fix_length_and_dec_int_or_decimal(), which did not
  produce a correct result data type for hex hybrid.
  Implementing a dedicated code instead, to return INT UNSIGNED or
  BIGINT UNSIGNED depending in the number of digits in the arguments.

1 files changed, 3 insertions, 1 deletions

diff --git a/sql/item_func.cc b/sql/item_func.cc
index 424437ead07..cf01934806c 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -2228,6 +2228,7 @@ bool Item_func_int_val::fix_length_and_dec()
 longlong Item_func_ceiling::int_op()
 {
   switch (args[0]->result_type()) {
+  case STRING_RESULT: // hex hybrid
   case INT_RESULT:
     return val_int_from_item(args[0]);
   case DECIMAL_RESULT:
@@ -2286,6 +2287,7 @@ bool Item_func_ceiling::time_op(THD *thd, MYSQL_TIME *to)
 longlong Item_func_floor::int_op()
 {
   switch (args[0]->result_type()) {
+  case STRING_RESULT: // hex hybrid
   case INT_RESULT:
     return val_int_from_item(args[0]);
   case DECIMAL_RESULT:
@@ -2452,7 +2454,7 @@ void Item_func_round::fix_arg_int()
     {
       // Length can increase in some cases: ROUND(9,-1) -> 10
       int length_can_increase= MY_TEST(!truncate && val1.neg());
-      max_length= args[0]->max_length + length_can_increase;
+      max_length= args[0]->decimal_precision() + length_can_increase;
       // Here we can keep INT_RESULT
       unsigned_flag= args[0]->unsigned_flag;
       decimals= 0;