Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect

Missing check for overflow added to the Item_decimal_typecast::val_decimal


include/decimal.h:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  decimal_intg() declaration
mysql-test/r/cast.result:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  result fixed
mysql-test/r/type_newdecimal.result:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  test result
mysql-test/t/type_newdecimal.test:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  test case added
sql/item_func.cc:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  now we check for possible ovreflow in Item_decimal_typecast::val_decimal
sql/my_decimal.h:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  my_decimal_intg() implemented
strings/decimal.c:
  Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect
  decimal_intg() implemented

7 files changed, 97 insertions, 1 deletions

diff --git a/include/decimal.h b/include/decimal.h
index 56962009025..c5385b58658 100644
--- a/include/decimal.h
+++ b/include/decimal.h
@@ -47,6 +47,7 @@ int decimal_bin_size(int precision, int scale);
 int decimal_result_size(decimal_t *from1, decimal_t *from2, char op,
                         int param);
 
+int decimal_intg(decimal_t *from);
 int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to);
 int decimal_sub(decimal_t *from1, decimal_t *from2, decimal_t *to);
 int decimal_cmp(decimal_t *from1, decimal_t *from2);
diff --git a/mysql-test/r/cast.result b/mysql-test/r/cast.result
index 454a3766572..c0dab1c3293 100644
--- a/mysql-test/r/cast.result
+++ b/mysql-test/r/cast.result
@@ -378,7 +378,9 @@ create table t1(s1 time);
 insert into t1 values ('11:11:11');
 select cast(s1 as decimal(7,2)) from t1;
 cast(s1 as decimal(7,2))
-111111.00
+99999.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(s1 as decimal(7,2))' at row 1
 drop table t1;
 CREATE TABLE t1 (v varchar(10), tt tinytext, t text,
 mt mediumtext, lt longtext);
diff --git a/mysql-test/r/type_newdecimal.result b/mysql-test/r/type_newdecimal.result
index c103de81bd7..cbcab126439 100644
--- a/mysql-test/r/type_newdecimal.result
+++ b/mysql-test/r/type_newdecimal.result
@@ -1430,4 +1430,39 @@ select * from t1;
 a
 123456789012345678
 drop table t1;
+select cast(11.1234 as DECIMAL(3,2));
+cast(11.1234 as DECIMAL(3,2))
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(11.1234 as DECIMAL(3,2))' at row 1
+select * from (select cast(11.1234 as DECIMAL(3,2))) t;
+cast(11.1234 as DECIMAL(3,2))
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(11.1234 as DECIMAL(3,2))' at row 1
+select cast(a as DECIMAL(3,2))
+from (select 11.1233 as a
+UNION select 11.1234
+UNION select 12.1234
+) t;
+cast(a as DECIMAL(3,2))
+9.99
+9.99
+9.99
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+select cast(a as DECIMAL(3,2)), count(*)
+from (select 11.1233 as a
+UNION select 11.1234
+UNION select 12.1234
+) t group by 1;
+cast(a as DECIMAL(3,2))	count(*)
+9.99	3
+Warnings:
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
+Error	1264	Out of range value adjusted for column 'cast(a as DECIMAL(3,2))' at row 1
 End of 5.0 tests
diff --git a/mysql-test/t/type_newdecimal.test b/mysql-test/t/type_newdecimal.test
index 4c6098d2121..a7906be79d4 100644
--- a/mysql-test/t/type_newdecimal.test
+++ b/mysql-test/t/type_newdecimal.test
@@ -1130,4 +1130,23 @@ alter table t1 modify column a decimal(19);
 select * from t1;
 drop table t1;
 
+#
+# Bug #27957 cast as decimal does not check overflow, also inconsistent with group, subselect 
+#
+
+select cast(11.1234 as DECIMAL(3,2));
+select * from (select cast(11.1234 as DECIMAL(3,2))) t;
+
+select cast(a as DECIMAL(3,2))
+ from (select 11.1233 as a
+  UNION select 11.1234
+  UNION select 12.1234
+ ) t;
+
+select cast(a as DECIMAL(3,2)), count(*)
+ from (select 11.1233 as a
+  UNION select 11.1234
+  UNION select 12.1234
+ ) t group by 1;
+
 --echo End of 5.0 tests
diff --git a/sql/item_func.cc b/sql/item_func.cc
index e761cf7fb43..382e197acfe 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1050,9 +1050,32 @@ longlong Item_decimal_typecast::val_int()
 my_decimal *Item_decimal_typecast::val_decimal(my_decimal *dec)
 {
   my_decimal tmp_buf, *tmp= args[0]->val_decimal(&tmp_buf);
+  bool sign;
   if ((null_value= args[0]->null_value))
     return NULL;
   my_decimal_round(E_DEC_FATAL_ERROR, tmp, decimals, FALSE, dec);
+  sign= dec->sign();
+  if (unsigned_flag)
+  {
+    if (sign)
+    {
+      my_decimal_set_zero(dec);
+      goto err;
+    }
+  }
+  if (max_length - 2 - decimals < (uint) my_decimal_intg(dec))
+  {
+    max_my_decimal(dec, max_length - 2, decimals);
+    dec->sign(sign);
+    goto err;
+  }
+  return dec;
+
+err:
+  push_warning_printf(current_thd, MYSQL_ERROR::WARN_LEVEL_ERROR,
+                      ER_WARN_DATA_OUT_OF_RANGE,
+                      ER(ER_WARN_DATA_OUT_OF_RANGE),
+                      name, 1);
   return dec;
 }
 
diff --git a/sql/my_decimal.h b/sql/my_decimal.h
index 45270150d22..731a3bd505c 100644
--- a/sql/my_decimal.h
+++ b/sql/my_decimal.h
@@ -387,5 +387,13 @@ int my_decimal_cmp(const my_decimal *a, const my_decimal *b)
   return decimal_cmp((decimal_t*) a, (decimal_t*) b);
 }
 
+
+inline
+int my_decimal_intg(const my_decimal *a)
+{
+    return decimal_intg((decimal_t*) a);
+}
+
+
 #endif /*my_decimal_h*/
 
diff --git a/strings/decimal.c b/strings/decimal.c
index ea92174bfc8..65db68b1b59 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1911,6 +1911,14 @@ static int do_sub(decimal_t *from1, decimal_t *from2, decimal_t *to)
   return error;
 }
 
+int decimal_intg(decimal_t *from)
+{
+  int res;
+  dec1 *tmp_res;
+  tmp_res= remove_leading_zeroes(from, &res);
+  return res;
+}
+
 int decimal_add(decimal_t *from1, decimal_t *from2, decimal_t *to)
 {
   if (likely(from1->sign == from2->sign))