Bug#57477 SIGFPE when dividing a huge number a negative number

The problem is dividing by const value when
the result is out of supported range.
The fix:
-return LONGLONG_MIN if the result is out of supported range for DIV operator.
-return 0 if divisor is -1 for MOD operator.


mysql-test/r/func_math.result:
  test case
mysql-test/t/func_math.test:
  test case
sql/item_func.cc:
  -return LONGLONG_MIN if the result is out of supported range for DIV operator.
  -return 0 if divisor is -1 for MOD operator.

1 files changed, 10 insertions, 6 deletions

diff --git a/sql/item_func.cc b/sql/item_func.cc
index 30d5d844f7c..3dbff43bb67 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1356,9 +1356,13 @@ longlong Item_func_int_div::val_int()
     signal_divide_by_null();
     return 0;
   }
-  return (unsigned_flag ?
-	  (ulonglong) value / (ulonglong) val2 :
-	  value / val2);
+
+  if (unsigned_flag)
+    return  ((ulonglong) value / (ulonglong) val2);
+  else if (value == LONGLONG_MIN && val2 == -1)
+    return LONGLONG_MIN;
+  else
+    return value / val2;
 }
 
 
@@ -1392,9 +1396,9 @@ longlong Item_func_mod::int_op()
   if (args[0]->unsigned_flag)
     result= args[1]->unsigned_flag ? 
       ((ulonglong) value) % ((ulonglong) val2) : ((ulonglong) value) % val2;
-  else
-    result= args[1]->unsigned_flag ?
-      value % ((ulonglong) val2) : value % val2;
+  else result= args[1]->unsigned_flag ?
+         value % ((ulonglong) val2) :
+         (val2 == -1) ? 0 : value % val2;
 
   return result;
 }