BUG#31799: Scrambled number output due to integer overflow

An integer overflow in number->string conversion caused completely
wrong output of the number LONGLONG_MIN with gcc 4.2.1.

Fixed by eliminating the overflow, using only operations that are
well-defined in ANSI C.


strings/ctype-simple.c:
  An integer overflow in number->string conversion caused completely
  wrong output of the number LONGLONG_MIN with gcc 4.2.1.
  
  Fixed by eliminating the overflow, using only operations that are
  well-defined in ANSI C.
strings/ctype-ucs2.c:
  An integer overflow in number->string conversion caused completely
  wrong output of the number LONGLONG_MIN with gcc 4.2.1.
  
  Fixed by eliminating the overflow, using only operations that are
  well-defined in ANSI C.
strings/int2str.c:
  An integer overflow in number->string conversion caused completely
  wrong output of the number LONGLONG_MIN with gcc 4.2.1.
  
  Fixed by eliminating the overflow, using only operations that are
  well-defined in ANSI C.
strings/longlong2str.c:
  An integer overflow in number->string conversion caused completely
  wrong output of the number LONGLONG_MIN with gcc 4.2.1.
  
  Fixed by eliminating the overflow, using only operations that are
  well-defined in ANSI C.

4 files changed, 56 insertions, 40 deletions

diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c
index e073262cd4c..99e02e02014 100644
--- a/strings/ctype-simple.c
+++ b/strings/ctype-simple.c
@@ -837,6 +837,7 @@ int my_long10_to_str_8bit(CHARSET_INFO *cs __attribute__((unused)),
   register char *p, *e;
   long int new_val;
   uint sign=0;
+  unsigned long int uval = (unsigned long int) val;
 
   e = p = &buffer[sizeof(buffer)-1];
   *p= 0;
@@ -845,15 +846,16 @@ int my_long10_to_str_8bit(CHARSET_INFO *cs __attribute__((unused)),
   {
     if (val < 0)
     {
-      val= -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval= (unsigned long int)0 - uval;
       *dst++= '-';
       len--;
       sign= 1;
     }
   }
   
-  new_val = (long) ((unsigned long int) val / 10);
-  *--p    = '0'+ (char) ((unsigned long int) val - (unsigned long) new_val * 10);
+  new_val = (long) (uval / 10);
+  *--p    = '0'+ (char) (uval - (unsigned long) new_val * 10);
   val     = new_val;
   
   while (val != 0)
@@ -876,12 +878,14 @@ int my_longlong10_to_str_8bit(CHARSET_INFO *cs __attribute__((unused)),
   register char *p, *e;
   long long_val;
   uint sign= 0;
+  ulonglong uval = (ulonglong)val;
   
   if (radix < 0)
   {
     if (val < 0)
     {
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (ulonglong)0 - uval;
       *dst++= '-';
       len--;
       sign= 1;
@@ -891,22 +895,22 @@ int my_longlong10_to_str_8bit(CHARSET_INFO *cs __attribute__((unused)),
   e = p = &buffer[sizeof(buffer)-1];
   *p= 0;
   
-  if (val == 0)
+  if (uval == 0)
   {
     *--p= '0';
     len= 1;
     goto cnv;
   }
   
-  while ((ulonglong) val > (ulonglong) LONG_MAX)
+  while (uval > (ulonglong) LONG_MAX)
   {
-    ulonglong quo=(ulonglong) val/(uint) 10;
-    uint rem= (uint) (val- quo* (uint) 10);
+    ulonglong quo= uval/(uint) 10;
+    uint rem= (uint) (uval- quo* (uint) 10);
     *--p = '0' + rem;
-    val= quo;
+    uval= quo;
   }
   
-  long_val= (long) val;
+  long_val= (long) uval;
   while (long_val != 0)
   {
     long quo= long_val/10;
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index b5353c55e4c..97e5defdec1 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -1018,6 +1018,7 @@ int my_l10tostr_ucs2(CHARSET_INFO *cs,
   register char *p, *db, *de;
   long int new_val;
   int  sl=0;
+  unsigned long int uval = (unsigned long int) val;
   
   p = &buffer[sizeof(buffer)-1];
   *p='\0';
@@ -1027,12 +1028,13 @@ int my_l10tostr_ucs2(CHARSET_INFO *cs,
     if (val < 0)
     {
       sl   = 1;
-      val  = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval  = (unsigned long int)0 - uval;
     }
   }
   
-  new_val = (long) ((unsigned long int) val / 10);
-  *--p    = '0'+ (char) ((unsigned long int) val - (unsigned long) new_val * 10);
+  new_val = (long) (uval / 10);
+  *--p    = '0'+ (char) (uval - (unsigned long) new_val * 10);
   val     = new_val;
   
   while (val != 0)
@@ -1065,34 +1067,36 @@ int my_ll10tostr_ucs2(CHARSET_INFO *cs __attribute__((unused)),
   register char *p, *db, *de;
   long long_val;
   int  sl=0;
+  ulonglong uval= (ulonglong) val;
   
   if (radix < 0)
   {
     if (val < 0)
     {
       sl=1;
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (ulonglong)0 - uval;
     }
   }
   
   p = &buffer[sizeof(buffer)-1];
   *p='\0';
   
-  if (val == 0)
+  if (uval == 0)
   {
     *--p='0';
     goto cnv;
   }
   
-  while ((ulonglong) val > (ulonglong) LONG_MAX)
+  while (uval > (ulonglong) LONG_MAX)
   {
-    ulonglong quo=(ulonglong) val/(uint) 10;
-    uint rem= (uint) (val- quo* (uint) 10);
+    ulonglong quo= uval/(uint) 10;
+    uint rem= (uint) (uval- quo* (uint) 10);
     *--p = '0' + rem;
-    val= quo;
+    uval= quo;
   }
   
-  long_val= (long) val;
+  long_val= (long) uval;
   while (long_val != 0)
   {
     long quo= long_val/10;
diff --git a/strings/int2str.c b/strings/int2str.c
index 9fc53032819..fba98aac3f1 100644
--- a/strings/int2str.c
+++ b/strings/int2str.c
@@ -57,6 +57,7 @@ int2str(register long int val, register char *dst, register int radix,
   register char *p;
   long int new_val;
   char *dig_vec= upcase ? _dig_vec_upper : _dig_vec_lower;
+  ulong uval= (ulong) val;
 
   if (radix < 0)
   {
@@ -65,7 +66,8 @@ int2str(register long int val, register char *dst, register int radix,
     if (val < 0)
     {
       *dst++ = '-';
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (ulong)0 - uval;
     }
     radix = -radix;
   }
@@ -86,8 +88,8 @@ int2str(register long int val, register char *dst, register int radix,
   */
   p = &buffer[sizeof(buffer)-1];
   *p = '\0';
-  new_val=(ulong) val / (ulong) radix;
-  *--p = dig_vec[(uchar) ((ulong) val- (ulong) new_val*(ulong) radix)];
+  new_val= uval / (ulong) radix;
+  *--p = dig_vec[(uchar) (uval- (ulong) new_val*(ulong) radix)];
   val = new_val;
 #ifdef HAVE_LDIV
   while (val != 0)
@@ -133,20 +135,22 @@ char *int10_to_str(long int val,char *dst,int radix)
   char buffer[65];
   register char *p;
   long int new_val;
+  unsigned long int uval = (unsigned long int) val;
 
   if (radix < 0)				/* -10 */
   {
     if (val < 0)
     {
       *dst++ = '-';
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (unsigned long int)0 - uval;
     }
   }
 
   p = &buffer[sizeof(buffer)-1];
   *p = '\0';
-  new_val= (long) ((unsigned long int) val / 10);
-  *--p = '0'+ (char) ((unsigned long int) val - (unsigned long) new_val * 10);
+  new_val= (long) (uval / 10);
+  *--p = '0'+ (char) (uval - (unsigned long) new_val * 10);
   val = new_val;
 
   while (val != 0)
diff --git a/strings/longlong2str.c b/strings/longlong2str.c
index c464abcfccd..d7de5bb0f7c 100644
--- a/strings/longlong2str.c
+++ b/strings/longlong2str.c
@@ -51,13 +51,15 @@ char *longlong2str(longlong val,char *dst,int radix)
   char buffer[65];
   register char *p;
   long long_val;
+  ulonglong uval= (ulonglong) val;
 
   if (radix < 0)
   {
     if (radix < -36 || radix > -2) return (char*) 0;
     if (val < 0) {
       *dst++ = '-';
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (ulonglong)0 - uval;
     }
     radix = -radix;
   }
@@ -65,7 +67,7 @@ char *longlong2str(longlong val,char *dst,int radix)
   {
     if (radix > 36 || radix < 2) return (char*) 0;
   }
-  if (val == 0)
+  if (uval == 0)
   {
     *dst++='0';
     *dst='\0';
@@ -74,14 +76,14 @@ char *longlong2str(longlong val,char *dst,int radix)
   p = &buffer[sizeof(buffer)-1];
   *p = '\0';
 
-  while ((ulonglong) val > (ulonglong) LONG_MAX)
+  while (uval > (ulonglong) LONG_MAX)
   {
-    ulonglong quo=(ulonglong) val/(uint) radix;
-    uint rem= (uint) (val- quo* (uint) radix);
+    ulonglong quo= uval/(uint) radix;
+    uint rem= (uint) (uval- quo* (uint) radix);
     *--p = _dig_vec_upper[rem];
-    val= quo;
+    uval= quo;
   }
-  long_val= (long) val;
+  long_val= (long) uval;
   while (long_val != 0)
   {
     long quo= long_val/radix;
@@ -100,17 +102,19 @@ char *longlong10_to_str(longlong val,char *dst,int radix)
   char buffer[65];
   register char *p;
   long long_val;
+  ulonglong uval= (ulonglong) val;
 
   if (radix < 0)
   {
     if (val < 0)
     {
       *dst++ = '-';
-      val = -val;
+      /* Avoid integer overflow in (-val) for LONGLONG_MIN (BUG#31799). */
+      uval = (ulonglong)0 - uval;
     }
   }
 
-  if (val == 0)
+  if (uval == 0)
   {
     *dst++='0';
     *dst='\0';
@@ -119,14 +123,14 @@ char *longlong10_to_str(longlong val,char *dst,int radix)
   p = &buffer[sizeof(buffer)-1];
   *p = '\0';
 
-  while ((ulonglong) val > (ulonglong) LONG_MAX)
+  while (uval > (ulonglong) LONG_MAX)
   {
-    ulonglong quo=(ulonglong) val/(uint) 10;
-    uint rem= (uint) (val- quo* (uint) 10);
+    ulonglong quo= uval/(uint) 10;
+    uint rem= (uint) (uval- quo* (uint) 10);
     *--p = _dig_vec_upper[rem];
-    val= quo;
+    uval= quo;
   }
-  long_val= (long) val;
+  long_val= (long) uval;
   while (long_val != 0)
   {
     long quo= long_val/10;