Bug#12563865 ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0

Buffer over-run on all platforms, crash on windows, wrong result on other platforms,
when rounding numbers which start with 999999999 and have
precision = 9 or 18 or 27 or 36 ...


mysql-test/r/type_newdecimal.result:
  New test cases.
mysql-test/t/type_newdecimal.test:
  New test cases.
sql/my_decimal.h:
  Add sanity checking code, to catch buffer over/under-run.
strings/decimal.c:
  The original initialization of intg1 (add 1 if buf[0] == DIG_MAX)
  will set p1 to point outside the buffer, and the loop to copy the original value
      while (buf0 < p0)
        *(--p1) = *(--p0);
  will overwrite memory outside the my_decimal object.

1 files changed, 13 insertions, 8 deletions

diff --git a/strings/decimal.c b/strings/decimal.c
index 1498aec15a1..87faff9b4cd 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1480,9 +1480,8 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
 {
   int frac0=scale>0 ? ROUND_UP(scale) : scale/DIG_PER_DEC1,
     frac1=ROUND_UP(from->frac), UNINIT_VAR(round_digit),
-      intg0=ROUND_UP(from->intg), error=E_DEC_OK, len=to->len,
-      intg1=ROUND_UP(from->intg +
-                     (((intg0 + frac0)>0) && (from->buf[0] == DIG_MAX)));
+    intg0=ROUND_UP(from->intg), error=E_DEC_OK, len=to->len;
+
   dec1 *buf0=from->buf, *buf1=to->buf, x, y, carry=0;
   int first_dig;
 
@@ -1497,6 +1496,12 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
   default: DBUG_ASSERT(0);
   }
 
+  /*
+    For my_decimal we always use len == DECIMAL_BUFF_LENGTH == 9
+    For internal testing here (ifdef MAIN) we always use len == 100/4
+   */
+  DBUG_ASSERT(from->len == to->len);
+
   if (unlikely(frac0+intg0 > len))
   {
     frac0=len-intg0;
@@ -1510,17 +1515,17 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
     return E_DEC_OK;
   }
 
-  if (to != from || intg1>intg0)
+  if (to != from)
   {
     dec1 *p0= buf0+intg0+max(frac1, frac0);
-    dec1 *p1= buf1+intg1+max(frac1, frac0);
+    dec1 *p1= buf1+intg0+max(frac1, frac0);
+
+    DBUG_ASSERT(p0 - buf0 <= len);
+    DBUG_ASSERT(p1 - buf1 <= len);
 
     while (buf0 < p0)
       *(--p1) = *(--p0);
-    if (unlikely(intg1 > intg0))
-      to->buf[0]= 0;
 
-    intg0= intg1;
     buf0=to->buf;
     buf1=to->buf;
     to->sign=from->sign;