Bug#12563865 ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0

Buffer over-run on all platforms, crash on windows, wrong result on other platforms,
when rounding numbers which start with 999999999 and have
precision = 9 or 18 or 27 or 36 ...


mysql-test/r/type_newdecimal.result:
  New test cases.
mysql-test/t/type_newdecimal.test:
  New test cases.
sql/my_decimal.h:
  Add sanity checking code, to catch buffer over/under-run.
strings/decimal.c:
  The original initialization of intg1 (add 1 if buf[0] == DIG_MAX)
  will set p1 to point outside the buffer, and the loop to copy the original value
      while (buf0 < p0)
        *(--p1) = *(--p0);
  will overwrite memory outside the my_decimal object.

4 files changed, 108 insertions, 8 deletions

diff --git a/mysql-test/r/type_newdecimal.result b/mysql-test/r/type_newdecimal.result
index 90b6f524692..43caaa2239b 100644
--- a/mysql-test/r/type_newdecimal.result
+++ b/mysql-test/r/type_newdecimal.result
@@ -1539,4 +1539,48 @@ select * from t1;
 5.05 / 0.014
 360.714286
 DROP TABLE t1;
+#
+# Bug#12563865
+# ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0
+#
+SELECT substring(('M') FROM (999999999999999999999999999999999999999999999999999999999999999999999999999999999)) AS foo;
+foo
+
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT min(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+999999999999999999999999999999999999999999999999999999999999999999999999999999999
+SELECT multipolygonfromtext(('4294967294.1'),(999999999999999999999999999999999999999999999999999999999999999999999999999999999)) AS foo;
+foo
+NULL
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT convert((999999999999999999999999999999999999999999999999999999999999999999999999999999999), decimal(30,30)) AS foo;
+foo
+0.999999999999999999999999999999
+Warnings:
+Error	1264	Out of range value adjusted for column 'foo' at row 1
+SELECT bit_xor(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+9223372036854775807
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT -(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+-999999999999999999999999999999999999999999999999999999999999999999999999999999999
+SELECT date_sub((999999999999999999999999999999999999999999999999999999999999999999999999999999999),
+interval ((SELECT date_add((0x77500000),
+interval ('Oml') second)))
+day_minute)
+AS foo;
+foo
+NULL
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Warning	1292	Truncated incorrect datetime value: '9223372036854775807'
+SELECT truncate(999999999999999999999999999999999999999999999999999999999999999999999999999999999, 28) AS foo;
+foo
+999999999999999999999999999999999999999999999999999999999999999999999999999999999
 End of 5.0 tests
diff --git a/mysql-test/t/type_newdecimal.test b/mysql-test/t/type_newdecimal.test
index a5331582df6..a55951caf63 100644
--- a/mysql-test/t/type_newdecimal.test
+++ b/mysql-test/t/type_newdecimal.test
@@ -1235,4 +1235,25 @@ show create table t1;
 select * from t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#12563865
+--echo # ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0
+--echo #
+
+let $nine_81=
+999999999999999999999999999999999999999999999999999999999999999999999999999999999;
+
+eval SELECT substring(('M') FROM ($nine_81)) AS foo;
+eval SELECT min($nine_81) AS foo;
+eval SELECT multipolygonfromtext(('4294967294.1'),($nine_81)) AS foo;
+eval SELECT convert(($nine_81), decimal(30,30)) AS foo;
+eval SELECT bit_xor($nine_81) AS foo;
+eval SELECT -($nine_81) AS foo;
+eval SELECT date_sub(($nine_81),
+                     interval ((SELECT date_add((0x77500000),
+                                                 interval ('Oml') second)))
+                     day_minute)
+AS foo;
+eval SELECT truncate($nine_81, 28) AS foo;
+
 --echo End of 5.0 tests
diff --git a/sql/my_decimal.h b/sql/my_decimal.h
index 6a0d05921ec..ee023438f20 100644
--- a/sql/my_decimal.h
+++ b/sql/my_decimal.h
@@ -86,12 +86,31 @@ inline int my_decimal_int_part(uint precision, uint decimals)
 
 class my_decimal :public decimal_t
 {
+  /*
+    Several of the routines in strings/decimal.c have had buffer
+    overrun/underrun problems. These are *not* caught by valgrind.
+    To catch them, we allocate dummy fields around the buffer,
+    and test that their values do not change.
+   */
+#if !defined(DBUG_OFF)
+  int foo1;
+#endif
+
   decimal_digit_t buffer[DECIMAL_BUFF_LENGTH];
 
+#if !defined(DBUG_OFF)
+  int foo2;
+  static const int test_value= 123;
+#endif
+
 public:
 
   void init()
   {
+#if !defined(DBUG_OFF)
+    foo1= test_value;
+    foo2= test_value;
+#endif
     len= DECIMAL_BUFF_LENGTH;
     buf= buffer;
 #if !defined (HAVE_purify) && !defined(DBUG_OFF)
@@ -104,6 +123,17 @@ public:
   {
     init();
   }
+  ~my_decimal()
+  {
+    sanity_check();
+  }
+
+  void sanity_check()
+  {
+    DBUG_ASSERT(foo1 == test_value);
+    DBUG_ASSERT(foo2 == test_value);
+  }
+
   void fix_buffer_pointer() { buf= buffer; }
 
   bool sign() const { return decimal_t::sign; }
diff --git a/strings/decimal.c b/strings/decimal.c
index 1498aec15a1..87faff9b4cd 100644
--- a/strings/decimal.c
+++ b/strings/decimal.c
@@ -1480,9 +1480,8 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
 {
   int frac0=scale>0 ? ROUND_UP(scale) : scale/DIG_PER_DEC1,
     frac1=ROUND_UP(from->frac), UNINIT_VAR(round_digit),
-      intg0=ROUND_UP(from->intg), error=E_DEC_OK, len=to->len,
-      intg1=ROUND_UP(from->intg +
-                     (((intg0 + frac0)>0) && (from->buf[0] == DIG_MAX)));
+    intg0=ROUND_UP(from->intg), error=E_DEC_OK, len=to->len;
+
   dec1 *buf0=from->buf, *buf1=to->buf, x, y, carry=0;
   int first_dig;
 
@@ -1497,6 +1496,12 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
   default: DBUG_ASSERT(0);
   }
 
+  /*
+    For my_decimal we always use len == DECIMAL_BUFF_LENGTH == 9
+    For internal testing here (ifdef MAIN) we always use len == 100/4
+   */
+  DBUG_ASSERT(from->len == to->len);
+
   if (unlikely(frac0+intg0 > len))
   {
     frac0=len-intg0;
@@ -1510,17 +1515,17 @@ decimal_round(decimal_t *from, decimal_t *to, int scale,
     return E_DEC_OK;
   }
 
-  if (to != from || intg1>intg0)
+  if (to != from)
   {
     dec1 *p0= buf0+intg0+max(frac1, frac0);
-    dec1 *p1= buf1+intg1+max(frac1, frac0);
+    dec1 *p1= buf1+intg0+max(frac1, frac0);
+
+    DBUG_ASSERT(p0 - buf0 <= len);
+    DBUG_ASSERT(p1 - buf1 <= len);
 
     while (buf0 < p0)
       *(--p1) = *(--p0);
-    if (unlikely(intg1 > intg0))
-      to->buf[0]= 0;
 
-    intg0= intg1;
     buf0=to->buf;
     buf1=to->buf;
     to->sign=from->sign;