Bug#12563865 ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0

Buffer over-run on all platforms, crash on windows, wrong result on other platforms,
when rounding numbers which start with 999999999 and have
precision = 9 or 18 or 27 or 36 ...


mysql-test/r/type_newdecimal.result:
  New test cases.
mysql-test/t/type_newdecimal.test:
  New test cases.
sql/my_decimal.h:
  Add sanity checking code, to catch buffer over/under-run.
strings/decimal.c:
  The original initialization of intg1 (add 1 if buf[0] == DIG_MAX)
  will set p1 to point outside the buffer, and the loop to copy the original value
      while (buf0 < p0)
        *(--p1) = *(--p0);
  will overwrite memory outside the my_decimal object.

2 files changed, 65 insertions, 0 deletions

diff --git a/mysql-test/r/type_newdecimal.result b/mysql-test/r/type_newdecimal.result
index 90b6f524692..43caaa2239b 100644
--- a/mysql-test/r/type_newdecimal.result
+++ b/mysql-test/r/type_newdecimal.result
@@ -1539,4 +1539,48 @@ select * from t1;
 5.05 / 0.014
 360.714286
 DROP TABLE t1;
+#
+# Bug#12563865
+# ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0
+#
+SELECT substring(('M') FROM (999999999999999999999999999999999999999999999999999999999999999999999999999999999)) AS foo;
+foo
+
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT min(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+999999999999999999999999999999999999999999999999999999999999999999999999999999999
+SELECT multipolygonfromtext(('4294967294.1'),(999999999999999999999999999999999999999999999999999999999999999999999999999999999)) AS foo;
+foo
+NULL
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT convert((999999999999999999999999999999999999999999999999999999999999999999999999999999999), decimal(30,30)) AS foo;
+foo
+0.999999999999999999999999999999
+Warnings:
+Error	1264	Out of range value adjusted for column 'foo' at row 1
+SELECT bit_xor(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+9223372036854775807
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+SELECT -(999999999999999999999999999999999999999999999999999999999999999999999999999999999) AS foo;
+foo
+-999999999999999999999999999999999999999999999999999999999999999999999999999999999
+SELECT date_sub((999999999999999999999999999999999999999999999999999999999999999999999999999999999),
+interval ((SELECT date_add((0x77500000),
+interval ('Oml') second)))
+day_minute)
+AS foo;
+foo
+NULL
+Warnings:
+Error	1292	Truncated incorrect DECIMAL value: ''
+Warning	1292	Truncated incorrect datetime value: '9223372036854775807'
+SELECT truncate(999999999999999999999999999999999999999999999999999999999999999999999999999999999, 28) AS foo;
+foo
+999999999999999999999999999999999999999999999999999999999999999999999999999999999
 End of 5.0 tests
diff --git a/mysql-test/t/type_newdecimal.test b/mysql-test/t/type_newdecimal.test
index a5331582df6..a55951caf63 100644
--- a/mysql-test/t/type_newdecimal.test
+++ b/mysql-test/t/type_newdecimal.test
@@ -1235,4 +1235,25 @@ show create table t1;
 select * from t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#12563865
+--echo # ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0
+--echo #
+
+let $nine_81=
+999999999999999999999999999999999999999999999999999999999999999999999999999999999;
+
+eval SELECT substring(('M') FROM ($nine_81)) AS foo;
+eval SELECT min($nine_81) AS foo;
+eval SELECT multipolygonfromtext(('4294967294.1'),($nine_81)) AS foo;
+eval SELECT convert(($nine_81), decimal(30,30)) AS foo;
+eval SELECT bit_xor($nine_81) AS foo;
+eval SELECT -($nine_81) AS foo;
+eval SELECT date_sub(($nine_81),
+                     interval ((SELECT date_add((0x77500000),
+                                                 interval ('Oml') second)))
+                     day_minute)
+AS foo;
+eval SELECT truncate($nine_81, 28) AS foo;
+
 --echo End of 5.0 tests