Bug#12563865 ROUNDED,TMP_BUF,DECIMAL_VALUE STACK CORRUPTION IN ALL VERSIONS >=5.0

Buffer over-run on all platforms, crash on windows, wrong result on other platforms,
when rounding numbers which start with 999999999 and have
precision = 9 or 18 or 27 or 36 ...


mysql-test/r/type_newdecimal.result:
  New test cases.
mysql-test/t/type_newdecimal.test:
  New test cases.
sql/my_decimal.h:
  Add sanity checking code, to catch buffer over/under-run.
strings/decimal.c:
  The original initialization of intg1 (add 1 if buf[0] == DIG_MAX)
  will set p1 to point outside the buffer, and the loop to copy the original value
      while (buf0 < p0)
        *(--p1) = *(--p0);
  will overwrite memory outside the my_decimal object.

1 files changed, 30 insertions, 0 deletions

diff --git a/sql/my_decimal.h b/sql/my_decimal.h
index 6a0d05921ec..ee023438f20 100644
--- a/sql/my_decimal.h
+++ b/sql/my_decimal.h
@@ -86,12 +86,31 @@ inline int my_decimal_int_part(uint precision, uint decimals)
 
 class my_decimal :public decimal_t
 {
+  /*
+    Several of the routines in strings/decimal.c have had buffer
+    overrun/underrun problems. These are *not* caught by valgrind.
+    To catch them, we allocate dummy fields around the buffer,
+    and test that their values do not change.
+   */
+#if !defined(DBUG_OFF)
+  int foo1;
+#endif
+
   decimal_digit_t buffer[DECIMAL_BUFF_LENGTH];
 
+#if !defined(DBUG_OFF)
+  int foo2;
+  static const int test_value= 123;
+#endif
+
 public:
 
   void init()
   {
+#if !defined(DBUG_OFF)
+    foo1= test_value;
+    foo2= test_value;
+#endif
     len= DECIMAL_BUFF_LENGTH;
     buf= buffer;
 #if !defined (HAVE_purify) && !defined(DBUG_OFF)
@@ -104,6 +123,17 @@ public:
   {
     init();
   }
+  ~my_decimal()
+  {
+    sanity_check();
+  }
+
+  void sanity_check()
+  {
+    DBUG_ASSERT(foo1 == test_value);
+    DBUG_ASSERT(foo2 == test_value);
+  }
+
   void fix_buffer_pointer() { buf= buffer; }
 
   bool sign() const { return decimal_t::sign; }