Fixed bug when accessing wrong decimal value in dynamic string (Fixed lp:781233)

Store decimal 0.0 in zero bytes in dynamic strings.
mysqltest: Don't ignore error from mysql_stmt_fetch;  This could cause rows to be missing from log when running with --ps-protocol
Fixed wrong result length for CAST(... as TIME)






client/mysqltest.cc:
  Don't ignore error from mysql_stmt_fetch;  This could cause rows to be missing from log when running with --ps-protocol
libmysql/libmysql.c:
  The max length for a TIME column is 17, not 15.
mysql-test/r/dyncol.result:
  More tests
mysql-test/t/dyncol.test:
  More tests
mysys/ma_dyncol.c:
  Check content of decimal value on read and store to not get assert in decimal_bin_size().
  Store decimal 0.0 in zero bytes in dynamic strings. This also solves a problem where decimal 0 had different internal representations.
sql-common/my_time.c:
  Fixed DBUG_PRINT
sql/item_timefunc.h:
  Fixed wrong result length for CAST(... as TIME). This was the cause of failures in buildbot when doing cast(... as time);
sql/protocol.cc:
  More DBUG_PRINT

8 files changed, 124 insertions, 19 deletions

diff --git a/client/mysqltest.cc b/client/mysqltest.cc
index 65ae93696ea..9e23ac231cf 100644
--- a/client/mysqltest.cc
+++ b/client/mysqltest.cc
@@ -6673,6 +6673,7 @@ void append_stmt_result(DYNAMIC_STRING *ds, MYSQL_STMT *stmt,
   my_bool *is_null;
   ulong *length;
   uint i;
+  int error;
 
   /* Allocate array with bind structs, lengths and NULL flags */
   my_bind= (MYSQL_BIND*) my_malloc(num_fields * sizeof(MYSQL_BIND),
@@ -6700,7 +6701,7 @@ void append_stmt_result(DYNAMIC_STRING *ds, MYSQL_STMT *stmt,
     die("mysql_stmt_bind_result failed: %d: %s",
 	mysql_stmt_errno(stmt), mysql_stmt_error(stmt));
 
-  while (mysql_stmt_fetch(stmt) == 0)
+  while ((error=mysql_stmt_fetch(stmt)) == 0)
   {
     for (i= 0; i < num_fields; i++)
       append_field(ds, i, &fields[i], (char*)my_bind[i].buffer,
@@ -6709,8 +6710,11 @@ void append_stmt_result(DYNAMIC_STRING *ds, MYSQL_STMT *stmt,
       dynstr_append_mem(ds, "\n", 1);
   }
 
+  if (error != MYSQL_NO_DATA)
+    die("mysql_fetch didn't end with MYSQL_NO_DATA from statement: error: %d",
+        error);
   if (mysql_stmt_fetch(stmt) != MYSQL_NO_DATA)
-    die("fetch didn't end with MYSQL_NO_DATA from statement: %d %s",
+    die("mysql_fetch didn't end with MYSQL_NO_DATA from statement: %d %s",
 	mysql_stmt_errno(stmt), mysql_stmt_error(stmt));
 
   for (i= 0; i < num_fields; i++)
diff --git a/libmysql/libmysql.c b/libmysql/libmysql.c
index 42db3b28ca3..cc7902a80d7 100644
--- a/libmysql/libmysql.c
+++ b/libmysql/libmysql.c
@@ -4381,7 +4381,7 @@ static my_bool setup_one_fetch_function(MYSQL_BIND *param, MYSQL_FIELD *field)
     field->max_length= MAX_DOUBLE_STRING_REP_LENGTH;
     break;
   case MYSQL_TYPE_TIME:
-    field->max_length= 15;                    /* 19:23:48.123456 */
+    field->max_length= 17;                    /* -819:23:48.123456 */
     param->skip_result= skip_result_with_length;
     break;
   case MYSQL_TYPE_DATE:
diff --git a/mysql-test/r/dyncol.result b/mysql-test/r/dyncol.result
index aaa942f455a..70fe6049076 100644
--- a/mysql-test/r/dyncol.result
+++ b/mysql-test/r/dyncol.result
@@ -165,6 +165,12 @@ Note	1003	select hex(column_create(1,'afaf' AS char charset utf8 ,2,1212 AS unsi
 6, "2011-04-05" AS date,
 7, "- 0:45:49.000001" AS time,
 8, "2011-04-05 0:45:49.000001" AS datetime))`
+select hex(column_create(1, 0.0 AS decimal));
+hex(column_create(1, 0.0 AS decimal))
+000100010004
+select hex(column_create(1, 1.0 AS decimal));
+hex(column_create(1, 1.0 AS decimal))
+00010001000401018100
 #
 # column get uint
 #
@@ -575,6 +581,9 @@ column_get(column_create(1, "1223.5555" AS decimal(10,5)), 1 as decimal(3,2))
 9.99
 Warnings:
 Error	1264	Out of range value for column 'column_get(column_create(1, "1223.5555" AS decimal(10,5)), 1 as decimal(3,2))' at row 1
+select column_get(column_create(1, 0.0 AS decimal,2, 0.0 as decimal), 1 as decimal);
+column_get(column_create(1, 0.0 AS decimal,2, 0.0 as decimal), 1 as decimal)
+0
 #
 # column get datetime
 #
@@ -1234,3 +1243,23 @@ DROP TABLE t1;
 set @a=0x0102000200030004000F0D086B74697A6A7176746F6B687563726A746E7A746A666163726C6F7A6B62636B6B756B666779666977617369796F67756C726D62677A72756E63626D78636D7077706A6F736C6D636464696770786B6371637A6A6A6463737A6A676879716462637178646C666E6B6C726A637677696E7271746C616D646368687A6C707869786D666F666261797470616A63797673737A796D74747475666B717573687A79696E7276706F796A6E767361796A6F6D646F6378677A667074746363736A796D67746C786F697873686464616265616A7A6F7168707A6B776B6376737A6B72666C6F666C69636163686F6B666D627166786A71616F;
 select column_add(@a, 3, "a");
 ERROR HY000: Encountered illegal format of dynamic column string
+#
+# LP#781233 mysqld: decimal.c:1459: decimal_bin_size: 
+# Assertion `scale >= 0 && precision > 0 && scale <= precision' ...
+#
+set @a=0x00020008000009000C2C010080;
+select COLUMN_GET(@a, 9 AS DECIMAL);
+COLUMN_GET(@a, 9 AS DECIMAL)
+0
+select hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL)));
+hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL)))
+000100000004
+select hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL(19,0))));
+hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL(19,0))))
+000100000004
+select hex(COLUMN_CREATE(0, COLUMN_GET(COLUMN_CREATE(0, 0.0 as decimal), 0 as decimal)));
+hex(COLUMN_CREATE(0, COLUMN_GET(COLUMN_CREATE(0, 0.0 as decimal), 0 as decimal)))
+000100000004
+select hex(COLUMN_CREATE(0, 0.0 as decimal));
+hex(COLUMN_CREATE(0, 0.0 as decimal))
+000100000004
diff --git a/mysql-test/t/dyncol.test b/mysql-test/t/dyncol.test
index 7eb687b7eff..af693b02f63 100644
--- a/mysql-test/t/dyncol.test
+++ b/mysql-test/t/dyncol.test
@@ -63,7 +63,8 @@ select hex(COLUMN_CREATE(1, "afaf" AS char character set utf8,
 			 6, "2011-04-05" AS date,
 			 7, "- 0:45:49.000001" AS time,
 			 8, "2011-04-05 0:45:49.000001" AS datetime));
-
+select hex(column_create(1, 0.0 AS decimal));
+select hex(column_create(1, 1.0 AS decimal));
 
 --echo #
 --echo # column get uint
@@ -204,6 +205,7 @@ select column_get(column_create(1, "1223.5555" as double), 1 as decimal(5,2));
 select column_get(column_create(1, "-1223.5555" as double), 1 as decimal(5,2));
 select column_get(column_create(1, "1223.5555" AS double), 1 as decimal(3,2));
 select column_get(column_create(1, "1223.5555" AS decimal(10,5)), 1 as decimal(3,2));
+select column_get(column_create(1, 0.0 AS decimal,2, 0.0 as decimal), 1 as decimal);
 
 --echo #
 --echo # column get datetime
@@ -506,3 +508,16 @@ DROP TABLE t1;
 set @a=0x0102000200030004000F0D086B74697A6A7176746F6B687563726A746E7A746A666163726C6F7A6B62636B6B756B666779666977617369796F67756C726D62677A72756E63626D78636D7077706A6F736C6D636464696770786B6371637A6A6A6463737A6A676879716462637178646C666E6B6C726A637677696E7271746C616D646368687A6C707869786D666F666261797470616A63797673737A796D74747475666B717573687A79696E7276706F796A6E767361796A6F6D646F6378677A667074746363736A796D67746C786F697873686464616265616A7A6F7168707A6B776B6376737A6B72666C6F666C69636163686F6B666D627166786A71616F;
 --error ER_DYN_COL_WRONG_FORMAT
 select column_add(@a, 3, "a");
+
+--echo #
+--echo # LP#781233 mysqld: decimal.c:1459: decimal_bin_size: 
+--echo # Assertion `scale >= 0 && precision > 0 && scale <= precision' ...
+--echo #
+
+set @a=0x00020008000009000C2C010080;
+select COLUMN_GET(@a, 9 AS DECIMAL);
+select hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL)));
+select hex(COLUMN_CREATE(0, COLUMN_GET(@a, 9 AS DECIMAL(19,0))));
+
+select hex(COLUMN_CREATE(0, COLUMN_GET(COLUMN_CREATE(0, 0.0 as decimal), 0 as decimal)));
+select hex(COLUMN_CREATE(0, 0.0 as decimal));
diff --git a/mysys/ma_dyncol.c b/mysys/ma_dyncol.c
index fd060e867ed..dcb03d7f073 100644
--- a/mysys/ma_dyncol.c
+++ b/mysys/ma_dyncol.c
@@ -312,7 +312,9 @@ dynamic_column_sint_read(DYNAMIC_COLUMN_VALUE *store_it_here,
 
   @param value          The value for which we are calculating length
 
-  @return number of bytes
+  @return
+  Error:  (size_t) ~0
+  ok      number of bytes
 */
 
 static size_t
@@ -331,11 +333,32 @@ dynamic_column_value_len(DYNAMIC_COLUMN_VALUE *value)
     return (dynamic_column_var_uint_bytes(value->charset->number) +
             value->string_value.length);
   case DYN_COL_DECIMAL:
+  {
+    int precision= value->decimal_value.intg + value->decimal_value.frac;
+    int scale= value->decimal_value.frac;
+
+    if (precision == 0 || decimal_is_zero(&value->decimal_value))
+    {
+      /* This is here to simplify dynamic_column_decimal_store() */
+      value->decimal_value.intg= value->decimal_value.frac= 0;
+      return 0;
+    }
+    /*
+      Check if legal decimal;  This is needed to not get an assert in
+      decimal_bin_size(). However this should be impossible as all
+      decimals entered here should be valid and we have the special check
+      above to handle the unlikely but possible case that decimal_value.intg
+      and decimal.frac is 0.
+    */
+    if (scale < 0 || precision <= 0)
+    {
+      DBUG_ASSERT(0);                           /* Impossible */
+      return (size_t) ~0;
+    }
     return (dynamic_column_var_uint_bytes(value->decimal_value.intg) +
             dynamic_column_var_uint_bytes(value->decimal_value.frac) +
-            decimal_bin_size(value->decimal_value.intg +
-                             value->decimal_value.frac,
-                             value->decimal_value.frac));
+            decimal_bin_size(precision, scale));
+  }
   case DYN_COL_DATETIME:
     /* date+time in bits: 14 + 4 + 5 + 10 + 6 + 6 + 20 + 1 66bits ~= 9 bytes */
     return 9;
@@ -455,7 +478,14 @@ static enum enum_dyncol_func_result
 dynamic_column_decimal_store(DYNAMIC_COLUMN *str,
                              decimal_t *value)
 {
-  uint bin_size= decimal_bin_size(value->intg + value->frac, value->frac);
+  uint bin_size;
+  int precision= value->intg + value->frac;
+  
+  /* Store decimal zero as empty string */
+  if (precision == 0)
+    return ER_DYNCOL_OK;
+
+  bin_size= decimal_bin_size(precision, value->frac);
   if (dynstr_realloc(str, bin_size + 20))
     return ER_DYNCOL_RESOURCE;
 
@@ -464,8 +494,7 @@ dynamic_column_decimal_store(DYNAMIC_COLUMN *str,
   (void) dynamic_column_var_uint_store(str, value->frac);
 
   decimal2bin(value, (uchar *) str->str + str->length,
-              value->intg + value->frac,
-              value->frac);
+              precision, value->frac);
   str->length+= bin_size;
   return ER_DYNCOL_OK;
 }
@@ -502,20 +531,29 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here,
                             uchar *data, size_t length)
 {
   size_t intg_len, frac_len;
-  int intg, frac;
+  int intg, frac, precision, scale;
 
   dynamic_column_prepare_decimal(store_it_here);
+  /* Decimals 0.0 is stored as a zero length string */
+  if (length == 0)
+    return ER_DYNCOL_OK;                        /* value contains zero */
+
   intg= (int)dynamic_column_var_uint_get(data, length, &intg_len);
   data+= intg_len;
   frac= (int)dynamic_column_var_uint_get(data, length - intg_len, &frac_len);
   data+= frac_len;
 
   /* Check the size of data is correct */
-  if (decimal_bin_size(intg + frac, frac) !=
+  precision= intg + frac;
+  scale=     frac;
+  if (scale < 0 || precision <= 0 || scale > precision ||
+      (length - intg_len - frac_len) >
+      (size_t) (DECIMAL_BUFF_LENGTH*sizeof(decimal_digit_t)) ||
+      decimal_bin_size(intg + frac, frac) !=
       (int) (length - intg_len - frac_len))
     return ER_DYNCOL_FORMAT;
 
-  if (bin2decimal(data, &store_it_here->decimal_value, intg + frac, frac) !=
+  if (bin2decimal(data, &store_it_here->decimal_value, precision, scale) !=
       E_DEC_OK)
     return ER_DYNCOL_FORMAT;
   return ER_DYNCOL_OK;
@@ -1113,8 +1151,11 @@ dynamic_column_create_many_internal(DYNAMIC_COLUMN *str,
   {
     if (values[i].type != DYN_COL_NULL)
     {
+      size_t tmp;
       not_null_column_count++;
-      data_size+= dynamic_column_value_len(values + i);
+      data_size+= (tmp=dynamic_column_value_len(values + i));
+      if (tmp == (size_t) ~0)
+        return ER_DYNCOL_DATA;
     }
   }
 
@@ -1822,7 +1863,12 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str,
 
         plan[i].act= PLAN_REPLACE;
         /* get data delta in bytes */
-        plan[i].length= dynamic_column_value_len(plan[i].val);
+        if ((plan[i].length= dynamic_column_value_len(plan[i].val)) ==
+            (size_t) ~0)
+        {
+          rc= ER_DYNCOL_DATA;
+          goto end;
+        }
         data_delta+= plan[i].length - entry_data_size;
       }
     }
@@ -1841,7 +1887,12 @@ dynamic_column_update_many(DYNAMIC_COLUMN *str,
         plan[i].act= PLAN_ADD;
         header_delta++;                         /* One more row in header */
         /* get data delta in bytes */
-        plan[i].length= dynamic_column_value_len(plan[i].val);
+        if ((plan[i].length= dynamic_column_value_len(plan[i].val)) ==
+            (size_t) ~0)
+        {
+          rc= ER_DYNCOL_DATA;
+          goto end;
+        }
         data_delta+= plan[i].length;
       }
     }
diff --git a/sql-common/my_time.c b/sql-common/my_time.c
index 2081d4315d6..0d69fbca385 100644
--- a/sql-common/my_time.c
+++ b/sql-common/my_time.c
@@ -171,7 +171,7 @@ str_to_datetime(const char *str, uint length, MYSQL_TIME *l_time,
   my_bool found_delimitier= 0, found_space= 0;
   uint frac_pos, frac_len;
   DBUG_ENTER("str_to_datetime");
-  DBUG_PRINT("ENTER",("str: %.*s",length,str));
+  DBUG_PRINT("enter",("str: %.*s",length,str));
 
   LINT_INIT(field_length);
 
diff --git a/sql/item_timefunc.h b/sql/item_timefunc.h
index 7e73f6a4765..9808f8f3382 100644
--- a/sql/item_timefunc.h
+++ b/sql/item_timefunc.h
@@ -946,6 +946,12 @@ public:
   {
     return tmp_table_field_from_field_type(table, 0);
   }
+  void fix_length_and_dec()
+  {
+    collation.set(&my_charset_bin);
+    max_length= 17;
+    maybe_null= 1;
+  }
   bool result_as_longlong() { return TRUE; }
   longlong val_int();
   double val_real() { return val_real_from_decimal(); }
diff --git a/sql/protocol.cc b/sql/protocol.cc
index 2db81e925a2..4a3220b96fc 100644
--- a/sql/protocol.cc
+++ b/sql/protocol.cc
@@ -617,7 +617,7 @@ bool Protocol::send_fields(List<Item> *list, uint flags)
   Protocol_text prot(thd);
   String *local_packet= prot.storage_packet();
   CHARSET_INFO *thd_charset= thd->variables.character_set_results;
-  DBUG_ENTER("send_fields");
+  DBUG_ENTER("Protocol::send_fields");
 
   if (flags & SEND_NUM_ROWS)
   {				// Packet with number of elements