WL#2649 Number-to-string conversions

added:
  include/ctype_numconv.inc
  mysql-test/include/ctype_numconv.inc
  mysql-test/r/ctype_binary.result
  mysql-test/t/ctype_binary.test
  Adding tests

modified:

  mysql-test/r/bigint.result
  mysql-test/r/case.result
  mysql-test/r/create.result
  mysql-test/r/ctype_cp1251.result
  mysql-test/r/ctype_latin1.result
  mysql-test/r/ctype_ucs.result
  mysql-test/r/func_gconcat.result
  mysql-test/r/func_str.result
  mysql-test/r/metadata.result
  mysql-test/r/ps_1general.result
  mysql-test/r/ps_2myisam.result
  mysql-test/r/ps_3innodb.result
  mysql-test/r/ps_4heap.result
  mysql-test/r/ps_5merge.result
  mysql-test/r/show_check.result
  mysql-test/r/type_datetime.result
  mysql-test/r/type_ranges.result
  mysql-test/r/union.result
  mysql-test/suite/ndb/r/ps_7ndb.result
  mysql-test/t/ctype_cp1251.test
  mysql-test/t/ctype_latin1.test
  mysql-test/t/ctype_ucs.test
  mysql-test/t/func_str.test
    Fixing tests


  @ sql/field.cc
     - Return str result using my_charset_numeric.
     - Using real multi-byte aware str_to_XXX functions
       to handle tricky charset values propely (e.g. UCS2)
  @ sql/field.h
     - Changing derivation of non-string field types to DERIVATION_NUMERIC.
     - Changing binary() for numeric/datetime fields to always
     return TRUE even if charset is not my_charset_bin. We need
     this to keep ha_base_keytype() return HA_KEYTYPE_BINARY.
     - Adding BINARY_FLAG into some fields, because it's not
     being set automatically anymore with
     "my_charset_bin to my_charset_numeric" change.
    - Changing derivation for numeric/datetime datatypes to a weaker
      value, to make "SELECT concat('string', field)" use character
      set of the string literal for the result of the function.
  @ sql/item.cc
     - Implementing generic val_str_ascii().
     - Using max_char_length() instead of direct read of max_length
       to make "tricky" charsets like UCS2 work.
       NOTE: in the future we'll possibly remove all direct reads of max_length
     - Fixing Item_num::safe_charset_converter().
       Previously it alligned binary string to
       character string (for example by adding leading 0x00
       when doing binary->UCS2 conversion). Now it just
       converts from my_charset_numbner to "tocs".
     - Using val_str_ascii() in Item::get_time() to make UCS2 arguments work.
     - Other misc changes
  @ sql/item.h
     - Changing MY_COLL_CMP_CONV and MY_COLL_ALLOW_CONV to
       bit operations instead of hard-coded bit masks.
     - Addding new method DTCollation.set_numeric().
     - Adding new methods to Item.
     - Adding helper functions to make code look nicer:
       agg_item_charsets_for_string_result()
       agg_item_charsets_for_comparison()
     - Changing charset for Item_num-derived items
       from my_charset_bin to my_charset_numeric
       (which is an alias for latin1).
  @ sql/item_cmpfunc.cc
     - Using new helper functions
     - Other misc changes
  @ sql/item_cmpfunc.h
     - Fixing strcmp() to return max_length=2.
       Previously it returned 1, which was wrong,
       because it did not fit '-1'.
  @ sql/item_func.cc
     - Using new helper functions
     - Other minor changes
  @ sql/item_func.h
     - Removing unused functions
     - Adding helper functions
       agg_arg_charsets_for_string_result()
       agg_arg_charsets_for_comparison()
     - Adding set_numeric() into constructors of numeric items.
     - Using fix_length_and_charset() and fix_char_length()
       instead of direct write to max_length.
  @ sql/item_geofunc.cc
     - Changing class for Item_func_geometry_type and
       Item_func_as_wkt from Item_str_func to
       Item_str_ascii_func, to make them return UCS2 result
       properly (when character_set_connection=ucs2).
  @ sql/item_geofunc.h
     - Changing class for Item_func_geometry_type and
       Item_func_as_wkt from Item_str_func to
       Item_str_ascii_func, to make them return UCS2 result
       properly (when @@character_set_connection=ucs2).
  @ sql/item_strfunc.cc
     - Implementing Item_str_func::val_str().
     - Renaming val_str to val_str_ascii for some items,
       to make them work with UCS2 properly.
     - Using new helper functions
     - All single-argument functions that expect string
       result now call this method:
       agg_arg_charsets_for_string_result(collation, args, 1);
       This enables character set conversion to @@character_set_connection
       in case of pure numeric input.
  @ sql/item_strfunc.h
     - Introducing Item_str_ascii_func - for functions
       which return pure ASCII data, for performance purposes,
       as well as for the cases when the old implementation
       of val_str() was heavily 8-bit oriented and implementing
       a UCS2-aware version is tricky.
  @ sql/item_sum.cc
     - Using new helper functions.
  @ sql/item_timefunc.cc
     - Using my_charset_numeric instead of my_charset_bin.
     - Using fix_char_length(), fix_length_and_charset()
       and fix_length_and_charset_datetime()
       instead of direct write to max_length.
     - Using tricky-charset aware function str_to_time_with_warn()
  @ sql/item_timefunc.h
     - Using new helper functions for charset and length initialization.
     - Changing base class for Item_func_get_format() to make
       it return UCS2 properly (when character_set_connection=ucs2).
  @ sql/item_xmlfunc.cc
     - Using new helper function
  @ sql/my_decimal.cc
     - Adding a new DECIMAL to CHAR converter
       with real multibyte support (e.g. UCS2)

  @ sql/mysql_priv.h
     - Introducing a new derivation level for numeric/datetime data types.
     - Adding macros for my_charset_numeric and MY_REPERTOIRE_NUMERIC.
     - Adding prototypes for str_set_decimal()
     - Adding prototypes for character-set aware str_to_xxx() functions.
  @ sql/protocol.cc
     - Changing charsetnr to "binary" client-side metadata for
       numeric/datetime data types.
  @ sql/time.cc
     - Adding to_ascii() helper function, to convert a string
       in any character set to ascii representation. In the
       future can be extended to understand digits written
       in various non-Latin word scripts.
     - Adding real multy-byte character set aware versions for str_to_XXXX,
       to make these these type of queries work correct:
         INSERT INTO t1 SET datetime_column=ucs2_expression;
   @  strings/ctype-ucs2.c
     - endptr was not calculated correctly. INSERTing of UCS2
       values into numeric columns returned warnings about
       truncated wrong data.

1 files changed, 40 insertions, 24 deletions

diff --git a/sql/field.cc b/sql/field.cc
index 51bb527fc85..8b48831dc57 100644
--- a/sql/field.cc
+++ b/sql/field.cc
@@ -1808,7 +1808,7 @@ bool Field::get_date(MYSQL_TIME *ltime,uint fuzzydate)
   char buff[40];
   String tmp(buff,sizeof(buff),&my_charset_bin),*res;
   if (!(res=val_str(&tmp)) ||
-      str_to_datetime_with_warn(res->ptr(), res->length(),
+      str_to_datetime_with_warn(res->charset(), res->ptr(), res->length(),
                                 ltime, fuzzydate) <= MYSQL_TIMESTAMP_ERROR)
     return 1;
   return 0;
@@ -1819,7 +1819,7 @@ bool Field::get_time(MYSQL_TIME *ltime)
   char buff[40];
   String tmp(buff,sizeof(buff),&my_charset_bin),*res;
   if (!(res=val_str(&tmp)) ||
-      str_to_time_with_warn(res->ptr(), res->length(), ltime))
+      str_to_time_with_warn(res->charset(), res->ptr(), res->length(), ltime))
     return 1;
   return 0;
 }
@@ -1836,7 +1836,9 @@ int Field::store_time(MYSQL_TIME *ltime, timestamp_type type_arg)
   ASSERT_COLUMN_MARKED_FOR_WRITE;
   char buff[MAX_DATE_STRING_REP_LENGTH];
   uint length= (uint) my_TIME_to_str(ltime, buff);
-  return store(buff, length, &my_charset_bin);
+  /* Avoid conversion when field character set is ASCII compatible */
+  return store(buff, length, (charset()->state & MY_CS_NONASCII) ?
+                              &my_charset_latin1 : charset());
 }
 
 
@@ -2428,7 +2430,7 @@ String *Field_decimal::val_str(String *val_buffer __attribute__((unused)),
   size_t tmp_length;
 
   for (str=ptr ; *str == ' ' ; str++) ;
-  val_ptr->set_charset(&my_charset_bin);
+  val_ptr->set_charset(&my_charset_numeric);
   tmp_length= (size_t) (str-ptr);
   if (field_length < tmp_length)		// Error in data
     val_ptr->length(0);
@@ -2558,7 +2560,7 @@ Field *Field_new_decimal::create_from_item (Item *item)
 {
   uint8 dec= item->decimals;
   uint8 intg= item->decimal_precision() - dec;
-  uint32 len= item->max_length;
+  uint32 len= item->max_char_length();
 
   DBUG_ASSERT (item->result_type() == DECIMAL_RESULT);
 
@@ -2850,6 +2852,7 @@ String *Field_new_decimal::val_str(String *val_buffer,
   uint fixed_precision= zerofill ? precision : 0;
   my_decimal2string(E_DEC_FATAL_ERROR, val_decimal(&decimal_value),
                     fixed_precision, dec, '0', val_buffer);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -3121,7 +3124,7 @@ String *Field_tiny::val_str(String *val_buffer,
 			    String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length;
   uint mlength=max(field_length+1,5*cs->mbmaxlen);
   val_buffer->alloc(mlength);
@@ -3137,6 +3140,7 @@ String *Field_tiny::val_str(String *val_buffer,
   val_buffer->length(length);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(cs);
   return val_buffer;
 }
 
@@ -3333,7 +3337,7 @@ String *Field_short::val_str(String *val_buffer,
 			     String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length;
   uint mlength=max(field_length+1,7*cs->mbmaxlen);
   val_buffer->alloc(mlength);
@@ -3354,6 +3358,7 @@ String *Field_short::val_str(String *val_buffer,
   val_buffer->length(length);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(cs);
   return val_buffer;
 }
 
@@ -3550,7 +3555,7 @@ String *Field_medium::val_str(String *val_buffer,
 			      String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length;
   uint mlength=max(field_length+1,10*cs->mbmaxlen);
   val_buffer->alloc(mlength);
@@ -3561,6 +3566,7 @@ String *Field_medium::val_str(String *val_buffer,
   val_buffer->length(length);
   if (zerofill)
     prepend_zeros(val_buffer); /* purecov: inspected */
+  val_buffer->set_charset(cs);
   return val_buffer;
 }
 
@@ -3769,7 +3775,7 @@ String *Field_long::val_str(String *val_buffer,
 			    String *val_ptr __attribute__((unused)))
 {
   ASSERT_COLUMN_MARKED_FOR_READ;
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length;
   uint mlength=max(field_length+1,12*cs->mbmaxlen);
   val_buffer->alloc(mlength);
@@ -3789,6 +3795,7 @@ String *Field_long::val_str(String *val_buffer,
   val_buffer->length(length);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(cs);
   return val_buffer;
 }
 
@@ -4010,7 +4017,7 @@ longlong Field_longlong::val_int(void)
 String *Field_longlong::val_str(String *val_buffer,
 				String *val_ptr __attribute__((unused)))
 {
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length;
   uint mlength=max(field_length+1,22*cs->mbmaxlen);
   val_buffer->alloc(mlength);
@@ -4028,6 +4035,7 @@ String *Field_longlong::val_str(String *val_buffer,
   val_buffer->length(length);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(cs);
   return val_buffer;
 }
 
@@ -4254,6 +4262,7 @@ String *Field_float::val_str(String *val_buffer,
   val_buffer->length((uint) len);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -4564,6 +4573,7 @@ String *Field_double::val_str(String *val_buffer,
   val_buffer->length((uint) len);
   if (zerofill)
     prepend_zeros(val_buffer);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -4698,7 +4708,7 @@ Field_timestamp::Field_timestamp(uchar *ptr_arg, uint32 len_arg,
 	     unireg_check_arg, field_name_arg, cs)
 {
   /* For 4.0 MYD and 4.0 InnoDB compatibility */
-  flags|= ZEROFILL_FLAG | UNSIGNED_FLAG;
+  flags|= ZEROFILL_FLAG | UNSIGNED_FLAG | BINARY_FLAG;
   if (!share->timestamp_field && unireg_check != NONE)
   {
     /* This timestamp has auto-update */
@@ -4718,7 +4728,7 @@ Field_timestamp::Field_timestamp(bool maybe_null_arg,
 	     NONE, field_name_arg, cs)
 {
   /* For 4.0 MYD and 4.0 InnoDB compatibility */
-  flags|= ZEROFILL_FLAG | UNSIGNED_FLAG;
+  flags|= ZEROFILL_FLAG | UNSIGNED_FLAG | BINARY_FLAG;
     if (unireg_check != TIMESTAMP_DN_FIELD)
       flags|= ON_UPDATE_NOW_FLAG;
 }
@@ -4770,7 +4780,7 @@ int Field_timestamp::store(const char *from,uint len,CHARSET_INFO *cs)
   THD *thd= table ? table->in_use : current_thd;
 
   /* We don't want to store invalid or fuzzy datetime values in TIMESTAMP */
-  have_smth_to_conv= (str_to_datetime(from, len, &l_time,
+  have_smth_to_conv= (str_to_datetime(cs, from, len, &l_time,
                                       (thd->variables.sql_mode &
                                        MODE_NO_ZERO_DATE) |
                                       MODE_NO_ZERO_IN_DATE, &error) >
@@ -4919,10 +4929,10 @@ String *Field_timestamp::val_str(String *val_buffer, String *val_ptr)
 
   if (temp == 0L)
   {				      /* Zero time is "000000" */
-    val_ptr->set(STRING_WITH_LEN("0000-00-00 00:00:00"), &my_charset_bin);
+    val_ptr->set(STRING_WITH_LEN("0000-00-00 00:00:00"), &my_charset_numeric);
     return val_ptr;
   }
-  val_buffer->set_charset(&my_charset_bin);	// Safety
+  val_buffer->set_charset(&my_charset_numeric);	// Safety
   
   thd->variables.time_zone->gmt_sec_to_TIME(&time_tmp,(my_time_t)temp);
 
@@ -4966,6 +4976,7 @@ String *Field_timestamp::val_str(String *val_buffer, String *val_ptr)
   *to++= (char) ('0'+(char) (temp2));
   *to++= (char) ('0'+(char) (temp));
   *to= 0;
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -5076,7 +5087,7 @@ int Field_time::store(const char *from,uint len,CHARSET_INFO *cs)
   int error= 0;
   int warning;
 
-  if (str_to_time(from, len, &ltime, &warning))
+  if (str_to_time(cs, from, len, &ltime, &warning))
   {
     tmp=0L;
     error= 2;
@@ -5235,6 +5246,7 @@ String *Field_time::val_str(String *val_buffer,
   ltime.minute= (uint) (tmp/100 % 100);
   ltime.second= (uint) (tmp % 100);
   make_time((DATE_TIME_FORMAT*) 0, &ltime, val_buffer);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -5432,6 +5444,7 @@ String *Field_year::val_str(String *val_buffer,
   val_buffer->length(field_length);
   char *to=(char*) val_buffer->ptr();
   sprintf(to,field_length == 2 ? "%02d" : "%04d",(int) Field_year::val_int());
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -5459,7 +5472,7 @@ int Field_date::store(const char *from, uint len,CHARSET_INFO *cs)
   int error;
   THD *thd= table ? table->in_use : current_thd;
 
-  if (str_to_datetime(from, len, &l_time, TIME_FUZZY_DATE |
+  if (str_to_datetime(cs, from, len, &l_time, TIME_FUZZY_DATE |
                       (thd->variables.sql_mode &
                        (MODE_NO_ZERO_IN_DATE | MODE_NO_ZERO_DATE |
                         MODE_INVALID_DATES)),
@@ -5606,6 +5619,7 @@ String *Field_date::val_str(String *val_buffer,
   ltime.month= (int) ((uint32) tmp/100 % 100);
   ltime.day= (int) ((uint32) tmp % 100);
   make_date((DATE_TIME_FORMAT *) 0, &ltime, val_buffer);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -5695,7 +5709,7 @@ int Field_newdate::store(const char *from,uint len,CHARSET_INFO *cs)
   int error;
   THD *thd= table ? table->in_use : current_thd;
   enum enum_mysql_timestamp_type ret;
-  if ((ret= str_to_datetime(from, len, &l_time,
+  if ((ret= str_to_datetime(cs, from, len, &l_time,
                             (TIME_FUZZY_DATE |
                              (thd->variables.sql_mode &
                               (MODE_NO_ZERO_IN_DATE | MODE_NO_ZERO_DATE |
@@ -5867,6 +5881,7 @@ String *Field_newdate::val_str(String *val_buffer,
   *pos--= (char) ('0'+part%10); part/=10;
   *pos--= (char) ('0'+part%10); part/=10;
   *pos=   (char) ('0'+part);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -5929,7 +5944,7 @@ int Field_datetime::store(const char *from,uint len,CHARSET_INFO *cs)
   enum enum_mysql_timestamp_type func_res;
   THD *thd= table ? table->in_use : current_thd;
 
-  func_res= str_to_datetime(from, len, &time_tmp,
+  func_res= str_to_datetime(cs, from, len, &time_tmp,
                             (TIME_FUZZY_DATE |
                              (thd->variables.sql_mode &
                               (MODE_NO_ZERO_IN_DATE | MODE_NO_ZERO_DATE |
@@ -6129,6 +6144,7 @@ String *Field_datetime::val_str(String *val_buffer,
   *pos--= (char) ('0'+(char) (part3%10)); part3/=10;
   *pos--= (char) ('0'+(char) (part3%10)); part3/=10;
   *pos=(char) ('0'+(char) part3);
+  val_buffer->set_charset(&my_charset_numeric);
   return val_buffer;
 }
 
@@ -6370,7 +6386,7 @@ int Field_str::store(double nr)
     else
       set_warning(MYSQL_ERROR::WARN_LEVEL_WARN, WARN_DATA_TRUNCATED, 1);
   }
-  return store(buff, length, charset());
+  return store(buff, length, &my_charset_numeric);
 }
 
 
@@ -6405,7 +6421,7 @@ int Field_string::store(longlong nr, bool unsigned_val)
 int Field_longstr::store_decimal(const my_decimal *d)
 {
   char buff[DECIMAL_MAX_STR_LENGTH+1];
-  String str(buff, sizeof(buff), &my_charset_bin);
+  String str(buff, sizeof(buff), &my_charset_numeric);
   my_decimal2string(E_DEC_FATAL_ERROR, d, 0, 0, 0, &str);
   return store(str.ptr(), str.length(), str.charset());
 }
@@ -9565,8 +9581,8 @@ bool Create_field::init(THD *thd, char *fld_name, enum_field_types fld_type,
   case MYSQL_TYPE_TIME:
   case MYSQL_TYPE_DATETIME:
   case MYSQL_TYPE_TIMESTAMP:
-    charset= &my_charset_bin;
-    flags|= BINCMP_FLAG;
+    charset= &my_charset_numeric;
+    flags|= BINARY_FLAG;
   default: break;
   }
 
@@ -9682,7 +9698,7 @@ Field *make_field(TABLE_SHARE *share, uchar *ptr, uint32 field_length,
   case MYSQL_TYPE_TIME:
   case MYSQL_TYPE_DATETIME:
   case MYSQL_TYPE_TIMESTAMP:
-    field_charset= &my_charset_bin;
+    field_charset= &my_charset_numeric;
   default: break;
   }