WL#2649 Number-to-string conversions

added:
  include/ctype_numconv.inc
  mysql-test/include/ctype_numconv.inc
  mysql-test/r/ctype_binary.result
  mysql-test/t/ctype_binary.test
  Adding tests

modified:

  mysql-test/r/bigint.result
  mysql-test/r/case.result
  mysql-test/r/create.result
  mysql-test/r/ctype_cp1251.result
  mysql-test/r/ctype_latin1.result
  mysql-test/r/ctype_ucs.result
  mysql-test/r/func_gconcat.result
  mysql-test/r/func_str.result
  mysql-test/r/metadata.result
  mysql-test/r/ps_1general.result
  mysql-test/r/ps_2myisam.result
  mysql-test/r/ps_3innodb.result
  mysql-test/r/ps_4heap.result
  mysql-test/r/ps_5merge.result
  mysql-test/r/show_check.result
  mysql-test/r/type_datetime.result
  mysql-test/r/type_ranges.result
  mysql-test/r/union.result
  mysql-test/suite/ndb/r/ps_7ndb.result
  mysql-test/t/ctype_cp1251.test
  mysql-test/t/ctype_latin1.test
  mysql-test/t/ctype_ucs.test
  mysql-test/t/func_str.test
    Fixing tests


  @ sql/field.cc
     - Return str result using my_charset_numeric.
     - Using real multi-byte aware str_to_XXX functions
       to handle tricky charset values propely (e.g. UCS2)
  @ sql/field.h
     - Changing derivation of non-string field types to DERIVATION_NUMERIC.
     - Changing binary() for numeric/datetime fields to always
     return TRUE even if charset is not my_charset_bin. We need
     this to keep ha_base_keytype() return HA_KEYTYPE_BINARY.
     - Adding BINARY_FLAG into some fields, because it's not
     being set automatically anymore with
     "my_charset_bin to my_charset_numeric" change.
    - Changing derivation for numeric/datetime datatypes to a weaker
      value, to make "SELECT concat('string', field)" use character
      set of the string literal for the result of the function.
  @ sql/item.cc
     - Implementing generic val_str_ascii().
     - Using max_char_length() instead of direct read of max_length
       to make "tricky" charsets like UCS2 work.
       NOTE: in the future we'll possibly remove all direct reads of max_length
     - Fixing Item_num::safe_charset_converter().
       Previously it alligned binary string to
       character string (for example by adding leading 0x00
       when doing binary->UCS2 conversion). Now it just
       converts from my_charset_numbner to "tocs".
     - Using val_str_ascii() in Item::get_time() to make UCS2 arguments work.
     - Other misc changes
  @ sql/item.h
     - Changing MY_COLL_CMP_CONV and MY_COLL_ALLOW_CONV to
       bit operations instead of hard-coded bit masks.
     - Addding new method DTCollation.set_numeric().
     - Adding new methods to Item.
     - Adding helper functions to make code look nicer:
       agg_item_charsets_for_string_result()
       agg_item_charsets_for_comparison()
     - Changing charset for Item_num-derived items
       from my_charset_bin to my_charset_numeric
       (which is an alias for latin1).
  @ sql/item_cmpfunc.cc
     - Using new helper functions
     - Other misc changes
  @ sql/item_cmpfunc.h
     - Fixing strcmp() to return max_length=2.
       Previously it returned 1, which was wrong,
       because it did not fit '-1'.
  @ sql/item_func.cc
     - Using new helper functions
     - Other minor changes
  @ sql/item_func.h
     - Removing unused functions
     - Adding helper functions
       agg_arg_charsets_for_string_result()
       agg_arg_charsets_for_comparison()
     - Adding set_numeric() into constructors of numeric items.
     - Using fix_length_and_charset() and fix_char_length()
       instead of direct write to max_length.
  @ sql/item_geofunc.cc
     - Changing class for Item_func_geometry_type and
       Item_func_as_wkt from Item_str_func to
       Item_str_ascii_func, to make them return UCS2 result
       properly (when character_set_connection=ucs2).
  @ sql/item_geofunc.h
     - Changing class for Item_func_geometry_type and
       Item_func_as_wkt from Item_str_func to
       Item_str_ascii_func, to make them return UCS2 result
       properly (when @@character_set_connection=ucs2).
  @ sql/item_strfunc.cc
     - Implementing Item_str_func::val_str().
     - Renaming val_str to val_str_ascii for some items,
       to make them work with UCS2 properly.
     - Using new helper functions
     - All single-argument functions that expect string
       result now call this method:
       agg_arg_charsets_for_string_result(collation, args, 1);
       This enables character set conversion to @@character_set_connection
       in case of pure numeric input.
  @ sql/item_strfunc.h
     - Introducing Item_str_ascii_func - for functions
       which return pure ASCII data, for performance purposes,
       as well as for the cases when the old implementation
       of val_str() was heavily 8-bit oriented and implementing
       a UCS2-aware version is tricky.
  @ sql/item_sum.cc
     - Using new helper functions.
  @ sql/item_timefunc.cc
     - Using my_charset_numeric instead of my_charset_bin.
     - Using fix_char_length(), fix_length_and_charset()
       and fix_length_and_charset_datetime()
       instead of direct write to max_length.
     - Using tricky-charset aware function str_to_time_with_warn()
  @ sql/item_timefunc.h
     - Using new helper functions for charset and length initialization.
     - Changing base class for Item_func_get_format() to make
       it return UCS2 properly (when character_set_connection=ucs2).
  @ sql/item_xmlfunc.cc
     - Using new helper function
  @ sql/my_decimal.cc
     - Adding a new DECIMAL to CHAR converter
       with real multibyte support (e.g. UCS2)

  @ sql/mysql_priv.h
     - Introducing a new derivation level for numeric/datetime data types.
     - Adding macros for my_charset_numeric and MY_REPERTOIRE_NUMERIC.
     - Adding prototypes for str_set_decimal()
     - Adding prototypes for character-set aware str_to_xxx() functions.
  @ sql/protocol.cc
     - Changing charsetnr to "binary" client-side metadata for
       numeric/datetime data types.
  @ sql/time.cc
     - Adding to_ascii() helper function, to convert a string
       in any character set to ascii representation. In the
       future can be extended to understand digits written
       in various non-Latin word scripts.
     - Adding real multy-byte character set aware versions for str_to_XXXX,
       to make these these type of queries work correct:
         INSERT INTO t1 SET datetime_column=ucs2_expression;
   @  strings/ctype-ucs2.c
     - endptr was not calculated correctly. INSERTing of UCS2
       values into numeric columns returned warnings about
       truncated wrong data.

1 files changed, 22 insertions, 27 deletions

diff --git a/sql/item_timefunc.cc b/sql/item_timefunc.cc
index 392bc936f17..49d31bf1252 100644
--- a/sql/item_timefunc.cc
+++ b/sql/item_timefunc.cc
@@ -57,7 +57,7 @@ static bool make_datetime(date_time_format_types format, MYSQL_TIME *ltime,
 			  String *str)
 {
   char *buff;
-  CHARSET_INFO *cs= &my_charset_bin;
+  CHARSET_INFO *cs= &my_charset_numeric;
   uint length= MAX_DATE_STRING_REP_LENGTH;
 
   if (str->alloc(length))
@@ -1586,9 +1586,7 @@ bool Item_func_from_days::get_date(MYSQL_TIME *ltime, uint fuzzy_date)
 
 void Item_func_curdate::fix_length_and_dec()
 {
-  collation.set(&my_charset_bin);
-  decimals=0; 
-  max_length=MAX_DATE_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  Item_date::fix_length_and_dec();
 
   store_now_in_TIME(&ltime);
   
@@ -1649,7 +1647,7 @@ bool Item_func_curdate::get_date(MYSQL_TIME *res,
 String *Item_func_curtime::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
-  str_value.set(buff, buff_length, &my_charset_bin);
+  str_value.set(buff, buff_length, &my_charset_latin1);
   return &str_value;
 }
 
@@ -1659,11 +1657,10 @@ void Item_func_curtime::fix_length_and_dec()
   MYSQL_TIME ltime;
 
   decimals= DATETIME_DEC;
-  collation.set(&my_charset_bin);
   store_now_in_TIME(&ltime);
   value= TIME_to_ulonglong_time(&ltime);
   buff_length= (uint) my_time_to_str(&ltime, buff);
-  max_length= buff_length;
+  fix_length_and_charset_datetime(buff_length);
 }
 
 
@@ -1698,7 +1695,7 @@ void Item_func_curtime_utc::store_now_in_TIME(MYSQL_TIME *now_time)
 String *Item_func_now::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
-  str_value.set(buff,buff_length, &my_charset_bin);
+  str_value.set(buff, buff_length, &my_charset_numeric);
   return &str_value;
 }
 
@@ -1706,13 +1703,12 @@ String *Item_func_now::val_str(String *str)
 void Item_func_now::fix_length_and_dec()
 {
   decimals= DATETIME_DEC;
-  collation.set(&my_charset_bin);
 
   store_now_in_TIME(&ltime);
   value= (longlong) TIME_to_ulonglong_datetime(&ltime);
 
   buff_length= (uint) my_datetime_to_str(&ltime, buff);
-  max_length= buff_length;
+  fix_length_and_charset_datetime(buff_length);
 }
 
 
@@ -1776,7 +1772,7 @@ String *Item_func_sysdate_local::val_str(String *str)
   DBUG_ASSERT(fixed == 1);
   store_now_in_TIME(&ltime);
   buff_length= (uint) my_datetime_to_str(&ltime, buff);
-  str_value.set(buff, buff_length, &my_charset_bin);
+  str_value.set(buff, buff_length, &my_charset_numeric);
   return &str_value;
 }
 
@@ -1800,8 +1796,7 @@ double Item_func_sysdate_local::val_real()
 void Item_func_sysdate_local::fix_length_and_dec()
 {
   decimals= 0;
-  collation.set(&my_charset_bin);
-  max_length= MAX_DATETIME_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  fix_length_and_charset_datetime(MAX_DATETIME_WIDTH);
 }
 
 
@@ -2005,7 +2000,8 @@ String *Item_func_date_format::val_str(String *str)
   {
     String *res;
     if (!(res=args[0]->val_str(str)) ||
-	(str_to_time_with_warn(res->ptr(), res->length(), &l_time)))
+	(str_to_time_with_warn(res->charset(), res->ptr(), res->length(),
+	                       &l_time)))
       goto null_date;
 
     l_time.year=l_time.month=l_time.day=0;
@@ -2049,9 +2045,8 @@ null_date:
 void Item_func_from_unixtime::fix_length_and_dec()
 { 
   thd= current_thd;
-  collation.set(&my_charset_bin);
   decimals= DATETIME_DEC;
-  max_length=MAX_DATETIME_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  fix_length_and_charset_datetime(MAX_DATETIME_WIDTH);
   maybe_null= 1;
   thd->time_zone_used= 1;
 }
@@ -2109,9 +2104,8 @@ bool Item_func_from_unixtime::get_date(MYSQL_TIME *ltime,
 
 void Item_func_convert_tz::fix_length_and_dec()
 {
-  collation.set(&my_charset_bin);
   decimals= 0;
-  max_length= MAX_DATETIME_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  fix_length_and_charset_datetime(MAX_DATETIME_WIDTH);
   maybe_null= 1;
 }
 
@@ -2155,13 +2149,13 @@ bool Item_func_convert_tz::get_date(MYSQL_TIME *ltime,
 
   if (!from_tz_cached)
   {
-    from_tz= my_tz_find(thd, args[1]->val_str(&str));
+    from_tz= my_tz_find(thd, args[1]->val_str_ascii(&str));
     from_tz_cached= args[1]->const_item();
   }
 
   if (!to_tz_cached)
   {
-    to_tz= my_tz_find(thd, args[2]->val_str(&str));
+    to_tz= my_tz_find(thd, args[2]->val_str_ascii(&str));
     to_tz_cached= args[2]->const_item();
   }
 
@@ -2195,9 +2189,8 @@ void Item_date_add_interval::fix_length_and_dec()
 {
   enum_field_types arg0_field_type;
 
-  collation.set(&my_charset_bin);
   maybe_null=1;
-  max_length=MAX_DATETIME_FULL_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  fix_length_and_charset_datetime(MAX_DATETIME_FULL_WIDTH);
   value.alloc(max_length);
 
   /*
@@ -2375,7 +2368,9 @@ longlong Item_extract::val_int()
   else
   {
     String *res= args[0]->val_str(&value);
-    if (!res || str_to_time_with_warn(res->ptr(), res->length(), &ltime))
+    if (!res ||
+        str_to_time_with_warn(res->charset(), res->ptr(), res->length(),
+                              &ltime))
     {
       null_value=1;
       return 0;
@@ -2814,7 +2809,7 @@ void Item_func_add_time::fix_length_and_dec()
 {
   enum_field_types arg0_field_type;
   decimals=0;
-  max_length=MAX_DATETIME_FULL_WIDTH*MY_CHARSET_BIN_MB_MAXLEN;
+  fix_length_and_charset_datetime(MAX_DATETIME_FULL_WIDTH);
   maybe_null= 1;
 
   /*
@@ -3225,12 +3220,12 @@ void Item_func_timestamp_diff::print(String *str, enum_query_type query_type)
 }
 
 
-String *Item_func_get_format::val_str(String *str)
+String *Item_func_get_format::val_str_ascii(String *str)
 {
   DBUG_ASSERT(fixed == 1);
   const char *format_name;
   KNOWN_DATE_TIME_FORMAT *format;
-  String *val= args[0]->val_str(str);
+  String *val= args[0]->val_str_ascii(str);
   ulong val_len;
 
   if ((null_value= args[0]->null_value))
@@ -3249,7 +3244,7 @@ String *Item_func_get_format::val_str(String *str)
 		      (const uchar *) format_name, val_len))
     {
       const char *format_str= get_date_time_format_str(format, type);
-      str->set(format_str, (uint) strlen(format_str), &my_charset_bin);
+      str->set(format_str, (uint) strlen(format_str), &my_charset_numeric);
       return str;
     }
   }