Bug#28875 Conversion between ASCII and LATIN1 charsets does not function

(Regression, caused by a patch for the bug 22646).
Problem: when result type of date_format() was changed from
binary string to character string, mixing date_format()
with a ascii column in CONCAT() stopped to work.
Fix:
- adding "repertoire" flag into DTCollation class,
to mark items which can return only pure ASCII strings.
- allow character set conversion from pure ASCII to other character sets.


include/m_ctype.h:
  Defining new flags.
  Adding new function prototypes.
mysql-test/r/ctype_ucs.result:
  Adding tests.
mysql-test/r/ctype_utf8.result:
  Adding tests.
mysql-test/r/func_time.result:
  Adding tests.
mysql-test/t/ctype_ucs.test:
  Adding tests.
mysql-test/t/ctype_utf8.test:
  Adding tests.
mysql-test/t/func_time.test:
  Adding test.
mysys/charset.c:
  Adding pure ASCII detection when loading a dynamic character set.
sql/item.cc:
  - Moving detection of a Unicode superset into function.
  - Adding detection of a ASCII subset.
  - Adding creation of to-ASCII character set convertor when
    safe_charset_converter() failed and when the argument.
    repertoire is know to be pure ASCII.
sql/item.h:
  - Adding "repertoire" member into DTCollation class.
  - Adding "repertoire" argument to constructors.
  - Adding new methods:
    set_repertoire_from_charset()
    set_repertoire_from_value()
sql/item_func.cc:
  Adding "repertoire" argument.
sql/item_strfunc.cc:
  Adding "repertoire" argument.
sql/item_timefunc.cc:
  Initializing the result repertoire taking into account the "is_ascii"
  flag of the current locale.
sql/sql_lex.cc:
  Detect 7bit strings, return in Lex->text_string_is_7bit.
sql/sql_lex.h:
  Adding new member into LEX structure.
  Adding new member into Lex_input_stream
sql/sql_string.cc:
  Allow simple copy from pure ASCII to a ASCII-based character set.
sql/sql_yacc.yy:
  Depening on Lex->text_string_is_7bit and character set features,
  create Item_string with MY_REPERTOIRE_ASCII when it is possible.
strings/conf_to_src.c:
  - Adding printing of the "MY_CS_PUREASCII" flag
  - Adding printing of copyright
strings/ctype-extra.c:
  Recreating ctype-extra.c: ascii_general_ci and ascii_bin
  are now marked with MY_CS_PUREASCII flag.
strings/ctype.c:
  Adding new functions.

1 files changed, 36 insertions, 7 deletions

diff --git a/sql/item.h b/sql/item.h
index 11dce3a7758..9a45314651a 100644
--- a/sql/item.h
+++ b/sql/item.h
@@ -49,29 +49,50 @@ class DTCollation {
 public:
   CHARSET_INFO     *collation;
   enum Derivation derivation;
+  uint repertoire;
   
+  void set_repertoire_from_charset(CHARSET_INFO *cs)
+  {
+    repertoire= cs->state & MY_CS_PUREASCII ?
+                MY_REPERTOIRE_ASCII : MY_REPERTOIRE_UNICODE30;
+  }
   DTCollation()
   {
     collation= &my_charset_bin;
     derivation= DERIVATION_NONE;
+    repertoire= MY_REPERTOIRE_UNICODE30;
   }
   DTCollation(CHARSET_INFO *collation_arg, Derivation derivation_arg)
   {
     collation= collation_arg;
     derivation= derivation_arg;
+    set_repertoire_from_charset(collation_arg);
   }
   void set(DTCollation &dt)
   { 
     collation= dt.collation;
     derivation= dt.derivation;
+    repertoire= dt.repertoire;
   }
   void set(CHARSET_INFO *collation_arg, Derivation derivation_arg)
   {
     collation= collation_arg;
     derivation= derivation_arg;
+    set_repertoire_from_charset(collation_arg);
+  }
+  void set(CHARSET_INFO *collation_arg,
+           Derivation derivation_arg,
+           uint repertoire_arg)
+  {
+    collation= collation_arg;
+    derivation= derivation_arg;
+    repertoire= repertoire_arg;
   }
   void set(CHARSET_INFO *collation_arg)
-  { collation= collation_arg; }
+  {
+    collation= collation_arg;
+    set_repertoire_from_charset(collation_arg);
+  }
   void set(Derivation derivation_arg)
   { derivation= derivation_arg; }
   bool aggregate(DTCollation &dt, uint flags= 0);
@@ -1650,10 +1671,11 @@ class Item_string :public Item
 {
 public:
   Item_string(const char *str,uint length,
-  	      CHARSET_INFO *cs, Derivation dv= DERIVATION_COERCIBLE)
+              CHARSET_INFO *cs, Derivation dv= DERIVATION_COERCIBLE,
+              uint repertoire= MY_REPERTOIRE_UNICODE30)
   {
-    collation.set(cs, dv);
-    str_value.set_or_copy_aligned(str,length,cs);
+    str_value.set_or_copy_aligned(str, length, cs);
+    collation.set(cs, dv, repertoire);
     /*
       We have to have a different max_length than 'length' here to
       ensure that we get the right length if we do use the item
@@ -1677,10 +1699,11 @@ public:
     fixed= 1;
   }
   Item_string(const char *name_par, const char *str, uint length,
-	      CHARSET_INFO *cs, Derivation dv= DERIVATION_COERCIBLE)
+              CHARSET_INFO *cs, Derivation dv= DERIVATION_COERCIBLE,
+              uint repertoire= MY_REPERTOIRE_UNICODE30)
   {
-    collation.set(cs, dv);
-    str_value.set_or_copy_aligned(str,length,cs);
+    str_value.set_or_copy_aligned(str, length, cs);
+    collation.set(cs, dv, repertoire);
     max_length= str_value.numchars()*cs->mbmaxlen;
     set_name(name_par, 0, cs);
     decimals=NOT_FIXED_DEC;
@@ -1696,6 +1719,12 @@ public:
     str_value.copy(str_arg, length_arg, collation.collation);
     max_length= str_value.numchars() * collation.collation->mbmaxlen;
   }
+  void set_repertoire_from_value()
+  {
+    collation.repertoire= my_string_repertoire(str_value.charset(),
+                                               str_value.ptr(),
+                                               str_value.length());
+  }
   enum Type type() const { return STRING_ITEM; }
   double val_real();
   longlong val_int();