Bug#20471 LIKE search fails with indexed utf8 char column

The main problem was already fixed by Igor under terms of 16674.
Adding some additional minor fixes and tests.


include/m_ctype.h:
  Adding reference to CHARSET_INFO.txt
mysql-test/r/ctype_utf8.result:
  Adding test case
mysql-test/t/ctype_utf8.test:
  Adding test case
strings/CHARSET_INFO.txt:
  Adding comment about max_sort_char
strings/ctype-mb.c:
  Restiring that non-Unicode character sets use 0xFF as pad character
  for max_str. Only Unicode character sets use wc_mb.
strings/ctype-utf8.c:
  Fixed that max_sort_char for UTF8 from U+00FF to U+FFFF.

3 files changed, 28 insertions, 7 deletions

diff --git a/strings/CHARSET_INFO.txt b/strings/CHARSET_INFO.txt
index f7a10f95880..3fd262c6f12 100644
--- a/strings/CHARSET_INFO.txt
+++ b/strings/CHARSET_INFO.txt
@@ -33,7 +33,7 @@ typedef struct charset_info_st
   uint      strxfrm_multiply;
   uint      mbminlen;
   uint      mbmaxlen;
-  char      max_sort_char; /* For LIKE optimization */
+  uint16    max_sort_char; /* For LIKE optimization */
 
   MY_CHARSET_HANDLER *cset;
   MY_COLLATION_HANDLER *coll;
@@ -134,7 +134,15 @@ Misc fields
   mbmaxlen         - maximum multibyte sequence length.
                      1 for 8bit charsets. Can be also 2 or 3.
 
-
+  max_sort_char    - for LIKE range
+                     in case of 8bit character sets - native code
+		     of maximum character (max_str pad byte);
+                     in case of UTF8 and UCS2 - Unicode code of the maximum
+		     possible character (usually U+FFFF). This code is
+		     converted to multibyte representation (usually 0xEFBFBF)
+		     and then used as a pad sequence for max_str.
+		     in case of other multibyte character sets -
+		     max_str pad byte (usually 0xFF).
 
 MY_CHARSET_HANDLER
 ==================
diff --git a/strings/ctype-mb.c b/strings/ctype-mb.c
index 4f57f7c78e4..6f63592c459 100644
--- a/strings/ctype-mb.c
+++ b/strings/ctype-mb.c
@@ -449,15 +449,28 @@ static void my_hash_sort_mb_bin(CHARSET_INFO *cs __attribute__((unused)),
 
 
 /* 
-  Write max key: create a buffer with multibyte
+  Write max key:
+- for non-Unicode character sets:
+  just set to 255.
+- for Unicode character set (utf-8):
+  create a buffer with multibyte
   representation of the max_sort_char character,
   and copy it into max_str in a loop. 
 */
 static void pad_max_char(CHARSET_INFO *cs, char *str, char *end)
 {
   char buf[10];
-  char buflen= cs->cset->wc_mb(cs, cs->max_sort_char, (uchar*) buf,
-                               (uchar*) buf + sizeof(buf));
+  char buflen;
+  
+  if (!(cs->state & MY_CS_UNICODE))
+  {
+    bfill(str, end - str, 255);
+    return;
+  }
+  
+  buflen= cs->cset->wc_mb(cs, cs->max_sort_char, (uchar*) buf,
+                          (uchar*) buf + sizeof(buf));
+  
   DBUG_ASSERT(buflen > 0);
   do
   {
@@ -894,7 +907,7 @@ MY_COLLATION_HANDLER my_collation_mb_bin_handler =
     my_strnncoll_mb_bin,
     my_strnncollsp_mb_bin,
     my_strnxfrm_mb_bin,
-    my_like_range_simple,
+    my_like_range_mb,
     my_wildcmp_mb_bin,
     my_strcasecmp_mb_bin,
     my_instr_mb,
diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c
index 94e8e6ba797..79239394816 100644
--- a/strings/ctype-utf8.c
+++ b/strings/ctype-utf8.c
@@ -2373,7 +2373,7 @@ CHARSET_INFO my_charset_utf8_bin=
     1,                  /* mbminlen     */
     3,                  /* mbmaxlen     */
     0,                  /* min_sort_char */
-    255,                /* max_sort_char */
+    0xFFFF,             /* max_sort_char */
     0,                  /* escape_with_backslash_is_dangerous */
     &my_charset_utf8_handler,
     &my_collation_mb_bin_handler