Bug#10504

  Character set does not support traditional mode
ctype_utf8.result, ctype_utf8.test:
  adding test case.
password.c, mysql_com.h
  Changeing octet2hex availability from static to public.
item_strfunc.cc:
  Result string is now checked to be well-formed.
  Warning/error is generated, depending on sql_mode.


include/mysql_com.h:
  Bug#10504
  Character set does not support traditional mode
  Changeing octet2hex from static to public.
sql/item_strfunc.cc:
  Result string is now checked to be well-formed.
  Warning/error is generated, depending on sql_mode.
sql/password.c:
  Changeing octet2hex from static to public.
mysql-test/t/ctype_utf8.test:
  adding test case.
mysql-test/r/ctype_utf8.result:
  adding test case.

2 files changed, 29 insertions, 2 deletions

diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index 4fd33c06095..309e6dcdcd2 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -1980,6 +1980,33 @@ b1:        str->append((char)(num>>8));
   }
   str->set_charset(collation.collation);
   str->realloc(str->length());			// Add end 0 (for Purify)
+
+  /* Check whether we got a well-formed string */
+  CHARSET_INFO *cs= collation.collation;
+  int well_formed_error;
+  uint wlen= cs->cset->well_formed_len(cs,
+                                       str->ptr(), str->ptr() + str->length(),
+                                       str->length(), &well_formed_error);
+  if (wlen < str->length())
+  {
+    THD *thd= current_thd;
+    char hexbuf[7];
+    enum MYSQL_ERROR::enum_warning_level level;
+    uint diff= str->length() - wlen;
+    set_if_smaller(diff, 3);
+    octet2hex(hexbuf, (const uchar*) str->ptr() + wlen, diff);
+    if (thd->variables.sql_mode &
+        (MODE_STRICT_TRANS_TABLES | MODE_STRICT_ALL_TABLES))
+    {
+      level= MYSQL_ERROR::WARN_LEVEL_ERROR;
+      null_value= 1;
+      str= 0;
+    }
+    else
+      level= MYSQL_ERROR::WARN_LEVEL_WARN;
+    push_warning_printf(thd, level, ER_INVALID_CHARACTER_STRING,
+                        ER(ER_INVALID_CHARACTER_STRING), cs->csname, hexbuf);
+  }
   return str;
 }
 
diff --git a/sql/password.c b/sql/password.c
index 60cc0ac0c97..aa05be8c740 100644
--- a/sql/password.c
+++ b/sql/password.c
@@ -318,8 +318,8 @@ void create_random_string(char *to, uint length, struct rand_struct *rand_st)
     str, len  IN  the beginning and the length of the input string
 */
 
-static void
-octet2hex(char *to, const uint8 *str, uint len)
+void
+octet2hex(char *to, const unsigned char *str, uint len)
 {
   const uint8 *str_end= str + len; 
   for (; str != str_end; ++str)