10 files changed, 80 insertions, 5 deletions
diff --git a/mysql-test/r/ctype_ucs.result b/mysql-test/r/ctype_ucs.result
index 97617f6c3df..56174311bf3 100644
--- a/mysql-test/r/ctype_ucs.result
+++ b/mysql-test/r/ctype_ucs.result
@@ -5640,5 +5640,14 @@ Warnings:
 Note	1003	select `test`.`t1`.`a` AS `a`,`test`.`t1`.`b` AS `b`,`test`.`t1`.`c` AS `c`,`test`.`t1`.`d` AS `d` from `test`.`t1` where ((coalesce(`test`.`t1`.`c`,0) = '3 ') and (coalesce(`test`.`t1`.`d`,0) = '3 '))
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED);
+CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1Ĳ3'
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/r/ctype_utf16.result b/mysql-test/r/ctype_utf16.result
index 77941ab4906..3bd37255b59 100644
--- a/mysql-test/r/ctype_utf16.result
+++ b/mysql-test/r/ctype_utf16.result
@@ -2190,5 +2190,14 @@ COUNT(DISTINCT a)
 7
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING utf16) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf16) AS SIGNED);
+CAST(CONVERT('1Ĳ3' USING utf16) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1Ĳ3'
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/r/ctype_utf16le.result b/mysql-test/r/ctype_utf16le.result
index a2d3ffb793f..93fce843dc8 100644
--- a/mysql-test/r/ctype_utf16le.result
+++ b/mysql-test/r/ctype_utf16le.result
@@ -2382,5 +2382,14 @@ COUNT(DISTINCT a)
 7
 DROP TABLE t1;
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf16le) AS SIGNED);
+CAST(CONVERT('1Ĳ3' USING utf16le) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1Ĳ3'
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/r/ctype_utf32.result b/mysql-test/r/ctype_utf32.result
index 16e045d0fbb..a0a8072265c 100644
--- a/mysql-test/r/ctype_utf32.result
+++ b/mysql-test/r/ctype_utf32.result
@@ -2226,5 +2226,14 @@ SELECT _utf32 0x10001=_utf32 0x10002;
 _utf32 0x10001=_utf32 0x10002
 1
 #
+# MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+#
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf32) AS SIGNED);
+CAST(CONVERT('1Ĳ3' USING utf32) AS SIGNED)
+1
+Warnings:
+Warning	1292	Truncated incorrect INTEGER value: '1Ĳ3'
+#
 # End of 10.1 tests
 #
diff --git a/mysql-test/t/ctype_ucs.test b/mysql-test/t/ctype_ucs.test
index 2875a2bfcb4..2f480621c56 100644
--- a/mysql-test/t/ctype_ucs.test
+++ b/mysql-test/t/ctype_ucs.test
@@ -948,6 +948,11 @@ EXPLAIN EXTENDED
 SELECT * FROM t1 WHERE COALESCE(c,0)='3 ' AND COALESCE(d,0)=COALESCE(c,0);
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED);
 
 --echo #
 --echo # End of 10.1 tests
diff --git a/mysql-test/t/ctype_utf16.test b/mysql-test/t/ctype_utf16.test
index 8ea6ea67f1f..bb7eb8cd959 100644
--- a/mysql-test/t/ctype_utf16.test
+++ b/mysql-test/t/ctype_utf16.test
@@ -886,5 +886,12 @@ SELECT COUNT(DISTINCT a) FROM t1;
 DROP TABLE t1;
 
 --echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING utf16) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf16) AS SIGNED);
+
+
+--echo #
 --echo # End of 10.1 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf16le.test b/mysql-test/t/ctype_utf16le.test
index f5998fec18c..42017f9a635 100644
--- a/mysql-test/t/ctype_utf16le.test
+++ b/mysql-test/t/ctype_utf16le.test
@@ -770,5 +770,11 @@ SELECT COUNT(DISTINCT a) FROM t1;
 DROP TABLE t1;
 
 --echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf16le) AS SIGNED);
+
+--echo #
 --echo # End of 10.1 tests
 --echo #
diff --git a/mysql-test/t/ctype_utf32.test b/mysql-test/t/ctype_utf32.test
index a75ac72b67f..4bff8a867f1 100644
--- a/mysql-test/t/ctype_utf32.test
+++ b/mysql-test/t/ctype_utf32.test
@@ -971,5 +971,11 @@ DROP TABLE t1;
 SELECT _utf32 0x10001=_utf32 0x10002;
 
 --echo #
+--echo # MDEV-9178 Wrong result for CAST(CONVERT('1Ĳ3' USING ucs2) AS SIGNED)
+--echo #
+SET NAMES utf8;
+SELECT CAST(CONVERT('1Ĳ3' USING utf32) AS SIGNED);
+
+--echo #
 --echo # End of 10.1 tests
 --echo #
diff --git a/strings/ctype-mb.ic b/strings/ctype-mb.ic
index 0ad945b685d..6fc4d6e3db4 100644
--- a/strings/ctype-mb.ic
+++ b/strings/ctype-mb.ic
@@ -269,7 +269,7 @@ my_native_to_mb_fixed2(my_wc_t wc, uchar *s, uchar *e)
 {
   /* The caller must insure there is a space for at least one byte */
   DBUG_ASSERT(s < e);
-  s[0]= wc >> 8;
+  s[0]= (uchar) (wc >> 8);
   if (s + 2 > e)
     return MY_CS_TOOSMALL2;
   s[1]= wc & 0xFF;
@@ -286,7 +286,7 @@ my_native_to_mb_fixed3(my_wc_t wc, uchar *s, uchar *e)
 {
   /* The caller must insure there is a space for at least one byte */
   DBUG_ASSERT(s < e);
-  s[0]= wc >> 16;
+  s[0]= (uchar) (wc >> 16);
   if (s + 2 > e)
     return MY_CS_TOOSMALL2;
   s[1]= (wc >> 8) & 0xFF;
diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c
index e7ba5cbc3c3..5780fff003a 100644
--- a/strings/ctype-ucs2.c
+++ b/strings/ctype-ucs2.c
@@ -818,6 +818,21 @@ cnv:
 
 
 #ifdef HAVE_CHARSET_mb2
+/**
+  Convert a Unicode code point to a digit.
+  @param      wc  - the input Unicode code point
+  @param[OUT] c   - the output character representing the digit value 0..9
+
+  @return   0     - if wc is a good digit
+  @return   1     - if wc is not a digit
+*/
+static inline my_bool
+wc2digit_uchar(uchar *c, my_wc_t wc)
+{
+  return wc > '9' || (c[0]= (uchar) (wc - '0')) > 9;
+}
+
+
 static longlong
 my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
                  const char *nptr, char **endptr, int *error)
@@ -921,7 +936,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
   {
     if ((res= mb_wc(cs, &wc, s, n_end)) <= 0)
       break;
-    if ((c= (wc - '0')) > 9)
+    if (wc2digit_uchar(&c, wc))
       goto end_i;
     i= i*10+c;
   }
@@ -938,7 +953,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
   {
     if ((res= mb_wc(cs, &wc, s, end)) <= 0)
       goto no_conv;
-    if ((c= (wc - '0')) > 9)
+    if (wc2digit_uchar(&c, wc))
       goto end_i_and_j;
     s+= res;
     j= j * 10 + c;
@@ -961,7 +976,7 @@ my_strtoll10_mb2(CHARSET_INFO *cs __attribute__((unused)),
     goto end4;
   if ((res= mb_wc(cs, &wc, s, end)) <= 0)
     goto no_conv;
-  if ((c= (wc - '0')) > 9)
+  if (wc2digit_uchar(&c, wc))
     goto end4;
   s+= res;
   k= k*10+c;