MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)

The code did not take into account that:
- U+005C (backslash) can occupy more than mbminlen characters (e.g. in sjis)
- Some character sets do not have a code for U+005C (e.g. swe7)

Adding a new function my_wc_to_printable into MY_CHARSET_HANDLER to
cover all special cases easier.

6 files changed, 128 insertions, 0 deletions

diff --git a/mysql-test/main/ctype_filename.result b/mysql-test/main/ctype_filename.result
index c6d7d1e39b9..ec4a14c4a23 100644
--- a/mysql-test/main/ctype_filename.result
+++ b/mysql-test/main/ctype_filename.result
@@ -21,3 +21,16 @@ SET NAMES utf8;
 SELECT @a:=CONVERT('aя' USING filename) AS `@a`, BINARY @a, REVERSE(@a), HEX(@a), HEX(REVERSE(@a));
 @a	BINARY @a	REVERSE(@a)	HEX(@a)	HEX(REVERSE(@a))
 aя	a@r1	яa	61407231	40723161
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+#
+SET NAMES filename;
+EXECUTE IMMEDIATE _latin1 0x01;
+ERROR 42000: You@0020have@0020an@0020error@0020in@0020your@0020SQL@0020syntax@003b@0020check@0020the@0020manual@0020that@0020corresponds@0020to@0020your@0020MariaDB@0020server@0020version@0020for@0020the@0020right@0020syntax@0020to@0020use@0020near@0020@0027@005c0001@0027@0020at@0020line@00201
+SET NAMES utf8;
+#
+# End of 10.5 tests
+#
diff --git a/mysql-test/main/ctype_filename.test b/mysql-test/main/ctype_filename.test
index 7ec07293a2b..bd82714ca0e 100644
--- a/mysql-test/main/ctype_filename.test
+++ b/mysql-test/main/ctype_filename.test
@@ -27,3 +27,22 @@ select convert(convert(',' using filename) using binary);
 --echo #
 SET NAMES utf8;
 SELECT @a:=CONVERT('aя' USING filename) AS `@a`, BINARY @a, REVERSE(@a), HEX(@a), HEX(REVERSE(@a));
+
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+--echo #
+
+SET NAMES filename;
+--error ER_PARSE_ERROR
+EXECUTE IMMEDIATE _latin1 0x01;
+SET NAMES utf8;
+
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
diff --git a/mysql-test/main/ctype_sjis.result b/mysql-test/main/ctype_sjis.result
index 4917a1973a1..fae022b60e1 100644
--- a/mysql-test/main/ctype_sjis.result
+++ b/mysql-test/main/ctype_sjis.result
@@ -19296,3 +19296,25 @@ SET DEFAULT_STORAGE_ENGINE=Default;
 #
 # End of 10.2 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+#
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+(a(b 'т'));
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'a(b '�_0143�_201A'))' at line 1
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+'т';
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near ''�_0143�_201A'' at line 1
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+EXECUTE IMMEDIATE _cp1257 0xD182;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '�_0143�_201A' at line 1
+SET NAMES sjis;
+#
+# End of 10.5 tests
+#
diff --git a/mysql-test/main/ctype_sjis.test b/mysql-test/main/ctype_sjis.test
index 9a8ce414c14..804ba557faa 100644
--- a/mysql-test/main/ctype_sjis.test
+++ b/mysql-test/main/ctype_sjis.test
@@ -260,3 +260,33 @@ let $coll_pad='sjis_bin';
 --echo #
 --echo # End of 10.2 tests
 --echo #
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+--echo #
+
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+--error ER_PARSE_ERROR
+(a(b 'т'));
+
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+--error ER_PARSE_ERROR
+'т';
+
+SET NAMES sjis;
+SET @@CHARACTER_SET_CLIENT='cp1257';
+--error ER_PARSE_ERROR
+EXECUTE IMMEDIATE _cp1257 0xD182;
+
+SET NAMES sjis;
+
+
+--echo #
+--echo # End of 10.5 tests
+--echo #
diff --git a/mysql-test/main/ctype_swe7.result b/mysql-test/main/ctype_swe7.result
index e1913fa5dcd..db65b6b8a1a 100644
--- a/mysql-test/main/ctype_swe7.result
+++ b/mysql-test/main/ctype_swe7.result
@@ -3635,3 +3635,21 @@ SET DEFAULT_STORAGE_ENGINE=Default;
 #
 # End of 10.2 tests
 #
+#
+# Start of 10.5 tests
+#
+#
+# MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+#
+SET NAMES swe7;
+SELECT `Ｔ`;
+ERROR HY000: Invalid swe7 character string: '.xEF.xBC.xB4'
+SET NAMES swe7;
+SELECT `龔`;
+ERROR HY000: Invalid swe7 character string: '.xE9.xBE.x94'
+SET NAMES swe7;
+EXECUTE IMMEDIATE _swe7 0x01;
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '.0001' at line 1
+#
+# End of 10.5 tests
+#
diff --git a/mysql-test/main/ctype_swe7.test b/mysql-test/main/ctype_swe7.test
index 2e648a89406..a8d79935aa6 100644
--- a/mysql-test/main/ctype_swe7.test
+++ b/mysql-test/main/ctype_swe7.test
@@ -38,3 +38,29 @@ let $coll_pad='swe7_bin';
 --echo #
 --echo # End of 10.2 tests
 --echo #
+
+
+--echo #
+--echo # Start of 10.5 tests
+--echo #
+
+--echo #
+--echo # MDEV-22043 Special character leads to assertion in my_wc_to_printable_generic on 10.5.2 (debug)
+--echo #
+
+SET NAMES swe7;
+--error ER_INVALID_CHARACTER_STRING
+SELECT `Ｔ`;
+
+SET NAMES swe7;
+--error ER_INVALID_CHARACTER_STRING
+SELECT `龔`;
+
+SET NAMES swe7;
+--error ER_PARSE_ERROR
+EXECUTE IMMEDIATE _swe7 0x01;
+
+
+--echo #
+--echo # End of 10.5 tests
+--echo #