A joint patch for:

Bug#51675 Server crashes on inserting 4 byte char. after ALTER TABLE to 'utf8mb4' 
Bug#51676 Server crashes on SELECT, ORDER BY on 'utf8mb4' column


include/m_ctype.h:
  Defining MY_CS_REPLACEMENT_CHARACTER
mysql-test/r/ctype_utf8mb4.result:
  Adding tests
mysql-test/t/ctype_utf8mb4.test:
  Adding tests
strings/ctype-uca.c:
  Don't use UCA data for characters higher than 0xFFFF.
strings/ctype-ucs2.c:
  Using newly defined MY_CS_REPLACEMENT_CHARACTER
strings/ctype-utf8.c:
  Using newly defined MY_CS_REPLACEMENT_CHARACTER
  Removing unesed variable "plane".

2 files changed, 101 insertions, 0 deletions

diff --git a/mysql-test/r/ctype_utf8mb4.result b/mysql-test/r/ctype_utf8mb4.result
index 40c7ce79cd0..26ac58fc79f 100644
--- a/mysql-test/r/ctype_utf8mb4.result
+++ b/mysql-test/r/ctype_utf8mb4.result
@@ -2412,6 +2412,62 @@ SELECT CONCAT('a', _utf8 '�') FROM t1;
 ERROR HY000: Illegal mix of collations (utf8mb4_general_ci,COERCIBLE) and (utf8_general_ci,COERCIBLE) for operation 'concat'
 DROP TABLE t1;
 #
+# Bug#51675 Server crashes on inserting 4 byte char.
+# after ALTER TABLE to 'utf8mb4' 
+#
+SET NAMES utf8;
+CREATE TABLE t1 ( 
+subject varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci, 
+p VARCHAR(15) CHARACTER SET utf8 
+) DEFAULT CHARSET=latin1;
+ALTER TABLE t1 ADD INDEX (subject);
+ALTER TABLE t1
+DEFAULT CHARACTER SET utf8,
+MODIFY subject varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
+MODIFY p varchar(255) CHARACTER SET utf8;
+Warnings:
+Warning	1071	Specified key was too long; max key length is 1000 bytes
+Warning	1071	Specified key was too long; max key length is 1000 bytes
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `subject` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `p` varchar(255) DEFAULT NULL,
+  KEY `subject` (`subject`(250))
+) ENGINE=MyISAM DEFAULT CHARSET=utf8
+INSERT INTO t1(subject) VALUES ('abcd');
+INSERT INTO t1(subject) VALUES(x'f0909080');
+DROP TABLE t1;
+#
+# Bug #51676 Server crashes on SELECT, ORDER BY on 'utf8mb4' column
+#
+SET NAMES utf8mb4;
+CREATE TABLE t1 ( 
+subject varchar(255) character set utf8mb4 collate utf8mb4_unicode_ci, 
+p varchar(15) character set utf8mb4 
+) DEFAULT CHARSET=latin1;
+INSERT INTO t1(subject) VALUES(0xF0909080);
+INSERT INTO t1(subject) VALUES(0x616263F0909080646566);
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `subject` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `p` varchar(15) CHARACTER SET utf8mb4 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+SELECT *  FROM t1 ORDER BY 1;
+subject	p
+𐐀	NULL
+abc𐐀def	NULL
+SELECT hex(subject), length(subject), char_length(subject), octet_length(subject) FROM t1 ORDER BY 1;
+hex(subject)	length(subject)	char_length(subject)	octet_length(subject)
+616263F0909080646566	10	7	10
+F0909080	4	1	4
+SELECT subject FROM t1 ORDER BY 1;
+subject
+𐐀
+abc𐐀def
+DROP TABLE t1;
+#
 # End of 5.5 tests
 #
 #
diff --git a/mysql-test/t/ctype_utf8mb4.test b/mysql-test/t/ctype_utf8mb4.test
index b4f5de96912..8916de670c1 100644
--- a/mysql-test/t/ctype_utf8mb4.test
+++ b/mysql-test/t/ctype_utf8mb4.test
@@ -1737,6 +1737,51 @@ SELECT CONCAT(utf8mb4, _utf8 '�') FROM t1;
 SELECT CONCAT('a', _utf8 '�') FROM t1;
 DROP TABLE t1;
 
+
+--echo #
+--echo # Bug#51675 Server crashes on inserting 4 byte char.
+--echo # after ALTER TABLE to 'utf8mb4' 
+--echo #
+SET NAMES utf8;
+CREATE TABLE t1 ( 
+  subject varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci, 
+  p VARCHAR(15) CHARACTER SET utf8 
+) DEFAULT CHARSET=latin1; 
+
+# Alter old table, add index 
+ALTER TABLE t1 ADD INDEX (subject); 
+
+# Alter old 'utf8' table to new 'utf8mb4'
+ALTER TABLE t1
+  DEFAULT CHARACTER SET utf8,
+  MODIFY subject varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,
+  MODIFY p varchar(255) CHARACTER SET utf8;
+SHOW CREATE TABLE t1; 
+
+INSERT INTO t1(subject) VALUES ('abcd'); 
+# Insert 4 byte characters
+INSERT INTO t1(subject) VALUES(x'f0909080');
+DROP TABLE t1;
+
+--echo #
+--echo # Bug #51676 Server crashes on SELECT, ORDER BY on 'utf8mb4' column
+--echo #
+SET NAMES utf8mb4; 
+CREATE TABLE t1 ( 
+  subject varchar(255) character set utf8mb4 collate utf8mb4_unicode_ci, 
+  p varchar(15) character set utf8mb4 
+) DEFAULT CHARSET=latin1; 
+# Insert 4 byte characters 
+# 4byte character 
+INSERT INTO t1(subject) VALUES(0xF0909080);
+# mix of 3 byte & 4 byte 
+INSERT INTO t1(subject) VALUES(0x616263F0909080646566);
+SHOW CREATE TABLE t1; 
+SELECT *  FROM t1 ORDER BY 1;
+SELECT hex(subject), length(subject), char_length(subject), octet_length(subject) FROM t1 ORDER BY 1; 
+SELECT subject FROM t1 ORDER BY 1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 5.5 tests
 --echo #