Backporting WL#1213

config/ac-macros/character_sets.m4:
  - Adding configure definitions for utf8mb4, utf16, utf32
include/config-win.h:
  - Enabling utf8mb4, utf16, utf32 in Windows build
include/m_ctype.h:
  - Adding new flags
  - Adding new shared functions prototypes
mysql-test/include/ctype_datetime.inc:
  - Adding test to check that datetime functions
    work with "real" multibyte character sets.
mysql-test/include/ctype_like.inc:
  - Adding LIKE tests
mysql-test/include/have_utf16.inc:
  New file
mysql-test/include/have_utf32.inc:
  New file
mysql-test/include/have_utf8mb4.inc:
  New file
mysql-test/r/ctype_ldml.result:
  - Adding tests for utf8mb4, utf16, utf32
mysql-test/r/ctype_many.result:
  - Adding tests to check superset/subset relations
    between all Unicode character sets.
mysql-test/r/ctype_utf16.result:
  New file
mysql-test/r/ctype_utf16_uca.result:
  New file
mysql-test/r/ctype_utf32.result:
  New file
mysql-test/r/ctype_utf32_uca.result:
  New file
mysql-test/r/ctype_utf8.result:
  - Adding tests for utf8mn3 alias
mysql-test/r/ctype_utf8mb4.result:
  - Adding tests for utf8mb4
mysql-test/r/have_utf16.require:
  New file
mysql-test/r/have_utf32.require:
  New file
mysql-test/r/have_utf8mb4.require:
  New file
mysql-test/std_data/Index.xml:
  - Adding tests for loadable utf8m4, utf16, utf32 collations
mysql-test/suite/sys_vars/r/character_set_client_basic.result:
  - Adding tests for utf16, utf32.
  - Fixing new number of character sets
mysql-test/suite/sys_vars/r/character_set_connection_basic.result:
  - Fixing new number of character sets
mysql-test/suite/sys_vars/r/character_set_database_basic.result:
  - Fixing new number of character sets
mysql-test/suite/sys_vars/r/character_set_filesystem_basic.result:
  - Fixing new number of character sets
mysql-test/suite/sys_vars/r/character_set_results_basic.result:
  - Fixing new number of character sets
mysql-test/suite/sys_vars/t/character_set_client_basic.test:
  - Adding tests for new character sets
mysql-test/suite/sys_vars/t/character_set_connection_basic.test:
  - Adding dependency on utf8mb4, utf16, utf32
mysql-test/suite/sys_vars/t/character_set_database_basic.test:
  - Adding dependency on utf8mb4, utf16, utf32
mysql-test/suite/sys_vars/t/character_set_filesystem_basic.test:
  - Adding dependency on utf8mb4, utf16, utf32
mysql-test/suite/sys_vars/t/character_set_results_basic.test:
  - Adding dependency on utf8mb4, utf16, utf32
mysql-test/t/ctype_ldml.test:
  - Adding tests for dynamic utf8mb4, utf16, utf32 collations
mysql-test/t/ctype_many.test:
  - Adding tests to check superset/subset relations
    between all Unicode character sets
mysql-test/t/ctype_utf16.test:
  New file
mysql-test/t/ctype_utf16_uca.test:
  New file
mysql-test/t/ctype_utf32.test:
  New file
mysql-test/t/ctype_utf32_uca.test:
  New file
mysql-test/t/ctype_utf8.test:
  - Adding tests for utf8mb4 alias
mysql-test/t/ctype_utf8mb4.test:
  New file
mysys/charset-def.c:
  - Adding initialization of utf8mb4, utf16, utf32 built-int collations
mysys/charset.c:
  - Adding initialization of utf8mb4, utf16, utf32 dynamic collations
sql/field.cc:
  - Fixing "truncated" error with datetime functions:
    Force conversion in case of non-ascii character sets.
sql/item.cc:
  - Adding superset/subset relation check for utf8mb4/utf8
sql/item_strfunc.cc:
  - Fixing a problem with CHAR(x USING utf32)
sql/sql_string.cc:
  - Fixing problems with zero padding for UTF32
sql/sql_table.cc:
  - Fixing buffer size, to make utf32 comma fit.
strings/ctype-mb.c:
  - Making handlers for multi-byte binary collations public
strings/ctype-uca.c:
  - Adding definitions for utf8mb4, utf16, utf32 UCA collations
strings/ctype-ucs2.c:
  - Adding functions which are shared between ucs2, utf16, utf32
  - Ading utf16 implementation
  - Adding utf32 implementation
strings/ctype-utf8.c:
  - Adding functions shared between utf8 and utf8mb4
  - Adding implementation of utf8mb4

1 files changed, 290 insertions, 0 deletions

diff --git a/mysql-test/t/ctype_utf16_uca.test b/mysql-test/t/ctype_utf16_uca.test
new file mode 100644
index 00000000000..5314777c6f4
--- /dev/null
+++ b/mysql-test/t/ctype_utf16_uca.test
@@ -0,0 +1,290 @@
+-- source include/have_utf16.inc
+
+--disable_warnings
+DROP TABLE IF EXISTS t1;
+--enable_warnings
+
+--echo #
+--echo # Start of 5.5 tests
+--echo #
+
+set names utf8;
+set collation_connection=utf16_unicode_ci;
+select hex('a'), hex('a ');
+-- source include/endspace.inc
+
+#
+# Bug #6787 LIKE not working properly with _ and utf8 data
+#
+select 'c' like '\_' as want0; 
+
+#
+# Bug #5679 utf8_unicode_ci LIKE--trailing % doesn't equal zero characters
+#
+CREATE TABLE t (
+  c char(20) NOT NULL
+) ENGINE=MyISAM DEFAULT CHARACTER SET utf16 COLLATE utf16_unicode_ci;
+INSERT INTO t VALUES ('a'),('ab'),('aba');
+ALTER TABLE t ADD INDEX (c);
+SELECT c FROM t WHERE c LIKE 'a%';
+DROP TABLE t;
+
+
+create table t1 (c1 char(10) character set utf16 collate utf16_bin);
+
+#
+# Basic Latin
+#
+insert into t1 values ('A'),('a');
+insert into t1 values ('B'),('b');
+insert into t1 values ('C'),('c');
+insert into t1 values ('D'),('d');
+insert into t1 values ('E'),('e');
+insert into t1 values ('F'),('f');
+insert into t1 values ('G'),('g');
+insert into t1 values ('H'),('h');
+insert into t1 values ('I'),('i');
+insert into t1 values ('J'),('j');
+insert into t1 values ('K'),('k');
+insert into t1 values ('L'),('l');
+insert into t1 values ('M'),('m');
+insert into t1 values ('N'),('n');
+insert into t1 values ('O'),('o');
+insert into t1 values ('P'),('p');
+insert into t1 values ('Q'),('q');
+insert into t1 values ('R'),('r');
+insert into t1 values ('S'),('s');
+insert into t1 values ('T'),('t');
+insert into t1 values ('U'),('u');
+insert into t1 values ('V'),('v');
+insert into t1 values ('W'),('w');
+insert into t1 values ('X'),('x');
+insert into t1 values ('Y'),('y');
+insert into t1 values ('Z'),('z');
+
+#
+# Latin1 suppliment
+#
+insert into t1 values (0x00e0),(0x00c0);
+insert into t1 values (0x00e1),(0x00c1);
+insert into t1 values (0x00e2),(0x00c2);
+insert into t1 values (0x00e3),(0x00c3);
+insert into t1 values (0x00e4),(0x00c4);
+insert into t1 values (0x00e5),(0x00c5);
+insert into t1 values (0x00e6),(0x00c6);
+insert into t1 values (0x00e7),(0x00c7);
+insert into t1 values (0x00e8),(0x00c8);
+insert into t1 values (0x00e9),(0x00c9);
+insert into t1 values (0x00ea),(0x00ca);
+insert into t1 values (0x00eb),(0x00cb);
+insert into t1 values (0x00ec),(0x00cc);
+insert into t1 values (0x00ed),(0x00cd);
+insert into t1 values (0x00ee),(0x00ce);
+insert into t1 values (0x00ef),(0x00cf);
+
+insert into t1 values (0x00f0),(0x00d0);
+insert into t1 values (0x00f1),(0x00d1);
+insert into t1 values (0x00f2),(0x00d2);
+insert into t1 values (0x00f3),(0x00d3);
+insert into t1 values (0x00f4),(0x00d4);
+insert into t1 values (0x00f5),(0x00d5);
+insert into t1 values (0x00f6),(0x00d6);
+insert into t1 values (0x00f7),(0x00d7);
+insert into t1 values (0x00f8),(0x00d8);
+insert into t1 values (0x00f9),(0x00d9);
+insert into t1 values (0x00fa),(0x00da);
+insert into t1 values (0x00fb),(0x00db);
+insert into t1 values (0x00fc),(0x00dc);
+insert into t1 values (0x00fd),(0x00dd);
+insert into t1 values (0x00fe),(0x00de);
+insert into t1 values (0x00ff),(0x00df);
+
+#
+# Latin extended-A, 0100-017F
+#
+insert into t1 values (0x0100),(0x0101),(0x0102),(0x0103);
+insert into t1 values (0x0104),(0x0105),(0x0106),(0x0107);
+insert into t1 values (0x0108),(0x0109),(0x010a),(0x010b);
+insert into t1 values (0x010c),(0x010d),(0x010e),(0x010f);
+insert into t1 values (0x0110),(0x0111),(0x0112),(0x0113);
+insert into t1 values (0x0114),(0x0115),(0x0116),(0x0117);
+insert into t1 values (0x0118),(0x0119),(0x011a),(0x011b);
+insert into t1 values (0x011c),(0x011d),(0x011e),(0x011f);
+insert into t1 values (0x0120),(0x0121),(0x0122),(0x0123);
+insert into t1 values (0x0124),(0x0125),(0x0126),(0x0127);
+insert into t1 values (0x0128),(0x0129),(0x012a),(0x012b);
+insert into t1 values (0x012c),(0x012d),(0x012e),(0x012f);
+insert into t1 values (0x0130),(0x0131),(0x0132),(0x0133);
+insert into t1 values (0x0134),(0x0135),(0x0136),(0x0137);
+insert into t1 values (0x0138),(0x0139),(0x013a),(0x013b);
+insert into t1 values (0x013c),(0x013d),(0x013e),(0x013f);
+insert into t1 values (0x0140),(0x0141),(0x0142),(0x0143);
+insert into t1 values (0x0144),(0x0145),(0x0146),(0x0147);
+insert into t1 values (0x0148),(0x0149),(0x014a),(0x014b);
+insert into t1 values (0x014c),(0x014d),(0x014e),(0x014f);
+insert into t1 values (0x0150),(0x0151),(0x0152),(0x0153);
+insert into t1 values (0x0154),(0x0155),(0x0156),(0x0157);
+insert into t1 values (0x0158),(0x0159),(0x015a),(0x015b);
+insert into t1 values (0x015c),(0x015d),(0x015e),(0x015f);
+insert into t1 values (0x0160),(0x0161),(0x0162),(0x0163);
+insert into t1 values (0x0164),(0x0165),(0x0166),(0x0167);
+insert into t1 values (0x0168),(0x0169),(0x016a),(0x016b);
+insert into t1 values (0x016c),(0x016d),(0x016e),(0x016f);
+insert into t1 values (0x0170),(0x0171),(0x0172),(0x0173);
+insert into t1 values (0x0174),(0x0175),(0x0176),(0x0177);
+insert into t1 values (0x0178),(0x0179),(0x017a),(0x017b);
+insert into t1 values (0x017c),(0x017d),(0x017e),(0x017f);
+
+#
+# Latin extended-B, 0180-024F
+#
+insert into t1 values (0x0180),(0x0181),(0x0182),(0x0183);
+insert into t1 values (0x0184),(0x0185),(0x0186),(0x0187);
+insert into t1 values (0x0188),(0x0189),(0x018a),(0x018b);
+insert into t1 values (0x018c),(0x018d),(0x018e),(0x018f);
+insert into t1 values (0x0190),(0x0191),(0x0192),(0x0193);
+insert into t1 values (0x0194),(0x0195),(0x0196),(0x0197);
+insert into t1 values (0x0198),(0x0199),(0x019a),(0x019b);
+insert into t1 values (0x019c),(0x019d),(0x019e),(0x019f);
+insert into t1 values (0x01a0),(0x01a1),(0x01a2),(0x01a3);
+insert into t1 values (0x01a4),(0x01a5),(0x01a6),(0x01a7);
+insert into t1 values (0x01a8),(0x01a9),(0x01aa),(0x01ab);
+insert into t1 values (0x01ac),(0x01ad),(0x01ae),(0x01af);
+insert into t1 values (0x01b0),(0x01b1),(0x01b2),(0x01b3);
+insert into t1 values (0x01b4),(0x01b5),(0x01b6),(0x01b7);
+insert into t1 values (0x01b8),(0x01b9),(0x01ba),(0x01bb);
+insert into t1 values (0x01bc),(0x01bd),(0x01be),(0x01bf);
+insert into t1 values (0x01c0),(0x01c1),(0x01c2),(0x01c3);
+insert into t1 values (0x01c4),(0x01c5),(0x01c6),(0x01c7);
+insert into t1 values (0x01c8),(0x01c9),(0x01ca),(0x01cb);
+insert into t1 values (0x01cc),(0x01cd),(0x01ce),(0x01cf);
+insert into t1 values (0x01d0),(0x01d1),(0x01d2),(0x01d3);
+insert into t1 values (0x01d4),(0x01d5),(0x01d6),(0x01d7);
+insert into t1 values (0x01d8),(0x01d9),(0x01da),(0x01db);
+insert into t1 values (0x01dc),(0x01dd),(0x01de),(0x01df);
+insert into t1 values (0x01e0),(0x01e1),(0x01e2),(0x01e3);
+insert into t1 values (0x01e4),(0x01e5),(0x01e6),(0x01e7);
+insert into t1 values (0x01e8),(0x01e9),(0x01ea),(0x01eb);
+insert into t1 values (0x01ec),(0x01ed),(0x01ee),(0x01ef);
+insert into t1 values (0x01f0),(0x01f1),(0x01f2),(0x01f3);
+insert into t1 values (0x01f4),(0x01f5),(0x01f6),(0x01f7);
+insert into t1 values (0x01f8),(0x01f9),(0x01fa),(0x01fb);
+insert into t1 values (0x01fc),(0x01fd),(0x01fe),(0x01ff);
+
+
+insert into t1 values ('AA'),('Aa'),('aa'),('aA');
+insert into t1 values ('CH'),('Ch'),('ch'),('cH');
+insert into t1 values ('DZ'),('Dz'),('dz'),('dZ');
+insert into t1 values ('IJ'),('Ij'),('ij'),('iJ');
+insert into t1 values ('LJ'),('Lj'),('lj'),('lJ');
+insert into t1 values ('LL'),('Ll'),('ll'),('lL');
+insert into t1 values ('NJ'),('Nj'),('nj'),('nJ');
+insert into t1 values ('OE'),('Oe'),('oe'),('oE');
+insert into t1 values ('SS'),('Ss'),('ss'),('sS');
+insert into t1 values ('RR'),('Rr'),('rr'),('rR');
+
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_unicode_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_icelandic_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_latvian_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_romanian_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_slovenian_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_polish_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_estonian_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_spanish_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_swedish_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_turkish_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_czech_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_danish_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_lithuanian_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_slovak_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_spanish2_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_roman_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_esperanto_ci;
+select group_concat(c1 order by binary c1 separator '') from t1 group by c1 collate utf16_hungarian_ci;
+
+drop table t1;
+
+#
+# Bug#5324
+#
+SET NAMES utf8;
+
+CREATE TABLE t1 (c varchar(200) CHARACTER SET utf16 COLLATE utf16_general_ci NOT NULL, INDEX (c));
+INSERT INTO t1 VALUES (0x039C03C903B403B11F770308);
+#Check one row
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025 COLLATE utf16_general_ci;
+INSERT INTO t1 VALUES (0x039C03C903B4);
+#Check two rows
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025
+COLLATE utf16_general_ci ORDER BY c;
+DROP TABLE t1;
+
+CREATE TABLE t1 (c varchar(200) CHARACTER SET utf16 COLLATE utf16_unicode_ci NOT NULL, INDEX (c));
+INSERT INTO t1 VALUES (0x039C03C903B403B11F770308);
+#Check one row
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025 COLLATE utf16_unicode_ci;
+INSERT INTO t1 VALUES (0x039C03C903B4);
+#Check two rows
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025
+COLLATE utf16_unicode_ci ORDER BY c;
+DROP TABLE t1;
+
+CREATE TABLE t1 (c varchar(200) CHARACTER SET utf16 COLLATE utf16_unicode_ci NOT NULL, INDEX (c));
+INSERT INTO t1 VALUES (0x039C03C903B403B11F770308);
+#Check one row row
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025 COLLATE utf16_unicode_ci;
+INSERT INTO t1 VALUES (0x039C03C903B4);
+#Check two rows
+SELECT * FROM t1 WHERE c LIKE _utf16 0x039C0025
+COLLATE utf16_unicode_ci ORDER BY c;
+DROP TABLE t1;
+
+
+SET NAMES utf8;
+SET @test_character_set='utf16';
+SET @test_collation='utf16_swedish_ci';
+-- source include/ctype_common.inc
+
+
+SET collation_connection='utf16_unicode_ci';
+-- source include/ctype_filesort.inc
+-- source include/ctype_like_escape.inc
+
+--echo End of 4.1 tests
+
+#
+# Check UPPER/LOWER changing length
+#
+# Result shorter than argument
+CREATE TABLE t1 (id int, a varchar(30) character set utf16);
+INSERT INTO t1 VALUES (1, 0x01310069), (2, 0x01310131);
+INSERT INTO t1 VALUES (3, 0x00690069), (4, 0x01300049);
+INSERT INTO t1 VALUES (5, 0x01300130), (6, 0x00490049);
+SELECT a, length(a) la, @l:=lower(a) l, length(@l) ll, @u:=upper(a) u, length(@u) lu
+FROM t1 ORDER BY id;
+ALTER TABLE t1 MODIFY a VARCHAR(30) character set utf16 collate utf16_turkish_ci;
+SELECT a, length(a) la, @l:=lower(a) l, length(@l) ll, @u:=upper(a) u, length(@u) lu
+FROM t1 ORDER BY id;
+DROP TABLE t1;
+
+#
+# Bug #27079 Crash while grouping empty ucs2 strings
+#
+CREATE TABLE t1 (
+ c1 text character set utf16 collate utf16_polish_ci NOT NULL
+) ENGINE=MyISAM;
+insert into t1 values (''),('a');
+SELECT COUNT(*), c1 FROM t1 GROUP BY c1;
+DROP TABLE IF EXISTS t1;
+
+
+#
+# Test basic regex functionality
+#
+set collation_connection=utf16_unicode_ci;
+--source include/ctype_regex.inc
+
+
+--echo #
+--echo # End of 5.5 tests
+--echo #