diff options
author | unknown <bar@bar.intranet.mysql.r18.ru> | 2004-06-08 17:56:15 +0500 |
---|---|---|
committer | unknown <bar@bar.intranet.mysql.r18.ru> | 2004-06-08 17:56:15 +0500 |
commit | 8ab01b335d502730bef01764d0cd4adae6f86ee0 (patch) | |
tree | 63ff39051c79dfeba8601f919a432d9944ab069f | |
parent | 19480ed616000e9e67e0adae52dc5dc9cb27fdcb (diff) | |
download | mariadb-git-8ab01b335d502730bef01764d0cd4adae6f86ee0.tar.gz |
WL#916: Unicode collations for some languages
-rw-r--r-- | include/m_ctype.h | 1 | ||||
-rw-r--r-- | mysys/charset.c | 210 | ||||
-rw-r--r-- | strings/ctype-big5.c | 2 | ||||
-rw-r--r-- | strings/ctype-bin.c | 1 | ||||
-rw-r--r-- | strings/ctype-czech.c | 1 | ||||
-rw-r--r-- | strings/ctype-euc_kr.c | 2 | ||||
-rw-r--r-- | strings/ctype-gb2312.c | 2 | ||||
-rw-r--r-- | strings/ctype-gbk.c | 2 | ||||
-rw-r--r-- | strings/ctype-latin1.c | 3 | ||||
-rw-r--r-- | strings/ctype-sjis.c | 2 | ||||
-rw-r--r-- | strings/ctype-tis620.c | 2 | ||||
-rw-r--r-- | strings/ctype-uca.c | 1 | ||||
-rw-r--r-- | strings/ctype-ucs2.c | 2 | ||||
-rw-r--r-- | strings/ctype-ujis.c | 2 | ||||
-rw-r--r-- | strings/ctype-utf8.c | 2 | ||||
-rw-r--r-- | strings/ctype-win1250ch.c | 1 | ||||
-rw-r--r-- | strings/ctype.c | 14 |
17 files changed, 214 insertions, 36 deletions
diff --git a/include/m_ctype.h b/include/m_ctype.h index 9502805b017..87b45bd4954 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -200,6 +200,7 @@ typedef struct charset_info_st const char *csname; const char *name; const char *comment; + const char *tailoring; uchar *ctype; uchar *to_lower; uchar *to_upper; diff --git a/mysys/charset.c b/mysys/charset.c index 7eccf2dab68..ea07708963d 100644 --- a/mysys/charset.c +++ b/mysys/charset.c @@ -624,6 +624,123 @@ err: #ifdef HAVE_CHARSET_ucs2 +typedef struct my_tailoring_st +{ + uint number; + const char *name; + const char *tailoring; +} my_tailoring; + +static my_tailoring tailoring[]= +{ + { + 0, "icelandic", + /* + Some sources treat LETTER A WITH DIARESIS (00E4,00C4) + secondary greater than LETTER AE (00E6,00C6). + http://www.evertype.com/alphabets/icelandic.pdf + http://developer.mimer.com/collations/charts/icelandic.htm + + Other sources do not provide any special rules + for LETTER A WITH DIARESIS: + http://www.omniglot.com/writing/icelandic.htm + http://en.wikipedia.org/wiki/Icelandic_alphabet + http://oss.software.ibm.com/icu/charts/collation/is.html + + Let's go the first way. + */ + "& A < \\u00E1 <<< \\u00C1 " + "& D < \\u00F0 <<< \\u00D0 " + "& E < \\u00E9 <<< \\u00C9 " + "& I < \\u00ED <<< \\u00CD " + "& O < \\u00F3 <<< \\u00D3 " + "& U < \\u00FA <<< \\u00DA " + "& Y < \\u00FD <<< \\u00DD " + "& Z < \\u00FE <<< \\u00DE " + "< \\u00E6 <<< \\u00C6 << \\u00E4 <<< \\u00C4 " + "< \\u00F6 <<< \\u00D6 << \\u00F8 <<< \\u00D8 " + "< \\u00E5 <<< \\u00C5 " + }, + { + 1, "latvian", + /* + Some sources treat I and Y primary different. + Other sources treat I and Y the same on primary level. + We'll go the first way. + */ + "& C < \\u010D <<< \\u010C " + "& G < \\u0123 <<< \\u0122 " + "& I < \\u0079 <<< \\u0059 " + "& K < \\u0137 <<< \\u0136 " + "& L < \\u013C <<< \\u013B " + "& N < \\u0146 <<< \\u0145 " + "& R < \\u0157 <<< \\u0156 " + "& S < \\u0161 <<< \\u0160 " + "& Z < \\u017E <<< \\u017D " + }, + { + 2, "romanian", + "& A < \\u0103 <<< \\u0102 < \\u00E2 <<< \\u00C2 " + "& I < \\u00EE <<< \\u00CE " + "& S < \\u0219 <<< \\u0218 << \\u015F <<< \\u015E " + "& T < \\u021B <<< \\u021A << \\u0163 <<< \\u0162 " + }, + { + 3, "slovenian", + "& C < \\u010D <<< \\u010C " + "& S < \\u0161 <<< \\u0160 " + "& Z < \\u017E <<< \\u017D " + }, + { + 4, "polish", + "& A < \\u0105 <<< \\u0104 " + "& C < \\u0107 <<< \\u0106 " + "& E < \\u0119 <<< \\u0118 " + "& L < \\u0142 <<< \\u0141 " + "& N < \\u0144 <<< \\u0143 " + "& O < \\u00F3 <<< \\u00D3 " + "& S < \\u015B <<< \\u015A " + "& Z < \\u017A <<< \\u017B " + }, + { + 5, "estonian", + "& S < \\u0161 <<< \\u0160 " + " < \\u007A <<< \\u005A " + " < \\u017E <<< \\u017D " + "& W < \\u00F5 <<< \\u00D5 " + "< \\u00E4 <<< \\u00C4 " + "< \\u00F6 <<< \\u00D6 " + "< \\u00FC <<< \\u00DC " + }, + { + 6, "spanish", + "& N < \\u00F1 <<< \\u00D1 " + }, + { + 7, "swedish", + /* + Some sources treat V and W as similar on primary level. + We'll treat V and W as different on primary level. + */ + "& Y <<\\u00FC <<< \\u00DC " + "& Z < \\u00E5 <<< \\u00C5 " + "< \\u00E4 <<< \\u00C4 << \\u00E6 <<< \\u00C6 " + "< \\u00F6 <<< \\u00D6 << \\u00F8 <<< \\u00D8 " + }, + { + 8, "turkish", + "& C < \\u00E7 <<< \\u00C7 " + "& G < \\u011F <<< \\u011E " + "& H < \\u0131 <<< \\u0049 " + "& O < \\u00F6 <<< \\u00D6 " + "& S < \\u015F <<< \\u015E " + "& U < \\u00FC <<< \\u00DC " + }, + { + 0, NULL, NULL + } +}; + #define MY_MAX_COLL_RULE 64 /* @@ -643,7 +760,7 @@ err: default weights. */ -static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) +static my_bool create_tailoring(CHARSET_INFO *cs) { MY_COLL_RULE rule[MY_MAX_COLL_RULE]; char errstr[128]; @@ -652,32 +769,14 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) const uchar *deflengths= my_charset_ucs2_general_uca.sort_order; uint16 **defweights= my_charset_ucs2_general_uca.sort_order_big; int rc, i; - - to->number= from->number ? from->number : to->number; - - if (from->csname) - if (!(to->csname= my_once_strdup(from->csname,MYF(MY_WME)))) - goto err; - - if (from->name) - if (!(to->name= my_once_strdup(from->name,MYF(MY_WME)))) - goto err; - - if (from->comment) - if (!(to->comment= my_once_strdup(from->comment,MYF(MY_WME)))) - goto err; - - to->strxfrm_multiply= my_charset_ucs2_general_uca.strxfrm_multiply; - to->min_sort_char= my_charset_ucs2_general_uca.min_sort_char; - to->max_sort_char= my_charset_ucs2_general_uca.max_sort_char; - to->mbminlen= 2; - to->mbmaxlen= 2; - + + if (!cs->tailoring) + return 1; /* Parse ICU Collation Customization expression */ if ((rc= my_coll_rule_parse(rule, MY_MAX_COLL_RULE, - from->sort_order, - from->sort_order + strlen(from->sort_order), + cs->tailoring, + cs->tailoring + strlen(cs->tailoring), errstr, sizeof(errstr))) <= 0) { /* @@ -687,13 +786,12 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) return 1; } - if (!(newweights= (uint16**) my_once_alloc(256*sizeof(uint16*),MYF(MY_WME)))) - goto err; + return 1; bzero(newweights, 256*sizeof(uint16*)); if (!(newlengths= (uchar*) my_once_memdup(deflengths,256,MYF(MY_WME)))) - goto err; + return 1; /* Calculate maximum lenghts for the pages @@ -720,7 +818,7 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) uint size= 256*newlengths[pagec]*sizeof(uint16); if (!(newweights[pagec]= (uint16*) my_once_alloc(size,MYF(MY_WME)))) - goto err; + return 1; bzero((void*) newweights[pagec], size); for (chc=0 ; chc < 256; chc++) @@ -749,10 +847,41 @@ static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) if (!newweights[i]) newweights[i]= defweights[i]; - to->sort_order= newlengths; - to->sort_order_big= newweights; + cs->sort_order= newlengths; + cs->sort_order_big= newweights; return 0; +} + + +static int ucs2_copy_data(CHARSET_INFO *to, CHARSET_INFO *from) +{ + + to->number= from->number ? from->number : to->number; + + if (from->csname) + if (!(to->csname= my_once_strdup(from->csname,MYF(MY_WME)))) + goto err; + + if (from->name) + if (!(to->name= my_once_strdup(from->name,MYF(MY_WME)))) + goto err; + + if (from->comment) + if (!(to->comment= my_once_strdup(from->comment,MYF(MY_WME)))) + goto err; + + if (from->tailoring) + if (!(to->tailoring= my_once_strdup(from->tailoring,MYF(MY_WME)))) + goto err; + + to->strxfrm_multiply= my_charset_ucs2_general_uca.strxfrm_multiply; + to->min_sort_char= my_charset_ucs2_general_uca.min_sort_char; + to->max_sort_char= my_charset_ucs2_general_uca.max_sort_char; + to->mbminlen= 2; + to->mbmaxlen= 2; + + return create_tailoring(to); err: return 1; @@ -848,6 +977,24 @@ static int add_collation(CHARSET_INFO *cs) return MY_XML_OK; } +#ifdef HAVE_CHARSET_ucs2 +static my_bool init_uca_charsets() +{ + my_tailoring *t; + CHARSET_INFO cs= my_charset_ucs2_general_uca; + cs.state= MY_CS_STRNXFRM|MY_CS_UNICODE|MY_CS_NONTEXT; + char name[64]; + for (t= tailoring; t->tailoring; t++) + { + cs.number= 128 + t->number; + cs.tailoring= t->tailoring; + cs.name= name; + sprintf(name, "ucs2_%s_ci", t->name); + add_collation(&cs); + } + return 0; +} +#endif #define MY_MAX_ALLOWED_BUF 1024*1024 #define MY_CHARSET_INDEX "Index.xml" @@ -947,6 +1094,9 @@ static my_bool init_available_charsets(myf myflags) bzero(&all_charsets,sizeof(all_charsets)); init_compiled_charsets(myflags); +#ifdef HAVE_CHARSET_ucs2 + init_uca_charsets(); +#endif /* Copy compiled charsets */ for (cs=all_charsets; diff --git a/strings/ctype-big5.c b/strings/ctype-big5.c index f024fa0cc14..7a3c4503d74 100644 --- a/strings/ctype-big5.c +++ b/strings/ctype-big5.c @@ -6313,6 +6313,7 @@ CHARSET_INFO my_charset_big5_chinese_ci= "big5", /* cs name */ "big5_chinese_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_big5, to_lower_big5, to_upper_big5, @@ -6339,6 +6340,7 @@ CHARSET_INFO my_charset_big5_bin= "big5", /* cs name */ "big5_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_big5, to_lower_big5, to_upper_big5, diff --git a/strings/ctype-bin.c b/strings/ctype-bin.c index 7cac8c7c337..48323018cca 100644 --- a/strings/ctype-bin.c +++ b/strings/ctype-bin.c @@ -378,6 +378,7 @@ CHARSET_INFO my_charset_bin = "binary", /* cs name */ "binary", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_bin, /* ctype */ bin_char_array, /* to_lower */ bin_char_array, /* to_upper */ diff --git a/strings/ctype-czech.c b/strings/ctype-czech.c index 2eb2fac46e9..dede737f361 100644 --- a/strings/ctype-czech.c +++ b/strings/ctype-czech.c @@ -589,6 +589,7 @@ CHARSET_INFO my_charset_latin2_czech_ci = "latin2", /* cs name */ "latin2_czech_cs", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_czech, to_lower_czech, to_upper_czech, diff --git a/strings/ctype-euc_kr.c b/strings/ctype-euc_kr.c index 8f955c15a73..2d4c68978a3 100644 --- a/strings/ctype-euc_kr.c +++ b/strings/ctype-euc_kr.c @@ -8681,6 +8681,7 @@ CHARSET_INFO my_charset_euckr_korean_ci= "euckr", /* cs name */ "euckr_korean_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_euc_kr, to_lower_euc_kr, to_upper_euc_kr, @@ -8707,6 +8708,7 @@ CHARSET_INFO my_charset_euckr_bin= "euckr", /* cs name */ "euckr_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_euc_kr, to_lower_euc_kr, to_upper_euc_kr, diff --git a/strings/ctype-gb2312.c b/strings/ctype-gb2312.c index b76511fc4f3..49ca736a3c2 100644 --- a/strings/ctype-gb2312.c +++ b/strings/ctype-gb2312.c @@ -5732,6 +5732,7 @@ CHARSET_INFO my_charset_gb2312_chinese_ci= "gb2312", /* cs name */ "gb2312_chinese_ci",/* name */ "", /* comment */ + NULL, /* tailoring */ ctype_gb2312, to_lower_gb2312, to_upper_gb2312, @@ -5757,6 +5758,7 @@ CHARSET_INFO my_charset_gb2312_bin= "gb2312", /* cs name */ "gb2312_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_gb2312, to_lower_gb2312, to_upper_gb2312, diff --git a/strings/ctype-gbk.c b/strings/ctype-gbk.c index 0dc00a73fa3..0273feb4c2c 100644 --- a/strings/ctype-gbk.c +++ b/strings/ctype-gbk.c @@ -9962,6 +9962,7 @@ CHARSET_INFO my_charset_gbk_chinese_ci= "gbk", /* cs name */ "gbk_chinese_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_gbk, to_lower_gbk, to_upper_gbk, @@ -9987,6 +9988,7 @@ CHARSET_INFO my_charset_gbk_bin= "gbk", /* cs name */ "gbk_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_gbk, to_lower_gbk, to_upper_gbk, diff --git a/strings/ctype-latin1.c b/strings/ctype-latin1.c index 0b439964c7c..fe39303e2ac 100644 --- a/strings/ctype-latin1.c +++ b/strings/ctype-latin1.c @@ -412,6 +412,7 @@ CHARSET_INFO my_charset_latin1= "latin1", /* cs name */ "latin1_swedish_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_latin1, to_lower_latin1, to_upper_latin1, @@ -690,6 +691,7 @@ CHARSET_INFO my_charset_latin1_german2_ci= "latin1", /* cs name */ "latin1_german2_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_latin1, to_lower_latin1, to_upper_latin1, @@ -715,6 +717,7 @@ CHARSET_INFO my_charset_latin1_bin= "latin1", /* cs name */ "latin1_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_latin1, to_lower_latin1, to_upper_latin1, diff --git a/strings/ctype-sjis.c b/strings/ctype-sjis.c index 72666175a1f..22c58360348 100644 --- a/strings/ctype-sjis.c +++ b/strings/ctype-sjis.c @@ -4579,6 +4579,7 @@ CHARSET_INFO my_charset_sjis_japanese_ci= "sjis", /* cs name */ "sjis_japanese_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_sjis, to_lower_sjis, to_upper_sjis, @@ -4604,6 +4605,7 @@ CHARSET_INFO my_charset_sjis_bin= "sjis", /* cs name */ "sjis_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_sjis, to_lower_sjis, to_upper_sjis, diff --git a/strings/ctype-tis620.c b/strings/ctype-tis620.c index e2a138300c3..b2b1ab98352 100644 --- a/strings/ctype-tis620.c +++ b/strings/ctype-tis620.c @@ -951,6 +951,7 @@ CHARSET_INFO my_charset_tis620_thai_ci= "tis620", /* cs name */ "tis620_thai_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_tis620, to_lower_tis620, to_upper_tis620, @@ -976,6 +977,7 @@ CHARSET_INFO my_charset_tis620_bin= "tis620", /* cs name */ "tis620_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_tis620, to_lower_tis620, to_upper_tis620, diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index 81073d47554..75e2c06eec2 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -7055,6 +7055,7 @@ CHARSET_INFO my_charset_ucs2_general_uca= "ucs2", /* cs name */ "ucs2_general_uca", /* name */ "", /* comment */ + NULL, /* tailoring */ NULL, /* ctype */ NULL, /* to_lower */ NULL, /* to_upper */ diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index 67340fdd4f4..d1ba63b8b84 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -1431,6 +1431,7 @@ CHARSET_INFO my_charset_ucs2_general_ci= "ucs2", /* cs name */ "ucs2_general_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_ucs2, /* ctype */ to_lower_ucs2, /* to_lower */ to_upper_ucs2, /* to_upper */ @@ -1456,6 +1457,7 @@ CHARSET_INFO my_charset_ucs2_bin= "ucs2", /* cs name */ "ucs2_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_ucs2, /* ctype */ to_lower_ucs2, /* to_lower */ to_upper_ucs2, /* to_upper */ diff --git a/strings/ctype-ujis.c b/strings/ctype-ujis.c index fd3692553be..668dc7beb8b 100644 --- a/strings/ctype-ujis.c +++ b/strings/ctype-ujis.c @@ -8468,6 +8468,7 @@ CHARSET_INFO my_charset_ujis_japanese_ci= "ujis", /* cs name */ "ujis_japanese_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_ujis, to_lower_ujis, to_upper_ujis, @@ -8494,6 +8495,7 @@ CHARSET_INFO my_charset_ujis_bin= "ujis", /* cs name */ "ujis_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_ujis, to_lower_ujis, to_upper_ujis, diff --git a/strings/ctype-utf8.c b/strings/ctype-utf8.c index 09b918b0777..2d0feb1c890 100644 --- a/strings/ctype-utf8.c +++ b/strings/ctype-utf8.c @@ -2090,6 +2090,7 @@ CHARSET_INFO my_charset_utf8_general_ci= "utf8", /* cs name */ "utf8_general_ci", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_utf8, /* ctype */ to_lower_utf8, /* to_lower */ to_upper_utf8, /* to_upper */ @@ -2116,6 +2117,7 @@ CHARSET_INFO my_charset_utf8_bin= "utf8", /* cs name */ "utf8_bin", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_utf8, /* ctype */ to_lower_utf8, /* to_lower */ to_upper_utf8, /* to_upper */ diff --git a/strings/ctype-win1250ch.c b/strings/ctype-win1250ch.c index 2eefb570170..bb287eb695e 100644 --- a/strings/ctype-win1250ch.c +++ b/strings/ctype-win1250ch.c @@ -623,6 +623,7 @@ CHARSET_INFO my_charset_cp1250_czech_ci = "cp1250", /* cs name */ "cp1250_czech_cs", /* name */ "", /* comment */ + NULL, /* tailoring */ ctype_win1250ch, to_lower_win1250ch, to_upper_win1250ch, diff --git a/strings/ctype.c b/strings/ctype.c index 44bf20ada5c..4454d3c45e1 100644 --- a/strings/ctype.c +++ b/strings/ctype.c @@ -124,6 +124,7 @@ static struct my_cs_file_section_st * cs_file_sec(const char *attr, uint len) } #define MY_CS_CSDESCR_SIZE 64 +#define MY_CS_TAILORING_SIZE 128 typedef struct my_cs_file_info { @@ -135,7 +136,8 @@ typedef struct my_cs_file_info uchar sort_order[MY_CS_SORT_ORDER_TABLE_SIZE]; uint16 tab_to_uni[MY_CS_TO_UNI_TABLE_SIZE]; char comment[MY_CS_CSDESCR_SIZE]; - size_t sort_order_length; + char tailoring[MY_CS_TAILORING_SIZE]; + size_t tailoring_length; CHARSET_INFO cs; int (*add_collation)(CHARSET_INFO *cs); } MY_CHARSET_LOADER; @@ -186,7 +188,7 @@ static int cs_enter(MY_XML_PARSER *st,const char *attr, uint len) bzero(&i->cs,sizeof(i->cs)); if (s && (s->state == _CS_COLLATION)) - i->sort_order_length= 0; + i->tailoring_length= 0; return MY_XML_OK; } @@ -283,12 +285,12 @@ static int cs_value(MY_XML_PARSER *st,const char *attr, uint len) */ char arg[16]; const char *cmd[]= {"&","<","<<","<<<"}; - i->cs.sort_order= i->sort_order; + i->cs.tailoring= i->tailoring; mstr(arg,attr,len,sizeof(arg)-1); - if (i->sort_order_length + 20 < sizeof(i->sort_order)) + if (i->tailoring_length + 20 < sizeof(i->tailoring)) { - char *dst= i->sort_order_length + i->sort_order; - i->sort_order_length+= sprintf(dst," %s %s",cmd[state-_CS_RESET],arg); + char *dst= i->tailoring_length + i->tailoring; + i->tailoring_length+= sprintf(dst," %s %s",cmd[state-_CS_RESET],arg); } } } |