summaryrefslogtreecommitdiff
path: root/strings/uca-dump.c
diff options
context:
space:
mode:
authorunknown <pem@mysql.com>2004-04-07 19:07:44 +0200
committerunknown <pem@mysql.com>2004-04-07 19:07:44 +0200
commitc635d37479fbf8aa0da03d47f264ab9907cfe72b (patch)
treee5ac1517ff3dccc42352b7bac39ecd1c093d0ae6 /strings/uca-dump.c
parent8468cde4f1e028b380c24d0bac28402678568d81 (diff)
parent0948f9769b3e3da2e064e65015b86644af4acd78 (diff)
downloadmariadb-git-c635d37479fbf8aa0da03d47f264ab9907cfe72b.tar.gz
Merge 4.1 -> 5.0.
BitKeeper/etc/ignore: auto-union BitKeeper/etc/logging_ok: auto-union VC++Files/sql/mysqld.dsp: Auto merged client/Makefile.am: Auto merged client/mysql.cc: Auto merged client/mysqltest.c: Auto merged configure.in: Auto merged include/my_global.h: Auto merged include/my_pthread.h: Auto merged include/my_sys.h: Auto merged include/mysql_com.h: Auto merged libmysqld/Makefile.am: Auto merged libmysqld/lib_sql.cc: Auto merged myisam/mi_check.c: Auto merged myisam/myisamchk.c: Auto merged myisam/myisamdef.h: Auto merged mysql-test/install_test_db.sh: Auto merged mysql-test/r/insert_select.result: Auto merged mysql-test/r/join_outer.result: Auto merged mysql-test/r/null.result: Auto merged mysql-test/r/connect.result: Auto merged mysql-test/r/func_sapdb.result: Auto merged mysql-test/r/mix_innodb_myisam_binlog.result: Auto merged mysql-test/r/mysqldump.result: Auto merged mysql-test/r/rpl_change_master.result: Auto merged mysql-test/r/rpl_log.result: Auto merged mysql-test/r/show_check.result: Auto merged mysql-test/r/symlink.result: Auto merged mysql-test/r/rpl_flush_log_loop.result: Auto merged mysql-test/r/rpl_flush_tables.result: Auto merged mysql-test/r/rpl_loaddata.result: Auto merged mysql-test/r/rpl_loaddata_rule_m.result: Auto merged mysql-test/r/rpl_loaddata_rule_s.result: Auto merged mysql-test/r/rpl_max_relay_size.result: Auto merged mysql-test/r/rpl_reset_slave.result: Auto merged mysql-test/r/rpl_temporary.result: Auto merged mysql-test/r/rpl_until.result: Auto merged mysql-test/r/rpl_user_variables.result: Auto merged mysql-test/r/subselect.result: Auto merged mysql-test/r/union.result: Auto merged mysql-test/r/variables.result: Auto merged mysql-test/t/func_sapdb.test: Auto merged mysql-test/t/mix_innodb_myisam_binlog.test: Auto merged mysql-test/t/mysqlbinlog.test: Auto merged mysql-test/t/rpl_change_master.test: Auto merged mysql-test/t/rpl_log.test: Auto merged mysql-test/t/variables.test: Auto merged netware/BUILD/compile-linux-tools: Auto merged netware/BUILD/compile-netware-END: Auto merged netware/BUILD/compile-netware-all: Auto merged netware/BUILD/compile-netware-standard: Auto merged mysql-test/t/rpl_empty_master_crash.test: Auto merged mysql-test/t/rpl_error_ignored_table.test: Auto merged mysql-test/t/rpl_flush_log_loop.test: Auto merged mysql-test/t/rpl_loaddata.test: Auto merged mysql-test/t/rpl_loaddata_rule_m.test: Auto merged mysql-test/t/rpl_loaddata_rule_s.test: Auto merged mysql-test/t/rpl_max_relay_size.test: Auto merged mysql-test/t/rpl_openssl.test: Auto merged mysql-test/t/rpl_relayrotate-slave.opt: Auto merged mysql-test/t/rpl_reset_slave.test: Auto merged mysql-test/t/rpl_trunc_binlog.test: Auto merged mysql-test/t/rpl_until.test: Auto merged mysql-test/t/rpl_user_variables.test: Auto merged mysql-test/t/subselect.test: Auto merged scripts/make_binary_distribution.sh: Auto merged scripts/mysql_create_system_tables.sh: Auto merged scripts/mysql_fix_privilege_tables.sql: Auto merged scripts/mysql_install_db.sh: Auto merged sql/filesort.cc: Auto merged sql/ha_berkeley.cc: Auto merged sql/ha_berkeley.h: Auto merged sql/ha_innodb.h: Auto merged sql/ha_myisam.cc: Auto merged sql/handler.cc: Auto merged sql/handler.h: Auto merged sql/item.cc: Auto merged sql/item.h: Auto merged sql/item_cmpfunc.cc: Auto merged sql/item_cmpfunc.h: Auto merged sql/item_create.cc: Auto merged sql/item_create.h: Auto merged sql/item_subselect.cc: Auto merged sql/item_sum.cc: Auto merged sql/item_sum.h: Auto merged sql/item_timefunc.h: Auto merged sql/lock.cc: Auto merged sql/log_event.h: Auto merged sql/mysqld.cc: Auto merged sql/net_serv.cc: Auto merged sql/protocol.cc: Auto merged sql/protocol.h: Auto merged sql/repl_failsafe.cc: Auto merged sql/sql_acl.cc: Auto merged sql/sql_acl.h: Auto merged sql/sql_base.cc: Auto merged sql/sql_cache.cc: Auto merged sql/sql_class.h: Auto merged sql/sql_db.cc: Auto merged sql/sql_delete.cc: Auto merged sql/sql_derived.cc: Auto merged sql/sql_lex.h: Auto merged sql/sql_list.h: Auto merged sql/sql_load.cc: Auto merged sql/sql_rename.cc: Auto merged sql/sql_repl.cc: Auto merged sql/sql_repl.h: Auto merged sql/sql_select.cc: Auto merged sql/sql_select.h: Auto merged sql/sql_show.cc: Auto merged sql/sql_table.cc: Auto merged sql/sql_test.cc: Auto merged sql/sql_update.cc: Auto merged sql-common/client.c: Auto merged tests/client_test.c: Auto merged
Diffstat (limited to 'strings/uca-dump.c')
-rw-r--r--strings/uca-dump.c276
1 files changed, 276 insertions, 0 deletions
diff --git a/strings/uca-dump.c b/strings/uca-dump.c
new file mode 100644
index 00000000000..6836c321526
--- /dev/null
+++ b/strings/uca-dump.c
@@ -0,0 +1,276 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+typedef unsigned char uchar;
+typedef unsigned short uint16;
+
+struct uca_item_st
+{
+ uchar num;
+ uint16 weight[4][9];
+};
+
+#if 0
+#define MY_UCA_NPAGES 1024
+#define MY_UCA_NCHARS 64
+#define MY_UCA_CMASK 63
+#define MY_UCA_PSHIFT 6
+#else
+#define MY_UCA_NPAGES 256
+#define MY_UCA_NCHARS 256
+#define MY_UCA_CMASK 255
+#define MY_UCA_PSHIFT 8
+#endif
+
+int main(int ac, char **av)
+{
+ char str[256];
+ char *weights[64];
+ struct uca_item_st uca[64*1024];
+ size_t code, page, w;
+ int pagemaxlen[MY_UCA_NPAGES];
+ int pageloaded[MY_UCA_NPAGES];
+
+ bzero(uca, sizeof(uca));
+ bzero(pageloaded, sizeof(pageloaded));
+
+ while (fgets(str,sizeof(str),stdin))
+ {
+ char *comment;
+ char *weight;
+ char *s;
+ size_t codenum;
+
+ code= strtol(str,NULL,16);
+
+ if (str[0]=='#' || (code > 0xFFFF))
+ continue;
+ if ((comment=strchr(str,'#')))
+ {
+ *comment++= '\0';
+ for ( ; *comment==' ' ; comment++);
+ }else
+ continue;
+
+ if ((weight=strchr(str,';')))
+ {
+ *weight++= '\0';
+ for ( ; *weight==' ' ; weight++);
+ }
+ else
+ continue;
+
+ codenum= 0;
+ s= strtok(str, " \t");
+ while (s)
+ {
+ s= strtok(NULL, " \t");
+ codenum++;
+ }
+
+ if (codenum>1)
+ {
+ /* Multi-character weight,
+ i.e. contraction.
+ Not supported yet.
+ */
+ continue;
+ }
+
+ uca[code].num= 0;
+ s= strtok(weight, " []");
+ while (s)
+ {
+ weights[uca[code].num]= s;
+ s= strtok(NULL, " []");
+ uca[code].num++;
+ }
+
+ for (w=0; w < uca[code].num; w++)
+ {
+ size_t partnum;
+
+ partnum= 0;
+ s= weights[w];
+ while (*s)
+ {
+ char *endptr;
+ size_t part;
+ part= strtol(s+1,&endptr,16);
+ uca[code].weight[partnum][w]= part;
+ s= endptr;
+ partnum++;
+ }
+ }
+ /* Mark that a character from this page was loaded */
+ pageloaded[code >> MY_UCA_PSHIFT]++;
+ }
+
+
+
+ /* Now set implicit weights */
+ for (code=0; code <= 0xFFFF; code++)
+ {
+ size_t base, aaaa, bbbb;
+
+ if (uca[code].num)
+ continue;
+
+ /*
+ 3400;<CJK Ideograph Extension A, First>
+ 4DB5;<CJK Ideograph Extension A, Last>
+ 4E00;<CJK Ideograph, First>
+ 9FA5;<CJK Ideograph, Last>
+ */
+
+ if (code >= 0x3400 && code <= 0x4DB5)
+ base= 0xFB80;
+ else if (code >= 0x4E00 && code <= 0x9FA5)
+ base= 0xFB40;
+ else
+ base= 0xFBC0;
+
+ aaaa= base + (code >> 15);
+ bbbb= (code & 0x7FFF) | 0x8000;
+ uca[code].weight[0][0]= aaaa;
+ uca[code].weight[0][1]= bbbb;
+
+ uca[code].weight[1][0]= 0x0020;
+ uca[code].weight[1][1]= 0x0000;
+
+ uca[code].weight[2][0]= 0x0002;
+ uca[code].weight[2][1]= 0x0000;
+
+ uca[code].weight[3][0]= 0x0001;
+ uca[code].weight[3][2]= 0x0000;
+
+ uca[code].num= 2;
+ }
+
+ printf("#include \"my_uca.h\"\n");
+
+ printf("#define MY_UCA_NPAGES %d\n",MY_UCA_NPAGES);
+ printf("#define MY_UCA_NCHARS %d\n",MY_UCA_NCHARS);
+ printf("#define MY_UCA_CMASK %d\n",MY_UCA_CMASK);
+ printf("#define MY_UCA_PSHIFT %d\n",MY_UCA_PSHIFT);
+
+ for (w=0; w<1; w++)
+ {
+ for (page=0; page < MY_UCA_NPAGES; page++)
+ {
+ size_t offs;
+ size_t maxnum= 0;
+ size_t nchars= 0;
+ size_t mchars;
+
+ /*
+ Skip this page if no weights were loaded
+ */
+
+ if (!pageloaded[page])
+ continue;
+
+ /*
+ Calculate maximum weight
+ length for this page
+ */
+
+ for (offs=0; offs < MY_UCA_NCHARS; offs++)
+ {
+ size_t i, num;
+
+ code= page*MY_UCA_NCHARS+offs;
+
+ /* Calculate only non-zero weights */
+ num=0;
+ for (i=0; i < uca[code].num; i++)
+ if (uca[code].weight[w][i])
+ num++;
+
+ maxnum= maxnum < num ? num : maxnum;
+ }
+ maxnum++;
+
+ switch (maxnum)
+ {
+ case 0: mchars= 8; break;
+ case 1: mchars= 8; break;
+ case 2: mchars= 8; break;
+ case 3: mchars= 9; break;
+ case 4: mchars= 8; break;
+ default: mchars= uca[code].num;
+ }
+
+ pagemaxlen[page]= maxnum;
+
+
+ /*
+ Now print this page
+ */
+
+
+ printf("uint16 page%03Xdata[]= { /* %04X (%d weights per char) */\n",
+ page, page*MY_UCA_NCHARS, maxnum);
+
+ for (offs=0; offs < MY_UCA_NCHARS; offs++)
+ {
+ uint16 weight[8];
+ size_t num, i;
+
+ code= page*MY_UCA_NCHARS+offs;
+
+ bzero(weight,sizeof(weight));
+
+ /* Copy non-zero weights */
+ for (num=0, i=0; i < uca[code].num; i++)
+ {
+ if (uca[code].weight[w][i])
+ {
+ weight[num]= uca[code].weight[w][i];
+ num++;
+ }
+ }
+
+ for (i=0; i < maxnum; i++)
+ {
+ printf("0x%04X",(int)weight[i]);
+ if ((offs+1 != MY_UCA_NCHARS) || (i+1!=maxnum))
+ printf(",");
+ nchars++;
+ }
+ if (nchars >=mchars)
+ {
+ printf("\n");
+ nchars=0;
+ }
+ else
+ {
+ printf(" ");
+ }
+ }
+ printf("};\n\n");
+ }
+ }
+
+ printf("uchar ucal[%d]={\n",MY_UCA_NPAGES);
+ for (page=0; page < MY_UCA_NPAGES; page++)
+ {
+ printf("%d%s%s",pagemaxlen[page],page<MY_UCA_NPAGES-1?",":"",(page+1) % 16 ? "":"\n");
+ }
+ printf("};\n");
+
+
+ printf("uint16 *ucaw[%d]={\n",MY_UCA_NPAGES);
+ for (page=0; page < MY_UCA_NPAGES; page++)
+ {
+ if (!pageloaded[page])
+ printf("NULL %s%s",page<MY_UCA_NPAGES-1?",":"", (page+1) % 4 ? "":"\n");
+ else
+ printf("page%03Xdata%s%s",page,page<MY_UCA_NPAGES-1?",":"", (page+1) % 4 ? "":"\n");
+ }
+ printf("};\n");
+
+ printf("int main(void){ return 0;};\n");
+ return 0;
+}