Unicode::Collate 0.11 Released

Message-Id: <20020503030904.35B3.BQW10602@nifty.com>

p4raw-id: //depot/perl@16357

4 files changed, 428 insertions, 48 deletions

diff --git a/lib/Unicode/Collate/Changes b/lib/Unicode/Collate/Changes
index d0ad3c124e..bb207d86b0 100644
--- a/lib/Unicode/Collate/Changes
+++ b/lib/Unicode/Collate/Changes
@@ -1,5 +1,16 @@
 Revision history for Perl extension Unicode::Collate.
 
+0.11  Fri May 03 02:28:10 2002
+    - fix: now derived collation elements can be used for Hangul Jamo
+      when their weights are not defined.
+      [reported by andreas.koenig@anima.de]
+    - fix: rearrangements had not worked.
+    - mentioned pleblem on index() in BUGS.
+    - more documents, more tests.
+    - tag names for 'alternate' are case-insensitive (i.e. 'SHIFTed' etc.).
+    - The <undef> value for the keys "overrideCJK", "overrideHangul",
+      "rearrange" has a special behavior (different from default).
+
 0.10  Tue Dec 11 23:26:42 2001
     - now you are allowed to use no table file.
     - fix: fetching CE with two or more combining characters.
diff --git a/lib/Unicode/Collate/README b/lib/Unicode/Collate/README
index 3ff073f3d4..68f7b80369 100644
--- a/lib/Unicode/Collate/README
+++ b/lib/Unicode/Collate/README
@@ -1,16 +1,17 @@
-Unicode/Collate version 0.10
+Unicode/Collate version 0.11
 ===============================
 
-use UCA (Unicode Collation Algorithm)
+Unicode::Collate - Unicode Collation Algorithm
 
-  See UCA - Unicode TR #10.
+  UCA - Unicode TR #10.
   http://www.unicode.org/unicode/reports/tr10/
 
-  Fetch the following file before use and put it into the Unicode/Collate dir.
+  Fetch the following file and put it into the Unicode/Collate directory.
   http://www.unicode.org/unicode/reports/tr10/allkeys.txt
 
   You can install this module using a subset "keys.txt"
-  contained in this distribution without the "allkeys.txt".
+  contained in this distribution without the "allkeys.txt";
+  but "keys.txt" is small and not very useful.
 
 SYNOPSIS
 
@@ -40,8 +41,8 @@ To install this module type the following:
 
 DEPENDENCIES
 
-It's better if you have Unicode::Normalize (v 0.10 or later)
-although Unicode::Collate can be used without Unicode::Normalize.
+  It's better if you have Unicode::Normalize (v 0.10 or later)
+  although Unicode::Collate can be used without Unicode::Normalize.
 
 COPYRIGHT AND LICENCE
 
@@ -49,8 +50,8 @@ SADAHIRO Tomoyuki <bqw10602@nifty.com>
 
   http://homepage1.nifty.com/nomenclator/perl/
 
-  Copyright(C) 2001, SADAHIRO Tomoyuki. Japan. All rights reserved.
+  Copyright(C) 2001-2002, SADAHIRO Tomoyuki. Japan. All rights reserved.
 
-  This program is free software; you can redistribute it and/or 
-  modify it under the same terms as Perl itself.
+  This library is free software; you can redistribute it
+  and/or modify it under the same terms as Perl itself.
 
diff --git a/lib/Unicode/Collate/keys.txt b/lib/Unicode/Collate/keys.txt
index 22ce6735d9..5fe3ebef62 100644
--- a/lib/Unicode/Collate/keys.txt
+++ b/lib/Unicode/Collate/keys.txt
@@ -1,5 +1,5 @@
 #### This file is generated from allkeys-3.0.1d9.txt (unicode.org)
-#### by deleting mamy many elements.
+#### by deleting many many elements.
 ####
 #### Remaining elements include LATIN, HANGUL, HIRAGANA, KATAKANA,
 #### BOPOMOFO, CJK UNIFIED IDEOGRAPHS.
diff --git a/lib/Unicode/Collate/t/test.t b/lib/Unicode/Collate/t/test.t
index 57e20579cb..957bad1f61 100644
--- a/lib/Unicode/Collate/t/test.t
+++ b/lib/Unicode/Collate/t/test.t
@@ -11,7 +11,7 @@ BEGIN {
 }
 
 use Test;
-BEGIN { plan tests => 54 };
+BEGIN { plan tests => 156 };
 use Unicode::Collate;
 ok(1); # If we made it this far, we're ok.
 
@@ -37,16 +37,16 @@ my $A_acute = pack('U', 0x00C1);
 my $acute   = pack('U', 0x0301);
 
 ok($Collator->cmp("A$acute", $A_acute), -1);
-
 ok($Collator->cmp("", ""), 0);
 ok(! $Collator->ne("", "") );
 ok(  $Collator->eq("", "") );
-
 ok($Collator->cmp("", "perl"), -1);
 
-eval "use Unicode::Normalize";
+##############
+
+eval { require Unicode::Normalize };
 
-if(!$@){
+if (!$@) {
   my $NFD = Unicode::Collate->new(
     table => 'keys.txt',
     entry => <<'ENTRIES',
@@ -69,7 +69,7 @@ ENTRIES
   ok($NFD->eq("\x{0430}\x{3099}\x{309A}\x{0308}",
               "\x{0430}\x{309A}\x{3099}\x{0308}") );
 }
-else{
+else {
   ok(1);
   ok(1);
   ok(1);
@@ -77,39 +77,43 @@ else{
   ok(1);
 }
 
-my $tr = Unicode::Collate->new(
+##############
+
+my $trad = Unicode::Collate->new(
   table => 'keys.txt',
   normalization => undef,
-  ignoreName => qr/^(?:HANGUL|HIRAGANA|KATAKANA|BOPOMOFO)$/,
-  entry => <<'ENTRIES',
-0063 0068 ; [.0893.0020.0002.0063]  # "ch" in traditional Spanish
-0043 0068 ; [.0893.0020.0008.0043]  # "Ch" in traditional Spanish
-00DF ; [.09F3.0154.0004.00DF] [.09F3.0020.0004.00DF] # eszet in Germany
+  ignoreName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
+  level => 4,
+  entry => << 'ENTRIES',
+ 0063 0068 ; [.0893.0020.0002.0063] % "ch" in traditional Spanish
+ 0043 0068 ; [.0893.0020.0008.0043] # "Ch" in traditional Spanish
+ 00DF ; [.09F3.0154.0004.00DF] [.09F3.0020.0004.00DF] # eszet in Germany
 ENTRIES
 );
 
 ok(
-  join(':', $tr->sort( 
-    qw/ acha aca ada acia acka /
-  ) ),
-  join(':',
-    qw/ aca acia acka acha ada /
-  ),
+  join(':', $trad->sort( qw/ acha aca ada acia acka / ) ),
+  join(':',              qw/ aca acia acka acha ada / ),
 );
 
 ok(
-  join(':', $Collator->sort( 
-    qw/ acha aca ada acia acka /
-  ) ),
-  join(':',
-    qw/ aca acha acia acka ada /
-  ),
+  join(':', $Collator->sort( qw/ acha aca ada acia acka / ) ),
+  join(':',                  qw/ aca acha acia acka ada / ),
 );
 
-my $old_level = $Collator->{level};
 my $hiragana = "\x{3042}\x{3044}";
 my $katakana = "\x{30A2}\x{30A4}";
 
+# HIRAGANA and KATAKANA are ignorable via ignoreName
+ok($trad->eq($hiragana, ""));
+ok($trad->eq("", $katakana));
+ok($trad->eq($hiragana, $katakana));
+ok($trad->eq($katakana, $hiragana));
+
+##############
+
+my $old_level = $Collator->{level};
+
 $Collator->{level} = 2;
 
 ok( $Collator->cmp("ABC","abc"), 0);
@@ -161,13 +165,50 @@ ok( $Collator->le("abc", "ABC") );
 ok( $Collator->cmp($hiragana, $katakana), -1);
 ok( $Collator->lt($hiragana, $katakana) );
 
-my $ign = Unicode::Collate->new(
+##############
+
+my $ignoreAE = Unicode::Collate->new(
+  table => 'keys.txt',
+  normalization => undef,
+  ignoreChar => qr/^[aAeE]$/,
+);
+
+ok($ignoreAE->eq("element","lament"));
+ok($ignoreAE->eq("Perl","ePrl"));
+
+##############
+
+my $onlyABC = Unicode::Collate->new(
+    table => undef,
+    entry => << 'ENTRIES',
+0061 ; [.0101.0020.0002.0061] # LATIN SMALL LETTER A
+0041 ; [.0101.0020.0008.0041] # LATIN CAPITAL LETTER A
+0062 ; [.0102.0020.0002.0062] # LATIN SMALL LETTER B
+0042 ; [.0102.0020.0008.0042] # LATIN CAPITAL LETTER B
+0063 ; [.0103.0020.0002.0063] # LATIN SMALL LETTER C
+0043 ; [.0103.0020.0008.0043] # LATIN CAPITAL LETTER C
+ENTRIES
+);
+
+ok(
+  join(':', $onlyABC->sort( qw/ ABA BAC cc A Ab cAc aB / ) ),
+  join(':',                 qw/ A aB Ab ABA BAC cAc cc / ),
+);
+
+##############
+
+my $undefAE = Unicode::Collate->new(
   table => 'keys.txt',
   normalization => undef,
-  ignoreChar => qr/^[ae]$/,
+  undefChar => qr/^[aAeE]$/,
 );
 
-ok( $ign->cmp("element","lament"), 0);
+ok($undefAE ->gt("edge","fog"));
+ok($Collator->lt("edge","fog"));
+ok($undefAE ->gt("lake","like"));
+ok($Collator->lt("lake","like"));
+
+##############
 
 $Collator->{level} = 2;
 
@@ -179,7 +220,7 @@ my $rep = "camel";
 my $ret = "This is a camel book.";
 
 $str = $orig;
-if(my($pos,$len) = $Collator->index($str, $sub)){
+if (my($pos,$len) = $Collator->index($str, $sub)) {
   substr($str, $pos, $len, $rep);
 }
 
@@ -188,41 +229,368 @@ ok($str, $ret);
 $Collator->{level} = $old_level;
 
 $str = $orig;
-if(my($pos,$len) = $Collator->index($str, $sub)){
+if (my($pos,$len) = $Collator->index($str, $sub)) {
   substr($str, $pos, $len, $rep);
 }
 
 ok($str, $orig);
 
-$tr->{level} = 1;
+##############
+
+my $match;
+
+$Collator->{level} = 1;
+
+$str = "Pe\x{300}rl";
+$sub = "pe";
+$match = undef;
+if (my($pos, $len) = $Collator->index($str, $sub)) {
+    $match = substr($str, $pos, $len);
+}
+ok($match, "Pe\x{300}");
+
+$str = "P\x{300}e\x{300}\x{301}\x{303}rl";
+$sub = "pE";
+$match = undef;
+if (my($pos, $len) = $Collator->index($str, $sub)) {
+    $match = substr($str, $pos, $len);
+}
+ok($match, "P\x{300}e\x{300}\x{301}\x{303}");
+
+$Collator->{level} = $old_level;
+
+##############
+
+$trad->{level} = 1;
 
 $str = "Ich mu\x{00DF} studieren.";
 $sub = "m\x{00FC}ss";
-my $match = undef;
-if(my($pos, $len) = $tr->index($str, $sub)){
+$match = undef;
+if (my($pos, $len) = $trad->index($str, $sub)) {
     $match = substr($str, $pos, $len);
 }
 ok($match, "mu\x{00DF}");
 
-$tr->{level} = $old_level;
+$trad->{level} = $old_level;
 
 $str = "Ich mu\x{00DF} studieren.";
 $sub = "m\x{00FC}ss";
 $match = undef;
-if(my($pos, $len) = $tr->index($str, $sub)){
+
+if (my($pos, $len) = $trad->index($str, $sub)) {
     $match = substr($str, $pos, $len);
 }
 ok($match, undef);
 
 $match = undef;
-if(my($pos,$len) = $Collator->index("", "")){
+if (my($pos,$len) = $Collator->index("", "")) {
     $match = substr("", $pos, $len);
 }
 ok($match, "");
 
 $match = undef;
-if(my($pos,$len) = $Collator->index("", "abc")){
+if (my($pos,$len) = $Collator->index("", "abc")) {
     $match = substr("", $pos, $len);
 }
 ok($match, undef);
 
+##############
+
+# Table is undefined, then no entry is defined.
+
+my $undef_table = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  level => 1,
+);
+
+# in the Unicode code point order
+ok($undef_table->lt('', 'A'));
+ok($undef_table->lt('ABC', 'B'));
+
+# Hangul should be decomposed (even w/o Unicode::Normalize).
+
+ok($undef_table->lt("Perl", "\x{AC00}"));
+ok($undef_table->eq("\x{AC00}", "\x{1100}\x{1161}"));
+ok($undef_table->eq("\x{AE00}", "\x{1100}\x{1173}\x{11AF}"));
+ok($undef_table->lt("\x{AE00}", "\x{3042}"));
+  # U+AC00: Hangul GA
+  # U+AE00: Hangul GEUL
+  # U+3042: Hiragana A
+
+# Weight for CJK Ideographs is defined, though.
+
+ok($undef_table->lt("", "\x{4E00}"));
+ok($undef_table->lt("\x{4E8C}","ABC"));
+ok($undef_table->lt("\x{4E00}","\x{3042}"));
+ok($undef_table->lt("\x{4E00}","\x{4E8C}"));
+  # U+4E00: Ideograph "ONE"
+  # U+4E8C: Ideograph "TWO"
+
+
+##############
+
+my $few_entries = Unicode::Collate->new(
+  entry => <<'ENTRIES',
+0050 ; [.0101.0020.0002.0050]  # P
+0045 ; [.0102.0020.0002.0045]  # E
+0052 ; [.0103.0020.0002.0052]  # R
+004C ; [.0104.0020.0002.004C]  # L
+1100 ; [.0105.0020.0002.1100]  # Hangul Jamo initial G
+1175 ; [.0106.0020.0002.1175]  # Hangul Jamo middle I
+5B57 ; [.0107.0020.0002.5B57]  # CJK Ideograph "Letter"
+ENTRIES
+  table => undef,
+  normalization => undef,
+);
+
+# defined before undefined
+
+my $sortABC = join '',
+    $few_entries->sort(split //, "ABCDEFGHIJKLMNOPQRSTUVWXYZ ");
+
+ok($sortABC eq "PERL ABCDFGHIJKMNOQSTUVWXYZ");
+
+ok($few_entries->lt('E', 'D'));
+ok($few_entries->lt("\x{5B57}", "\x{4E00}"));
+ok($few_entries->lt("\x{AE30}", "\x{AC00}"));
+
+# Hangul must be decomposed.
+
+ok($few_entries->eq("\x{AC00}", "\x{1100}\x{1161}"));
+
+##############
+
+my $all_undef = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  overrideCJK => undef,
+  overrideHangul => undef,
+);
+
+# All in the Unicode code point order.
+# No hangul decomposition.
+
+ok($all_undef->lt("\x{3042}", "\x{4E00}"));
+ok($all_undef->lt("\x{4DFF}", "\x{4E00}"));
+ok($all_undef->lt("\x{4E00}", "\x{AC00}"));
+ok($all_undef->gt("\x{AC00}", "\x{1100}\x{1161}"));
+ok($all_undef->gt("\x{AC00}", "\x{ABFF}"));
+
+##############
+
+my $ignoreCJK = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  overrideCJK => sub {()},
+  entry => <<'ENTRIES',
+5B57 ; [.0107.0020.0002.5B57]  # CJK Ideograph "Letter"
+ENTRIES
+);
+
+# All CJK Unified Ideographs except U+5B57 are ignored.
+
+ok($ignoreCJK->eq("\x{4E00}", ""));
+ok($ignoreCJK->lt("\x{4E00}", "\0"));
+ok($ignoreCJK->eq("Pe\x{4E00}rl", "Perl")); # U+4E00 is a CJK.
+ok($ignoreCJK->gt("\x{4DFF}", "\x{4E00}")); # U+4DFF is not CJK.
+ok($ignoreCJK->lt("Pe\x{5B57}rl", "Perl")); # 'r' is unassigned.
+
+##############
+
+my $ignoreHangul = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  overrideHangul => sub {()},
+  entry => <<'ENTRIES',
+AE00 ; [.0100.0020.0002.AE00]  # Hangul GEUL
+ENTRIES
+);
+
+# All Hangul Syllables except U+AE00 are ignored.
+
+ok($ignoreHangul->eq("\x{AC00}", ""));
+ok($ignoreHangul->lt("\x{AC00}", "\0"));
+ok($ignoreHangul->lt("\x{AC00}", "\x{AE00}"));
+ok($ignoreHangul->lt("\x{AC00}", "\x{1100}\x{1161}")); # Jamo are not ignored.
+ok($ignoreHangul->lt("Pe\x{AE00}rl", "Perl")); # 'r' is unassigned.
+
+##############
+
+my $blanked = Unicode::Collate->new(
+  table => 'keys.txt',
+  normalization => undef,
+  alternate => 'Blanked',
+);
+
+ok($blanked->lt("death", "de luge"));
+ok($blanked->lt("de luge", "de-luge"));
+ok($blanked->lt("de-luge", "deluge"));
+ok($blanked->lt("deluge", "de\x{2010}luge"));
+ok($blanked->lt("deluge", "de Luge"));
+
+##############
+
+my $nonIgn = Unicode::Collate->new(
+  table => 'keys.txt',
+  normalization => undef,
+  alternate => 'Non-ignorable',
+);
+
+ok($nonIgn->lt("de luge", "de Luge"));
+ok($nonIgn->lt("de Luge", "de-luge"));
+ok($nonIgn->lt("de-Luge", "de\x{2010}luge"));
+ok($nonIgn->lt("de-luge", "death"));
+ok($nonIgn->lt("death", "deluge"));
+
+##############
+
+my $shifted = Unicode::Collate->new(
+  table => 'keys.txt',
+  normalization => undef,
+  alternate => 'Shifted',
+);
+
+ok($shifted->lt("death", "de luge"));
+ok($shifted->lt("de luge", "de-luge"));
+ok($shifted->lt("de-luge", "deluge"));
+ok($shifted->lt("deluge", "de Luge"));
+ok($shifted->lt("de Luge", "deLuge"));
+
+##############
+
+my $shTrim = Unicode::Collate->new(
+  table => 'keys.txt',
+  normalization => undef,
+  alternate => 'Shift-Trimmed',
+);
+
+ok($shTrim->lt("death", "deluge"));
+ok($shTrim->lt("deluge", "de luge"));
+ok($shTrim->lt("de luge", "de-luge"));
+ok($shTrim->lt("de-luge", "deLuge"));
+ok($shTrim->lt("deLuge", "de Luge"));
+
+##############
+
+my $overCJK = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  entry => <<'ENTRIES',
+0061 ; [.0101.0020.0002.0061] # latin a
+0041 ; [.0101.0020.0008.0041] # LATIN A
+4E00 ; [.B1FC.0030.0004.4E00] # Ideograph; B1FC = FFFF - 4E03.
+ENTRIES
+  overrideCJK => sub {
+    my $u = 0xFFFF - $_[0]; # reversed
+    [$u, 0x20, 0x2, $u];
+  },
+);
+
+ok($overCJK->lt("a", "A")); # diff. at level 3.
+ok($overCJK->lt( "\x{4E03}",  "\x{4E00}")); # diff. at level 2.
+ok($overCJK->lt("A\x{4E03}", "A\x{4E00}"));
+ok($overCJK->lt("A\x{4E03}", "a\x{4E00}"));
+ok($overCJK->lt("a\x{4E03}", "A\x{4E00}"));
+
+##############
+
+# rearranged : 0x0E40..0x0E44, 0x0EC0..0x0EC4
+
+ok($Collator->lt("A", "B"));
+ok($Collator->lt("\x{0E40}", "\x{0E41}"));
+ok($Collator->lt("\x{0E40}A", "\x{0E41}B"));
+ok($Collator->lt("\x{0E41}A", "\x{0E40}B"));
+ok($Collator->lt("A\x{0E41}A", "A\x{0E40}B"));
+
+ok($all_undef->lt("A", "B"));
+ok($all_undef->lt("\x{0E40}", "\x{0E41}"));
+ok($all_undef->lt("\x{0E40}A", "\x{0E41}B"));
+ok($all_undef->lt("\x{0E41}A", "\x{0E40}B"));
+ok($all_undef->lt("A\x{0E41}A", "A\x{0E40}B"));
+
+##############
+
+my $no_rearrange = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  rearrange => [],
+);
+
+ok($no_rearrange->lt("A", "B"));
+ok($no_rearrange->lt("\x{0E40}", "\x{0E41}"));
+ok($no_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
+ok($no_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
+ok($no_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
+
+##############
+
+# equivalent to $no_rearrange
+
+my $undef_rearrange = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  rearrange => undef,
+);
+
+ok($undef_rearrange->lt("A", "B"));
+ok($undef_rearrange->lt("\x{0E40}", "\x{0E41}"));
+ok($undef_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
+ok($undef_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
+ok($undef_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
+
+##############
+
+my $dropArticles = Unicode::Collate->new(
+  table => "keys.txt",
+  normalization => undef,
+  preprocess => sub {
+    my $string = shift;
+    $string =~ s/\b(?:an?|the)\s+//ig;
+    $string;
+  },
+);
+
+ok($dropArticles->eq("camel", "a    camel"));
+ok($dropArticles->eq("Perl", "The Perl"));
+ok($dropArticles->lt("the pen", "a pencil"));
+ok($Collator->lt("Perl", "The Perl"));
+ok($Collator->gt("the pen", "a pencil"));
+
+##############
+
+my $backLevel1 = Unicode::Collate->new(
+  table => undef,
+  normalization => undef,
+  backwards => [ 1 ],
+);
+
+# all strings are reversed at level 1.
+
+ok($backLevel1->gt("AB", "BA"));
+ok($backLevel1->gt("\x{3042}\x{3044}", "\x{3044}\x{3042}"));
+
+##############
+
+my $backLevel2 = Unicode::Collate->new(
+  table => "keys.txt",
+  normalization => undef,
+  undefName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
+  backwards => 2,
+);
+
+ok($backLevel2->gt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
+ok($backLevel2->gt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
+ok($Collator  ->lt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
+ok($Collator  ->lt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
+
+
+# HIRAGANA and KATAKANA are undefined via ignoreName.
+# So they are after CJK Ideographs.
+
+ok($backLevel2->lt("\x{4E00}", $hiragana));
+ok($backLevel2->lt("\x{4E03}", $katakana));
+ok($Collator  ->gt("\x{4E00}", $hiragana));
+ok($Collator  ->gt("\x{4E03}", $katakana));
+
+##############