2002-03-04 Eric Blake <ebb9@email.byu.edu>

	* scripts/unicode-decomp.pl: Move from chartables.pl, and remove
	the code for generating include/java-chartables.h.
	* scripts/unicode-blocks.pl: Move from scripts/blocks.pl, and
	merge with Classpath.
	* scripts/unicode-muncher.pl: Copy from Classpath.
	* scritps/MakeCharTables.java: New file.
	* gnu/gcj/convert/Blocks-3.txt: New file.
	* gnu/gcj/convert/UnicodeData-3.0.0.txt: New file.
	* gnu/gcj/convert/UnicodeCharacterDatabase-3.0.0.html: New file.
	* gnu/java/lang/CharData.java: Copy from Classpath.
	* Makefile.am (ordinary_java_source_files): Add
	gnu/java/lang/CharData.java.
	* configure.in: Remove --enable-fast-character option.
	* java/lang/Character.java: Merge algorithms and Javadoc with
	Classpath.
	* java/lang/natCharacter.cc: Implement Unicode lookup table more
	efficiently.
	* include/java-chardecomp.h: Regenerate.
	* include/java-chartables.h: Regenerate.


git-svn-id: svn+ssh://gcc.gnu.org/svn/gcc/trunk@50368 138bc75d-0d04-0410-961f-82ee72b054a4

5 files changed, 1113 insertions, 65 deletions

diff --git a/libjava/scripts/MakeCharTables.java b/libjava/scripts/MakeCharTables.java
new file mode 100644
index 00000000000..d69a00db6bd
--- /dev/null
+++ b/libjava/scripts/MakeCharTables.java
@@ -0,0 +1,212 @@
+/* MakeCharTables.java - converts gnu.java.lang.CharData into
+                         include/java-chartables.h
+   Copyright (C) 2002 Free Software Foundation, Inc.
+
+This file is part of GNU Classpath.
+
+GNU Classpath is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+GNU Classpath is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GNU Classpath; see the file COPYING.  If not, write to the
+Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+02111-1307 USA.
+
+Linking this library statically or dynamically with other modules is
+making a combined work based on this library.  Thus, the terms and
+conditions of the GNU General Public License cover the whole
+combination.
+
+As a special exception, the copyright holders of this library give you
+permission to link this library with independent modules to produce an
+executable, regardless of the license terms of these independent
+modules, and to copy and distribute the resulting executable under
+terms of your choice, provided that you also meet, for each linked
+independent module, the terms and conditions of the license of that
+module.  An independent module is a module which is not derived from
+or based on this library.  If you modify this library, you may extend
+this exception to your version of the library, but you are not
+obligated to do so.  If you do not wish to do so, delete this
+exception statement from your version. */
+
+import gnu.java.lang.CharData;
+
+public class MakeCharTables implements CharData
+{
+  public static void main(String[] args)
+  {
+    System.out.println("/* java-chartables.h -- Character tables for java.lang.Character -*- c++ -*-\n"
+                       + "   Copyright (C) 2002 Free Software Foundation, Inc.\n"
+                       + "   *** This file is generated by scripts/MakeCharTables.java ***\n"
+                       + "\n"
+                       + "This file is part of GNU Classpath.\n"
+                       + "\n"
+                       + "GNU Classpath is free software; you can redistribute it and/or modify\n"
+                       + "it under the terms of the GNU General Public License as published by\n"
+                       + "the Free Software Foundation; either version 2, or (at your option)\n"
+                       + "any later version.\n"
+                       + "\n"
+                       + "GNU Classpath is distributed in the hope that it will be useful, but\n"
+                       + "WITHOUT ANY WARRANTY; without even the implied warranty of\n"
+                       + "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU\n"
+                       + "General Public License for more details.\n"
+                       + "\n"
+                       + "You should have received a copy of the GNU General Public License\n"
+                       + "along with GNU Classpath; see the file COPYING.  If not, write to the\n"
+                       + "Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA\n"
+                       + "02111-1307 USA.\n"
+                       + "\n"
+                       + "Linking this library statically or dynamically with other modules is\n"
+                       + "making a combined work based on this library.  Thus, the terms and\n"
+                       + "conditions of the GNU General Public License cover the whole\n"
+                       + "combination.\n"
+                       + "\n"
+                       + "As a special exception, the copyright holders of this library give you\n"
+                       + "permission to link this library with independent modules to produce an\n"
+                       + "executable, regardless of the license terms of these independent\n"
+                       + "modules, and to copy and distribute the resulting executable under\n"
+                       + "terms of your choice, provided that you also meet, for each linked\n"
+                       + "independent module, the terms and conditions of the license of that\n"
+                       + "module.  An independent module is a module which is not derived from\n"
+                       + "or based on this library.  If you modify this library, you may extend\n"
+                       + "this exception to your version of the library, but you are not\n"
+                       + "obligated to do so.  If you do not wish to do so, delete this\n"
+                       + "exception statement from your version. */\n"
+                       + "\n"
+                       + "#ifndef __JAVA_CHARTABLES_H__\n"
+                       + "#define __JAVA_CHARTABLES_H__\n"
+                       + "\n"
+                       + "// These tables are automatically generated by scripts/MakeCharTables.java.\n"
+                       + "// This is in turn parsing gnu.java.lang.CharData, which is generated by\n"
+                       + "// scripts/unicode-muncher.pl.  The Unicode data comes from\n"
+                       + "// www.unicode.org; this header is based on\n"
+                       + "// " + SOURCE + ". JDK 1.4 uses Unicode version 3.0.0.\n"
+                       + "// DO NOT EDIT the tables.  Instead, fix the upstream scripts and run\n"
+                       + "// them again.\n"
+                       + "\n"
+                       + "// The data is stored in C style arrays of the appropriate CNI types, to\n"
+                       + "// guarantee that the data is constant and non-relocatable.  The field\n"
+                       + "// <code>blocks</code> stores the offset of a block of 2<supSHIFT</sup>\n"
+                       + "// characters within <code>data</code>. The data field, in turn, stores\n"
+                       + "// information about each character in the low order bits, and an offset\n"
+                       + "// into the attribute tables <code>upper</code>, <code>lower</code>,\n"
+                       + "// <code>numValue</code>, and <code>direction</code>.  Notice that the\n"
+                       + "// attribute tables are much smaller than 0xffff entries; as many characters\n"
+                       + "// in Unicode share common attributes.  Finally, there is a listing for\n"
+                       + "// <code>title</code> exceptions (most characters just have the same title\n"
+                       + "// case as upper case).\n"
+                       + "\n"
+                       + "// This file should only be included by natCharacter.cc\n"
+                       + "\n"
+                       + "/**\n"
+                       + " * The character shift amount to look up the block offset. In other words,\n"
+                       + " * <code>(char) (blocks[ch >> SHIFT] + ch)</code> is the index where\n"
+                       + " * <code>ch</code> is described in <code>data</code>.\n"
+                       + " */\n"
+                       + "#define SHIFT " + SHIFT);
+
+    convertString("/**\n"
+                  + " * The mapping of character blocks to their location in <code>data</code>.\n"
+                  + " * Each entry has been adjusted so that a modulo 16 sum with the desired\n"
+                  + " * character gives the actual index into <code>data</code>.\n"
+                  + " */",
+                  char.class, "blocks", BLOCKS);
+
+    convertString("/**\n"
+                  + " * Information about each character.  The low order 5 bits form the\n"
+                  + " * character type, the next bit is a flag for non-breaking spaces, and the\n"
+                  + " * next bit is a flag for mirrored directionality.  The high order 9 bits\n"
+                  + " * form the offset into the attribute tables.  Note that this limits the\n"
+                  + " * number of unique character attributes to 512, which is not a problem\n"
+                  + " * as of Unicode version 3.2.0, but may soon become one.\n"
+                  + " */",
+                  char.class, "data", DATA);
+
+    convertString("/**\n"
+                  + " * This is the attribute table for computing the numeric value of a\n"
+                  + " * character.  The value is -1 if Unicode does not define a value, -2\n"
+                  + " * if the value is not a positive integer, otherwise it is the value.\n"
+                  + " */",
+                  short.class, "numValue", NUM_VALUE);
+
+    convertString("/**\n"
+                  + " * This is the attribute table for computing the uppercase representation\n"
+                  + " * of a character.  The value is the difference between the character and\n"
+                  + " * its uppercase version.\n"
+                  + " */",
+                  short.class, "upper", UPPER);
+
+    convertString("/**\n"
+                  + " * This is the attribute table for computing the lowercase representation\n"
+                  + " * of a character.  The value is the difference between the character and\n"
+                  + " * its lowercase version.\n"
+                  + " */",
+                  short.class, "lower", LOWER);
+
+    convertString("/**\n"
+                  + " * This is the attribute table for computing the directionality class\n"
+                  + " * of a character.  At present, the value is in the range 0 - 18 if the\n"
+                  + " * character has a direction, otherwise it is -1.\n"
+                  + " */",
+                  byte.class, "direction", DIRECTION);
+
+    convertString("/**\n"
+                  + " * This is the listing of titlecase special cases (all other character\n"
+                  + " * can use <code>upper</code> to determine their titlecase).  The listing\n"
+                  + " * is a sequence of character pairs; converting the first character of the\n"
+                  + " * pair to titlecase produces the second character.\n"
+                  + " */",
+                  char.class, "title", TITLE);
+
+    System.out.println();
+    System.out.println("#endif /* __JAVA_CHARTABLES_H__ */");
+  }
+
+  private static void convertString(String header, Class type,
+                                    String name, String field)
+  {
+    System.out.println();
+    System.out.println(header);
+    System.out.println("static const j" + type.getName() + " " + name
+                       + "[] = {");
+    char[] data = field.toCharArray();
+    int wrap;
+    if (type == char.class)
+      wrap = 10;
+    else if (type == byte.class)
+      wrap = 21;
+    else if (type == short.class)
+      wrap = 13;
+    else
+      throw new Error("Unexpeced type");
+    for (int i = 0; i < data.length; i += wrap)
+      {
+        System.out.print("   ");
+        for (int j = 0; j < wrap; j++)
+          {
+            if (i + j >= data.length)
+              break;
+            System.out.print(" ");
+            if (type == char.class)
+              System.out.print((int) data[i + j]);
+            else if (type == byte.class)
+              System.out.print((byte) data[i + j]);
+            else if (type == short.class)
+              System.out.print((short) data[i + j]);
+            System.out.print(",");
+          }
+        System.out.println();
+      }
+    System.out.println("  };\n"
+                       + "/** Length of " + name + ". */\n"
+                       + "static const int " + name + "_length = "
+                       + data.length + ";");
+  }
+}
diff --git a/libjava/scripts/blocks.pl b/libjava/scripts/blocks.pl
deleted file mode 100644
index 9142a49f87a..00000000000
--- a/libjava/scripts/blocks.pl
+++ /dev/null
@@ -1,65 +0,0 @@
-#! /usr/bin/perl
-
-if ($ARGV[0] eq '')
-{
-    $file = 'Blocks.txt';
-    if (! -f $file)
-    {
-	# Too painful to figure out how to get Perl to do it.
-	system 'wget -o .wget-log http://www.unicode.org/Public/UNIDATA/Blocks.txt';
-    }
-}
-else
-{
-    $file = $ARGV[0];
-}
-
-open (INPUT, "< $file") || die "couldn't open $file: $!";
-
-@array = ();
-while (<INPUT>)
-{
-    next if /^#/;
-    chop;
-    next if /^$/;
-
-    ($start, $to, $text) = split (/; /);
-    ($symbol = $text) =~ tr/a-z/A-Z/;
-    $symbol =~ s/[- ]/_/g;
-
-    # Special case for one of the SPECIALS.
-    next if $start eq 'FEFF';
-
-    # Special case some areas that our heuristic mishandles.
-    if ($symbol eq 'HIGH_SURROGATES')
-    {
-	$symbol = 'SURROGATES_AREA';
-	$text = 'Surrogates Area';
-	$to = 'DFFF';
-    }
-    elsif ($symbol =~ /SURROGATES/)
-    {
-	next;
-    }
-    elsif ($symbol eq 'PRIVATE_USE')
-    {
-	$symbol .= '_AREA';
-	$text = 'Private Use Area';
-    }
-
-    printf "    public static final UnicodeBlock %s = new UnicodeBlock (\"%s\", '\\u%s', '\\u%s');\n",
-           $symbol, $text, $start, $to;
-
-    push (@array, $symbol);
-}
-
-printf "    private static final UnicodeBlock[] blocks = {\n";
-foreach (@array)
-{
-    printf "      %s", $_;
-    printf "," unless $_ eq 'SPECIALS';
-    printf "\n";
-}
-printf "    };\n";
-
-close (INPUT);
diff --git a/libjava/scripts/unicode-blocks.pl b/libjava/scripts/unicode-blocks.pl
new file mode 100755
index 00000000000..22e58ec5d52
--- /dev/null
+++ b/libjava/scripts/unicode-blocks.pl
@@ -0,0 +1,210 @@
+#!/usr/bin/perl -w
+# unicode-blocks.pl -- Script to generate java.lang.Character.UnicodeBlock
+# Copyright (C) 2002 Free Software Foundation, Inc.
+#
+# This file is part of GNU Classpath.
+#
+# GNU Classpath is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# GNU Classpath is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Classpath; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# 02111-1307 USA.
+#
+# Linking this library statically or dynamically with other modules is
+# making a combined work based on this library.  Thus, the terms and
+# conditions of the GNU General Public License cover the whole
+# combination.
+#
+# As a special exception, the copyright holders of this library give you
+# permission to link this library with independent modules to produce an
+# executable, regardless of the license terms of these independent
+# modules, and to copy and distribute the resulting executable under
+# terms of your choice, provided that you also meet, for each linked
+# independent module, the terms and conditions of the license of that
+# module.  An independent module is a module which is not derived from
+# or based on this library.  If you modify this library, you may extend
+# this exception to your version of the library, but you are not
+# obligated to do so.  If you do not wish to do so, delete this
+# exception statement from your version.
+
+
+# Code for reading Blocks.txt and generating (to standard out) the code for
+# java.lang.Character.UnicodeBlock, for pasting into java/lang/Character.java.
+# You should probably check that the results are accurate to the
+# specification, but I made sure it works OOB for Unicode 3.0.0 and JDK 1.4.
+# As the grammar for the Blocks.txt file is changing in Unicode 3.2.0, you
+# will have to tweak this some for future use.  For now, the relevant
+# Unicode definition files are found in libjava/gnu/gcj/convert/.
+#
+# author Eric Blake <ebb9@email.byu.edu>
+#
+# usage: unicode-blocks.pl <blocks.txt>
+#    where <blocks.txt> is obtained from www.unicode.org (named Blocks-3.txt
+#    for Unicode version 3.0.0).
+
+
+die "Usage: $0 <blocks.txt>" unless @ARGV == 1;
+open (BLOCKS, $ARGV[0]) || die "Can't open Unicode block file: $!\n";
+
+# A hash of added fields and the JDK they were added in, to automatically
+# print @since tags.  Maintaining this is optional (and tedious), but nice.
+my %additions = ("SYRIAC" => "1.4",
+                 "THAANA" => "1.4",
+                 "SINHALA" => "1.4",
+                 "MYANMAR" => "1.4",
+                 "ETHIOPIC" => "1.4",
+                 "CHEROKEE" => "1.4",
+                 "UNIFIED_CANADIAN_ABORIGINAL_SYLLABICS" => "1.4",
+                 "OGHAM" => "1.4",
+                 "RUNIC" => "1.4",
+                 "KHMER" => "1.4",
+                 "MONGOLIAN" => "1.4",
+                 "BRAILLE_PATTERNS" => "1.4",
+                 "CJK_RADICALS_SUPPLEMENT" => "1.4",
+                 "KANGXI_RADICALS" => "1.4",
+                 "IDEOGRAPHIC_DESCRIPTION_CHARACTERS" => "1.4",
+                 "BOPOMOFO_EXTENDED" => "1.4",
+                 "CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A" => "1.4",
+                 "YI_SYLLABLES" => "1.4",
+                 "YI_RADICALS" => "1.4",
+                 );
+
+print <<'EOF';
+  /**
+   * A family of character subsets in the Unicode specification. A character
+   * is in at most one of these blocks.
+   *
+   * This inner class was generated automatically from
+   * <code>$ARGV[0]</code>, by some perl scripts.
+   * This Unicode definition file can be found on the
+   * <a href="http://www.unicode.org">http://www.unicode.org</a> website.
+   * JDK 1.4 uses Unicode version 3.0.0.
+   *
+   * @author scripts/unicode-blocks.pl (written by Eric Blake)
+   * @since 1.2
+   */
+  public static final class UnicodeBlock extends Subset
+  {
+    /** The start of the subset. */
+    private final char start;
+
+    /** The end of the subset. */
+    private final char end;
+
+    /**
+     * Constructor for strictly defined blocks.
+     *
+     * @param start the start character of the range
+     * @param end the end character of the range
+     * @param name the block name
+     */
+    private UnicodeBlock(char start, char end, String name)
+    {
+      super(name);
+      this.start = start;
+      this.end = end;
+    }
+
+    /**
+     * Returns the Unicode character block which a character belongs to.
+     *
+     * @param ch the character to look up
+     * @return the set it belongs to, or null if it is not in one
+     */
+    public static UnicodeBlock of(char ch)
+    {
+      // Special case, since SPECIALS contains two ranges.
+      if (ch == '\uFEFF')
+        return SPECIALS;
+      // Simple binary search for the correct block.
+      int low = 0;
+      int hi = sets.length - 1;
+      while (low <= hi)
+        {
+          int mid = (low + hi) >> 1;
+          UnicodeBlock b = sets[mid];
+          if (ch < b.start)
+            hi = mid - 1;
+          else if (ch > b.end)
+            low = mid + 1;
+          else
+            return b;
+        }
+      return null;
+    }
+EOF
+
+my $seenSpecials = 0;
+my $seenSurrogates = 0;
+my $surrogateStart = 0;
+my @names = ();
+while (<BLOCKS>) {
+    next if /^\#/;
+    my ($start, $end, $block) = split(/; /);
+    next unless defined $block;
+    chomp $block;
+    $block =~ s/ *$//;
+    if (! $seenSpecials and $block =~ /Specials/) {
+        # Special case SPECIALS, since it is two disjoint ranges
+        $seenSpecials = 1;
+        next;              
+    }
+    if ($block =~ /Surrogates/) {
+        # Special case SURROGATES_AREA, since it one range, not three
+        # consecutive, in Java
+        $seenSurrogates++;
+        if ($seenSurrogates == 1) {
+            $surrogateStart = $start;
+            next;
+        } elsif ($seenSurrogates == 2) {
+            next;
+        } else {
+            $start = $surrogateStart;
+            $block = "Surrogates Area";
+        }
+    }
+    # Special case the name of PRIVATE_USE_AREA.
+    $block =~ s/(Private Use)/$1 Area/;
+
+    (my $name = $block) =~ tr/a-z -/A-Z__/;
+    push @names, $name;
+    my $since = (defined $additions{$name}
+                 ? "\n     * \@since $additions{$name}" : "");
+    my $extra = ($block =~ /Specials/ ? "'\\uFEFF', " : "");
+    print <<EOF;
+
+    /**
+     * $block.
+     * $extra'\\u$start' - '\\u$end'.$since
+     */
+    public final static UnicodeBlock $name
+      = new UnicodeBlock('\\u$start', '\\u$end',
+                         "$name");
+EOF
+}
+
+print <<EOF;
+
+    /**
+     * The defined subsets.
+     */
+    private static final UnicodeBlock sets[] = {
+EOF
+
+foreach (@names) {
+    print "      $_,\n";
+}
+
+print <<EOF;
+    };
+  } // class UnicodeBlock
+EOF
diff --git a/libjava/scripts/unicode-decomp.pl b/libjava/scripts/unicode-decomp.pl
new file mode 100755
index 00000000000..8aeed152adf
--- /dev/null
+++ b/libjava/scripts/unicode-decomp.pl
@@ -0,0 +1,146 @@
+#!/usr/bin/perl -w
+# unicode-decomp.pl - script to generate database for java.text.Collator
+# Copyright (C) 1998, 1999, 2002 Free Software Foundation, Inc.
+#
+# This file is part of libjava.
+# 
+# This software is copyrighted work licensed under the terms of the
+# Libjava License.  Please consult the file "LIBJAVA_LICENSE" for
+# details.
+
+# Code for reading UnicodeData.txt and generating the code for
+# gnu.java.lang.CharData.  For now, the relevant Unicode definition files
+# are found in libjava/gnu/gcj/convert/.
+#
+# Usage: ./unicode-decomp.pl [-n] <UnicodeData.txt> <decomp.h>
+#   where <UnicodeData.txt> is obtained from www.unicode.org (named
+#   UnicodeData-3.0.0.txt for Unicode version 3.0.0), and <CharData.java>
+#   is the final location of include/java-chardecomp.h.
+#   As of JDK 1.4, use Unicode version 3.0.0 for best results.
+#
+# If this exits with nonzero status, then you must investigate the
+# cause of the problem.
+# Diagnostics and other information to stderr.
+# With -n, the files are not created, but all processing still occurs.
+
+# These maps characters to their decompositions.
+my %canonical_decomposition = ();
+my %full_decomposition = ();
+
+# Handle `-n' and open output files.
+if ($ARGV[0] && $ARGV[0] eq '-n')
+{
+    shift @ARGV;
+    $ARGV[1] = '/dev/null';
+}
+die "Usage: $0 <UnicodeData.txt> <java-chardecomp.h>" unless @ARGV == 2;
+open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n";
+
+# Process the Unicode file.
+$| = 1;
+my $count = 0;
+print STDERR "Parsing attributes file";
+while (<UNICODE>)
+{
+    print STDERR "." unless $count++ % 1000;
+    chomp;
+    s/\r//g;
+    my ($ch, undef, undef, undef, undef, $decomp) = split ';';
+    $ch = hex($ch);
+
+    if ($decomp ne '')
+    {
+        my $is_full = 0;
+        my @decomp = ();
+        foreach (split (' ', $decomp))
+        {
+            if (/^\<.*\>$/)
+            {
+                $is_full = 1;
+                next;
+            }
+	    push (@decomp, hex ($_));
+	}
+        my $s = pack "n*", @decomp;
+        if ($is_full)
+        {
+            $full_decomposition{$ch} = $s;
+        }
+        else
+        {
+            $canonical_decomposition{$ch} = $s;
+        }
+    }
+}
+
+# Now generate decomposition tables.
+open DECOMP, "> $ARGV[1]" or die "Can't open output file: $!\n";
+print STDERR "\nGenerating tables\n";
+print DECOMP <<EOF;
+// java-chardecomp.h - Decomposition character tables -*- c++ -*-
+
+#ifndef __JAVA_CHARDECOMP_H__
+#define __JAVA_CHARDECOMP_H__
+
+
+// These tables are automatically generated by the $0
+// script.  DO NOT EDIT the tables.  Instead, fix the script
+// and run it again.
+
+// This file should only be included by natCollator.cc
+
+struct decomp_entry
+{
+  jchar key;
+  const char *value;
+};
+
+EOF
+
+&write_decompositions;
+
+print DECOMP "#endif /* __JAVA_CHARDECOMP_H__ */\n";
+
+close(DECOMP);
+print STDERR "Done\n";
+exit;
+
+
+# Write a single decomposition table.
+sub write_single_decomposition($$%)
+{
+    my ($name, $is_canon, %table) = @_;
+    my $first_line = 1;
+    print DECOMP "static const decomp_entry ${name}_decomposition[] =\n{\n";
+
+    for my $key (0 .. 0xffff)
+    {
+	next if ! defined $table{$key};
+        print DECOMP ",\n" unless $first_line;
+	$first_line = 0;
+
+	printf DECOMP "  { 0x%04x, \"", $key;
+
+	# We represent the expansion as a series of bytes, terminated
+	# with a double nul.  This is ugly, but relatively
+	# space-efficient.  Most expansions are short, but there are a
+	# few that are very long (e.g. \uFDFA).  This means that if we
+	# chose a fixed-space representation we would waste a lot of
+	# space.
+	my @expansion = unpack "n*", $table{$key};
+	foreach my $char (@expansion)
+	{
+	    printf DECOMP "\\x%02x\\x%02x", ($char / 256), ($char % 256);
+	}
+
+	print DECOMP "\" }";
+    }
+
+    print DECOMP "\n};\n\n";
+}
+
+sub write_decompositions()
+{
+    &write_single_decomposition ('canonical', 1, %canonical_decomposition);
+    &write_single_decomposition ('full', 0, %full_decomposition);
+}
diff --git a/libjava/scripts/unicode-muncher.pl b/libjava/scripts/unicode-muncher.pl
new file mode 100755
index 00000000000..073bc699414
--- /dev/null
+++ b/libjava/scripts/unicode-muncher.pl
@@ -0,0 +1,545 @@
+#!/usr/bin/perl -w
+# unicode-muncher.pl -- generate Unicode database for java.lang.Character
+# Copyright (C) 1998, 2002 Free Software Foundation, Inc.
+#
+# This file is part of GNU Classpath.
+#
+# GNU Classpath is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2, or (at your option)
+# any later version.
+#
+# GNU Classpath is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Classpath; see the file COPYING.  If not, write to the
+# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+# 02111-1307 USA.
+#
+# Linking this library statically or dynamically with other modules is
+# making a combined work based on this library.  Thus, the terms and
+# conditions of the GNU General Public License cover the whole
+# combination.
+#
+# As a special exception, the copyright holders of this library give you
+# permission to link this library with independent modules to produce an
+# executable, regardless of the license terms of these independent
+# modules, and to copy and distribute the resulting executable under
+# terms of your choice, provided that you also meet, for each linked
+# independent module, the terms and conditions of the license of that
+# module.  An independent module is a module which is not derived from
+# or based on this library.  If you modify this library, you may extend
+# this exception to your version of the library, but you are not
+# obligated to do so.  If you do not wish to do so, delete this
+# exception statement from your version.
+
+# Code for reading UnicodeData.txt and generating the code for
+# gnu.java.lang.CharData.  For now, the relevant Unicode definition files
+# are found in libjava/gnu/gcj/convert/.
+#
+# Inspired by code from Jochen Hoenicke.
+# author Eric Blake <ebb9@email.byu.edu>
+#
+# Usage: ./unicode-muncher <UnicodeData.txt> <CharData.java>
+#   where <UnicodeData.txt> is obtained from www.unicode.org (named
+#   UnicodeData-3.0.0.txt for Unicode version 3.0.0), and <CharData.java>
+#   is the final location for the Java interface gnu.java.lang.CharData.
+#   As of JDK 1.4, use Unicode version 3.0.0 for best results.
+
+##
+## Convert a 16-bit integer to a Java source code String literal character
+##
+sub javaChar($) {
+    my ($char) = @_;
+    die "Out of range: $char\n" if $char < -0x8000 or $char > 0xffff;
+    $char += 0x10000 if $char < 0;
+    # Special case characters that must be escaped, or are shorter as ASCII
+    return sprintf("\\%03o", $char) if $char < 0x20;
+    return "\\\"" if $char == 0x22;
+    return "\\\\" if $char == 0x5c;
+    return pack("C", $char) if $char < 0x7f;
+    return sprintf("\\u%04x", $char);
+}
+
+##
+## Convert the text UnicodeData file from www.unicode.org into a Java
+## interface with string constants holding the compressed information.
+##
+my @TYPECODES = qw(Cn Lu Ll Lt Lm Lo Mn Me Mc Nd Nl No Zs Zl Zp Cc Cf
+                   SKIPPED Co Cs Pd Ps Pe Pc Po Sm Sc Sk So Pi Pf);
+my @DIRCODES = qw(L R AL EN ES ET AN CS NSM BN B S WS ON LRE LRO RLE RLO PDF);
+
+my $NOBREAK_FLAG  = 32;
+my $MIRRORED_FLAG = 64;
+
+my @info = ();
+my $titlecase = "";
+my $count = 0;
+my $range = 0;
+
+die "Usage: $0 <UnicodeData.txt> <CharData.java>" unless @ARGV == 2;
+open (UNICODE, "< $ARGV[0]") || die "Can't open Unicode attribute file: $!\n";
+
+# Stage 1: Parse the attribute file
+$| = 1;
+print "GNU Classpath Unicode Attribute Database Generator 2.0\n";
+print "Copyright (C) 1998, 2002 Free Software Foundation, Inc.\n";
+print "Parsing attributes file";
+while(<UNICODE>) {
+    print "." unless $count++ % 1000;
+    chomp;
+    s/\r//g;
+    my ($ch, $name, $category, undef, $bidir, $decomp, undef, undef, $numeric,
+        $mirrored, undef, undef, $upcase, $lowcase, $title) = split ';';
+    $ch = hex($ch);
+    next if $ch > 0xffff; # Ignore surrogate pairs, since Java does
+
+    my ($type, $numValue, $upperchar, $lowerchar, $direction);
+
+    $type = 0;
+    while ($category !~ /^$TYPECODES[$type]$/) {
+        if (++$type == @TYPECODES) {
+            die "$ch: Unknown type: $category";
+        }
+    }
+    $type |= $NOBREAK_FLAG if ($decomp =~ /noBreak/);
+    $type |= $MIRRORED_FLAG if ($mirrored =~ /Y/);
+
+    if ($numeric =~ /^[0-9]+$/) {
+        $numValue = $numeric;
+        die "numValue too big: $ch, $numValue\n" if $numValue >= 0x7fff;
+    } elsif ($numeric eq "") {
+        # Special case sequences of 'a'-'z'
+        if ($ch >= 0x0041 && $ch <= 0x005a) {
+            $numValue = $ch - 0x0037;
+        } elsif ($ch >= 0x0061 && $ch <= 0x007a) {
+            $numValue = $ch - 0x0057;
+        } elsif ($ch >= 0xff21 && $ch <= 0xff3a) {
+            $numValue = $ch - 0xff17;
+        } elsif ($ch >= 0xff41 && $ch <= 0xff5a) {
+            $numValue = $ch - 0xff37;
+        } else {
+            $numValue = -1;
+        }
+    } else {
+        $numValue = -2;
+    }
+
+    $upperchar = $upcase ? hex($upcase) - $ch : 0;
+    $lowerchar = $lowcase ? hex($lowcase) - $ch : 0;
+    if ($title ne $upcase) {
+        my $titlechar = $title ? hex($title) : $ch;
+        $titlecase .= pack("n2", $ch, $titlechar);
+    }
+
+    $direction = 0;
+    while ($bidir !~ /^$DIRCODES[$direction]$/) {
+        if (++$direction == @DIRCODES) {
+            $direction = -1;
+            last;
+        }
+    }
+
+    if ($range) {
+        die "Expecting end of range at $ch\n" unless $name =~ /Last>$/;
+        for ($range + 1 .. $ch - 1) {
+            $info[$_] = pack("n5", $type, $numValue, $upperchar,
+                             $lowerchar, $direction);
+        }
+        $range = 0;
+    } elsif ($name =~ /First>$/) {
+        $range = $ch;
+    }
+    $info[$ch] = pack("n5", $type, $numValue, $upperchar, $lowerchar,
+                      $direction);
+}
+close UNICODE;
+
+# Stage 2: Compress the data structures
+printf "\nCompressing data structures";
+$count = 0;
+my $info = ();
+my %charhash = ();
+my @charinfo = ();
+
+for my $ch (0 .. 0xffff) {
+    print "." unless $count++ % 0x1000;
+    if (! defined $info[$ch]) {
+        $info[$ch] = pack("n5", 0, -1, 0, 0, -1);
+    }
+
+    my ($type, $numVal, $upper, $lower, $direction) = unpack("n5", $info[$ch]);
+    if (! exists $charhash{$info[$ch]}) {
+        push @charinfo, [ $numVal, $upper, $lower, $direction ];
+        $charhash{$info[$ch]} = $#charinfo;
+    }
+    $info .= pack("n", ($charhash{$info[$ch]} << 7) | $type);
+}
+
+my $charlen = @charinfo;
+my $bestshift;
+my $bestest = 1000000;
+my $bestblkstr;
+die "Too many unique character entries: $charlen\n" if $charlen > 512;
+print "\nUnique character entries: $charlen\n";
+
+for my $i (3 .. 8) {
+    my $blksize = 1 << $i;
+    my %blocks = ();
+    my @blkarray = ();
+    my ($j, $k);
+    print "shift: $i";
+
+    for ($j = 0; $j < 0x10000; $j += $blksize) {
+        my $blkkey = substr $info, 2 * $j, 2 * $blksize;
+        if (! exists $blocks{$blkkey}) {
+            push @blkarray, $blkkey;
+            $blocks{$blkkey} = $#blkarray;
+        }
+    }
+    my $blknum = @blkarray;
+    my $blocklen = $blknum * $blksize;
+    printf " before %5d", $blocklen;
+
+    # Now we try to pack the blkarray as tight as possible by finding matching
+    # heads and tails.
+    for ($j = $blksize - 1; $j > 0; $j--) {
+        my %tails = ();
+        for $k (0 .. $#blkarray) {
+            next if ! defined $blkarray[$k];
+            my $len = length $blkarray[$k];
+            my $tail = substr $blkarray[$k], $len - $j * 2;
+            if (exists $tails{$tail}) {
+                push @{$tails{$tail}}, $k;
+            } else {
+                $tails{$tail} = [ $k ];
+            }
+        }
+
+        # tails are calculated, now calculate the heads and merge.
+      BLOCK:
+        for $k (0 .. $#blkarray) {
+            next if ! defined $blkarray[$k];
+            my $tomerge = $k;
+            while (1) {
+                my $head = substr($blkarray[$tomerge], 0, $j * 2);
+                my $entry = $tails{$head};
+                next BLOCK if ! defined $entry;
+
+                my $other = shift @{$entry};
+                if ($other == $tomerge) {
+                    if (@{$entry}) {
+                        push @{$entry}, $other;
+                        $other = shift @{$entry};
+                    } else {
+                        push @{$entry}, $other;
+                        next BLOCK;
+                    }
+                }
+                if (@{$entry} == 0) {
+                    delete $tails{$head};
+                }
+
+                # a match was found
+                my $merge = $blkarray[$other]
+                    . substr($blkarray[$tomerge], $j * 2);
+                $blocklen -= $j;
+                $blknum--;
+
+                if ($other < $tomerge) {
+                    $blkarray[$tomerge] = undef;
+                    $blkarray[$other] = $merge;
+                    my $len = length $merge;
+                    my $tail = substr $merge, $len - $j * 2;
+                    $tails{$tail} = [ map { $_ == $tomerge ? $other : $_ }
+                                      @{$tails{$tail}} ];
+                    next BLOCK;
+                }
+                $blkarray[$tomerge] = $merge;
+                $blkarray[$other] = undef;
+            }
+        }
+    }
+    my $blockstr;
+    for $k (0 .. $#blkarray) {
+        $blockstr .= $blkarray[$k] if defined $blkarray[$k];
+    }
+
+    die "Unexpected $blocklen" if length($blockstr) != 2 * $blocklen;
+    my $estimate = 2 * $blocklen + (0x20000 >> $i);
+
+    printf " after merge %5d: %6d bytes\n", $blocklen, $estimate;
+    if ($estimate < $bestest) {
+        $bestest = $estimate;
+        $bestshift = $i;
+        $bestblkstr = $blockstr;
+    }
+}
+
+my @blocks;
+my $blksize = 1 << $bestshift;
+for (my $j = 0; $j < 0x10000; $j += $blksize) {
+    my $blkkey = substr $info, 2 * $j, 2 * $blksize;
+    my $index = index $bestblkstr, $blkkey;
+    while ($index & 1) {
+        die "not found: $j" if $index == -1;
+        $index = index $bestblkstr, $blkkey, $index + 1;
+    }
+    push @blocks, ($index / 2 - $j) & 0xffff;
+}
+
+# Phase 3: Generate the file
+die "UTF-8 limit of blocks may be exceeded: " . scalar(@blocks) . "\n"
+    if @blocks > 0xffff / 3;
+die "UTF-8 limit of data may be exceeded: " . length($bestblkstr) . "\n"
+    if length($bestblkstr) > 0xffff / 3;
+{
+    print "Generating $ARGV[1] with shift of $bestshift";
+    my ($i, $j);
+
+    open OUTPUT, "> $ARGV[1]" or die "Failed creating output file: $!\n";
+    print OUTPUT <<EOF;
+/* gnu/java/lang/CharData -- Database for java.lang.Character Unicode info
+   Copyright (C) 2002 Free Software Foundation, Inc.
+   *** This file is generated by scripts/unicode-muncher.pl ***
+
+This file is part of GNU Classpath.
+
+GNU Classpath is free software; you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation; either version 2, or (at your option)
+any later version.
+
+GNU Classpath is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with GNU Classpath; see the file COPYING.  If not, write to the
+Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+02111-1307 USA.
+
+Linking this library statically or dynamically with other modules is
+making a combined work based on this library.  Thus, the terms and
+conditions of the GNU General Public License cover the whole
+combination.
+
+As a special exception, the copyright holders of this library give you
+permission to link this library with independent modules to produce an
+executable, regardless of the license terms of these independent
+modules, and to copy and distribute the resulting executable under
+terms of your choice, provided that you also meet, for each linked
+independent module, the terms and conditions of the license of that
+module.  An independent module is a module which is not derived from
+or based on this library.  If you modify this library, you may extend
+this exception to your version of the library, but you are not
+obligated to do so.  If you do not wish to do so, delete this
+exception statement from your version. */
+
+package gnu.java.lang;
+
+/**
+ * This contains the info about the unicode characters, that
+ * java.lang.Character needs.  It is generated automatically from
+ * <code>$ARGV[0]</code>, by some
+ * perl scripts. This Unicode definition file can be found on the
+ * <a href="http://www.unicode.org">http://www.unicode.org</a> website.
+ * JDK 1.4 uses Unicode version 3.0.0.
+ *
+ * The data is stored as string constants, but Character will convert these
+ * Strings to their respective <code>char[]</code> components.  The field
+ * <code>BLOCKS</code> stores the offset of a block of 2<sup>SHIFT</sup>
+ * characters within <code>DATA</code>.  The DATA field, in turn, stores
+ * information about each character in the low order bits, and an offset
+ * into the attribute tables <code>UPPER</code>, <code>LOWER</code>,
+ * <code>NUM_VALUE</code>, and <code>DIRECTION</code>.  Notice that the
+ * attribute tables are much smaller than 0xffff entries; as many characters
+ * in Unicode share common attributes.  Finally, there is a listing for
+ * <code>TITLE</code> exceptions (most characters just have the same
+ * title case as upper case).
+ *
+ * \@author scripts/unicode-muncher.pl (written by Jochen Hoenicke,
+ *         Eric Blake)
+ * \@see Character
+ */
+public interface CharData
+{
+  /**
+   * The Unicode definition file that was parsed to build this database.
+   */
+  String SOURCE = \"$ARGV[0]\";
+
+  /**
+   * The character shift amount to look up the block offset. In other words,
+   * <code>(char) (BLOCKS.value[ch >> SHIFT] + ch)</code> is the index where
+   * <code>ch</code> is described in <code>DATA</code>.
+   */
+  int SHIFT = $bestshift;
+
+  /**
+   * The mapping of character blocks to their location in <code>DATA</code>.
+   * Each entry has been adjusted so that the 16-bit sum with the desired
+   * character gives the actual index into <code>DATA</code>.
+   */
+  String BLOCKS
+EOF
+
+    for ($i = 0; $i < @blocks / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if @blocks <= $i * 11 + $j;
+            my $val = $blocks[$i * 11 + $j];
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * Information about each character.  The low order 5 bits form the
+   * character type, the next bit is a flag for non-breaking spaces, and the
+   * next bit is a flag for mirrored directionality.  The high order 9 bits
+   * form the offset into the attribute tables.  Note that this limits the
+   * number of unique character attributes to 512, which is not a problem
+   * as of Unicode version 3.2.0, but may soon become one.
+   */
+  String DATA
+EOF
+
+    my $len = length($bestblkstr) / 2;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = unpack "n", substr($bestblkstr, 2 * ($i*11 + $j), 2);
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * This is the attribute table for computing the numeric value of a
+   * character.  The value is -1 if Unicode does not define a value, -2
+   * if the value is not a positive integer, otherwise it is the value.
+   * Note that this is a signed value, but stored as an unsigned char
+   * since this is a String literal.
+   */
+  String NUM_VALUE
+EOF
+
+    $len = @charinfo;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = $charinfo[$i * 11 + $j][0];
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * This is the attribute table for computing the uppercase representation
+   * of a character.  The value is the signed difference between the
+   * character and its uppercase version.  Note that this is stored as an
+   * unsigned char since this is a String literal.
+   */
+  String UPPER
+EOF
+
+    $len = @charinfo;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = $charinfo[$i * 11 + $j][1];
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * This is the attribute table for computing the lowercase representation
+   * of a character.  The value is the signed difference between the
+   * character and its lowercase version.  Note that this is stored as an
+   * unsigned char since this is a String literal.
+   */
+  String LOWER
+EOF
+
+    $len = @charinfo;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = $charinfo[$i * 11 + $j][2];
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * This is the attribute table for computing the directionality class
+   * of a character.  At present, the value is in the range 0 - 18 if the
+   * character has a direction, otherwise it is -1.  Note that this is
+   * stored as an unsigned char since this is a String literal.
+   */
+  String DIRECTION
+EOF
+
+    $len = @charinfo;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = $charinfo[$i * 11 + $j][3];
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT <<EOF;
+;
+
+  /**
+   * This is the listing of titlecase special cases (all other character
+   * can use <code>UPPER</code> to determine their titlecase).  The listing
+   * is a sequence of character pairs; converting the first character of the
+   * pair to titlecase produces the second character.
+   */
+  String TITLE
+EOF
+
+    $len = length($titlecase) / 2;
+    for ($i = 0; $i < $len / 11; $i++) {
+        print OUTPUT $i ? "\n    + \"" : "    = \"";
+        for $j (0 .. 10) {
+            last if $len <= $i * 11 + $j;
+            my $val = unpack "n", substr($titlecase, 2 * ($i*11 + $j), 2);
+            print OUTPUT javaChar($val);
+        }
+        print OUTPUT "\"";
+    }
+
+    print OUTPUT ";\n}\n";
+    close OUTPUT;
+}
+print "\nDone.\n";