Exercise the utf8:: "internal" functions.

p4raw-id: //depot/perl@19778

2 files changed, 96 insertions, 18 deletions

diff --git a/lib/utf8.pm b/lib/utf8.pm
index 0727c4dca6..0c8a99135d 100644
--- a/lib/utf8.pm
+++ b/lib/utf8.pm
@@ -80,14 +80,14 @@ you should not unless you really want to have UTF-8 source code.
 
 =item * $num_octets = utf8::upgrade($string);
 
-Converts (in-place) internal representation of string to Perl's internal
-I<UTF-X> form.  Returns the number of octets necessary to represent
-the string as I<UTF-X>.  Can be used to make sure that the
+Converts (in-place) internal representation of string to Perl's
+internal I<UTF-X> form.  Returns the number of octets necessary to
+represent the string as I<UTF-X>.  Can be used to make sure that the
 UTF-8 flag is on, so that C<\w> or C<lc()> work as expected on strings
-containing characters in the range 0x80-0xFF.  Note that this should
-not be used to convert
-a legacy byte encoding to Unicode: use Encode for that.  Affected
-by the encoding pragma.
+containing characters in the range 0x80-0xFF (oon ASCII and
+derivatives).  Note that this should not be used to convert a legacy
+byte encoding to Unicode: use Encode for that.  Affected by the
+encoding pragma.
 
 =item * utf8::downgrade($string[, FAIL_OK])
 
@@ -103,20 +103,21 @@ pragma.
 =item * utf8::encode($string)
 
 Converts (in-place) I<$string> from logical characters to octet
-sequence representing it in Perl's I<UTF-X> encoding. Same as
-Encode::encode_utf8(). Note that this should not be used to convert
-a legacy byte encoding to Unicode: use Encode for that.
+sequence representing it in Perl's I<UTF-X> encoding.  Returns
+nothing.  Same as Encode::encode_utf8(). Note that this should not be
+used to convert a legacy byte encoding to Unicode: use Encode for
+that.
 
 =item * $flag = utf8::decode($string)
 
 Attempts to convert I<$string> in-place from Perl's I<UTF-X> encoding
-into logical characters. Same as Encode::decode_utf8(). Note that this
-should not be used to convert Unicode back to a legacy byte encoding:
-use Encode for that.
+into logical characters. Returns nothing.  Same as Encode::decode_utf8().
+Note that this should not be used to convert Unicode back to a legacy
+byte encoding: use Encode for that.
 
 =item * $flag = utf8::is_utf8(STRING)
 
-Test whether STRING is in UTF-8.
+Test whether STRING is in UTF-8.  Same as Encode::is_utf8().
 
 =item * $flag = utf8::valid(STRING)
 
@@ -135,9 +136,9 @@ functions C<sv_utf8_upgrade>, C<sv_utf8_downgrade>, C<sv_utf8_encode>,
 and C<sv_utf8_decode>, which are wrapped by the Perl functions
 C<utf8::upgrade>, C<utf8::downgrade>, C<utf8::encode> and
 C<utf8::decode>.  Note that in the Perl 5.8.0 implementation the
-functions utf8::valid, utf8::encode, utf8::decode, utf8::upgrade,
-and utf8::downgrade are always available, without a C<require utf8>
-statement-- this may change in future releases.
+functions utf8::is_utf8, utf8::valid, utf8::encode, utf8::decode,
+utf8::upgrade, and utf8::downgrade are always available, without a
+C<require utf8> statement-- this may change in future releases.
 
 =head1 BUGS
 
diff --git a/lib/utf8.t b/lib/utf8.t
index 223bb1d825..33cd5966af 100644
--- a/lib/utf8.t
+++ b/lib/utf8.t
@@ -37,7 +37,7 @@ no utf8; # Ironic, no?
 #
 #
 
-plan tests => 99;
+plan tests => 143;
 
 {
     # bug id 20001009.001
@@ -332,3 +332,80 @@ SKIP: {
     eval qq{is(q \xc3\xbc test \xc3\xbc, qq\xc2\xb7 test \xc2\xb7,
 	       "utf8 quote delimiters [perl #16823]");};
 }
+
+# Test the "internals".
+
+{
+    my $a = "A";
+    my $b = chr(0x0FF);
+    my $c = chr(0x100);
+
+    ok( utf8::valid($a), "utf8::valid basic");
+    ok( utf8::valid($b), "utf8::valid beyond");
+    ok( utf8::valid($c), "utf8::valid unicode");
+
+    ok(!utf8::is_utf8($a), "!utf8::is_utf8 basic");
+    ok(!utf8::is_utf8($b), "!utf8::is_utf8 beyond");
+    ok( utf8::is_utf8($c), "utf8::is_utf8 unicode");
+
+    is(utf8::upgrade($a), 1, "utf8::upgrade basic");
+    is(utf8::upgrade($b), 2, "utf8::upgrade beyond");
+    is(utf8::upgrade($c), 2, "utf8::upgrade unicode");
+
+    is($a, "A",       "basic");
+    is($b, "\xFF",    "beyond");
+    is($c, "\x{100}", "unicode");
+
+    ok( utf8::valid($a), "utf8::valid basic");
+    ok( utf8::valid($b), "utf8::valid beyond");
+    ok( utf8::valid($c), "utf8::valid unicode");
+
+    ok( utf8::is_utf8($a), "utf8::is_utf8 basic");
+    ok( utf8::is_utf8($b), "utf8::is_utf8 beyond");
+    ok( utf8::is_utf8($c), "utf8::is_utf8 unicode");
+
+    is(utf8::downgrade($a), 1, "utf8::downgrade basic");
+    is(utf8::downgrade($b), 1, "utf8::downgrade beyond");
+
+    is($a, "A",       "basic");
+    is($b, "\xFF",    "beyond");
+
+    ok( utf8::valid($a), "utf8::valid basic");
+    ok( utf8::valid($b), "utf8::valid beyond");
+
+    ok(!utf8::is_utf8($a), "!utf8::is_utf8 basic");
+    ok(!utf8::is_utf8($b), "!utf8::is_utf8 beyond");
+
+    utf8::encode($a);
+    utf8::encode($b);
+    utf8::encode($c);
+
+    is($a, "A",       "basic");
+    is(length($b), 2, "beyond length");
+    is(length($c), 2, "unicode length");
+
+    ok(utf8::valid($a), "utf8::valid basic");
+    ok(utf8::valid($b), "utf8::valid beyond");
+    ok(utf8::valid($c), "utf8::valid unicode");
+
+    # encode() clears the UTF-8 flag (unlike upgrade()).
+    ok(!utf8::is_utf8($a), "!utf8::is_utf8 basic");
+    ok(!utf8::is_utf8($b), "!utf8::is_utf8 beyond");
+    ok(!utf8::is_utf8($c), "!utf8::is_utf8 unicode");
+
+    utf8::decode($a);
+    utf8::decode($b);
+    utf8::decode($c);
+
+    is($a, "A",       "basic");
+    is($b, "\xFF",    "beyond");
+    is($c, "\x{100}", "unicode");
+
+    ok(utf8::valid($a), "!utf8::valid basic");
+    ok(utf8::valid($b), "!utf8::valid beyond");
+    ok(utf8::valid($c), " utf8::valid unicode");
+
+    ok(!utf8::is_utf8($a), "!utf8::is_utf8 basic");
+    ok( utf8::is_utf8($b), " utf8::is_utf8 beyond"); # $b stays in UTF-8.
+    ok( utf8::is_utf8($c), " utf8::is_utf8 unicode");
+}