diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | lib/byte.pm | 21 | ||||
| -rw-r--r-- | lib/byte_heavy.pl | 6 | ||||
| -rw-r--r-- | lib/utf8.pm | 175 | ||||
| -rw-r--r-- | op.c | 6 | ||||
| -rw-r--r-- | perl.h | 5 | ||||
| -rw-r--r-- | pod/perldelta.pod | 9 | ||||
| -rw-r--r-- | pod/perlunicode.pod | 202 | ||||
| -rw-r--r-- | pp_ctl.c | 2 | ||||
| -rw-r--r-- | regcomp.c | 3 | ||||
| -rw-r--r-- | regexec.c | 1 | ||||
| -rw-r--r-- | utf8.h | 2 |
12 files changed, 284 insertions, 149 deletions
@@ -1104,6 +1104,7 @@ pod/perltodo.pod Todo list explained pod/perltoot.pod Tom's object-oriented tutorial pod/perltootc.pod Tom's object-oriented tutorial (more on class data) pod/perltrap.pod Trap info +pod/perlunicode.pod Unicode support info pod/perlvar.pod Variable info pod/perlxs.pod XS api info pod/perlxstut.pod XS tutorial diff --git a/lib/byte.pm b/lib/byte.pm index cc23b40f4f..33ffb769e8 100644 --- a/lib/byte.pm +++ b/lib/byte.pm @@ -20,7 +20,7 @@ __END__ =head1 NAME -byte - Perl pragma to turn force treating strings as bytes not UNICODE +byte - Perl pragma to force byte semantics rather than character semantics =head1 SYNOPSIS @@ -29,5 +29,24 @@ byte - Perl pragma to turn force treating strings as bytes not UNICODE =head1 DESCRIPTION +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! + +The C<use byte> pragma disables character semantics for the rest of the +lexical scope in which it appears. C<no byte> can be used to reverse +the effect of C<use byte> within the current lexical scope. + +Perl normally assumes character semantics in the presence of +character data (i.e. data that has come from a source that has +been marked as being of a particular character encoding) or when +the global $^U flag is enabled. [XXX: implement -C command line +switch and mention that instead of $^U?] + +To understand the implications and differences between character +semantics and byte semantics, see L<perlunicode>. + +=head1 SEE ALSO + +L<perlunicode>, L<utf8> =cut diff --git a/lib/byte_heavy.pl b/lib/byte_heavy.pl index 07c908a689..ec0558561d 100644 --- a/lib/byte_heavy.pl +++ b/lib/byte_heavy.pl @@ -1,8 +1,8 @@ package byte; -sub length ($) -{ - return CORE::length($_[0]); +sub length ($) { + BEGIN { byte::import() } + return CORE::length($_[0]); } 1; diff --git a/lib/utf8.pm b/lib/utf8.pm index 691de0d630..be7cc0bf0c 100644 --- a/lib/utf8.pm +++ b/lib/utf8.pm @@ -1,14 +1,15 @@ package utf8; -$^U = 1; +$^U = 1 if caller and caller eq 'main'; # they are unicode aware + # XXX split this out? sub import { - $^H |= 0x00000008; + $^H |= 0x00800000; $enc{caller()} = $_[1] if $_[1]; } sub unimport { - $^H &= ~0x00000008; + $^H &= ~0x00800000; } sub AUTOLOAD { @@ -21,7 +22,7 @@ __END__ =head1 NAME -utf8 - Perl pragma to turn on UTF-8 and Unicode support +utf8 - Perl pragma to enable/disable UTF-8 in source code =head1 SYNOPSIS @@ -30,154 +31,60 @@ utf8 - Perl pragma to turn on UTF-8 and Unicode support =head1 DESCRIPTION -The utf8 pragma tells Perl to use UTF-8 as its internal string -representation for the rest of the enclosing block. (The "no utf8" -pragma tells Perl to switch back to ordinary byte-oriented processing -for the rest of the enclosing block.) Under utf8, many operations that -formerly operated on bytes change to operating on characters. For -ASCII data this makes no difference, because UTF-8 stores ASCII in -single bytes, but for any character greater than C<chr(127)>, the -character is stored in a sequence of two or more bytes, all of which -have the high bit set. But by and large, the user need not worry about -this, because the utf8 pragma hides it from the user. A character -under utf8 is logically just a number ranging from 0 to 2**32 or so. -Larger characters encode to longer sequences of bytes, but again, this -is hidden. +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! -Use of the utf8 pragma has the following effects: +The C<use utf8> pragma tells the Perl parser to allow UTF-8 in the +program text in the current lexical scope. The C<no utf8> pragma +tells Perl to switch back to treating the source text as literal +bytes in the current lexical scope. -=over 4 +This pragma is primarily a compatibility device. Perl versions +earlier than 5.6 allowed arbitrary bytes in source code, whereas +in future we would like to standardize on the UTF-8 encoding for +source text. Until UTF-8 becomes the default format for source +text, this pragma should be used to recognize UTF-8 in the source. +When UTF-8 becomes the standard source format, this pragma will +effectively become a no-op. -=item * - -Strings and patterns may contain characters that have an ordinal value -larger than 255. Presuming you use a Unicode editor to edit your -program, these will typically occur directly within the literal strings -as UTF-8 characters, but you can also specify a particular character -with an extension of the C<\x> notation. UTF-8 characters are -specified by putting the hexadecimal code within curlies after the -C<\x>. For instance, a Unicode smiley face is C<\x{263A}>. A -character in the Latin-1 range (128..255) should be written C<\x{ab}> -rather than C<\xab>, since the former will turn into a two-byte UTF-8 -code, while the latter will continue to be interpreted as generating a -8-bit byte rather than a character. In fact, if C<-w> is turned on, it will -produce a warning that you might be generating invalid UTF-8. - -=item * +Enabling the C<utf8> pragma has the following effects: -Identifiers within the Perl script may contain Unicode alphanumeric -characters, including ideographs. (You are currently on your own when -it comes to using the canonical forms of characters--Perl doesn't (yet) -attempt to canonicalize variable names for you.) +=over =item * -Regular expressions match characters instead of bytes. For instance, -"." matches a character instead of a byte. (However, the C<\C> pattern -is provided to force a match a single byte ("C<char>" in C, hence -C<\C>).) +Bytes in the source text that have their high-bit set will be treated +as being part of a literal UTF-8 character. This includes most literals +such as identifiers, string constants, constant regular expression patterns +and package names. =item * -Character classes in regular expressions match characters instead of -bytes, and match against the character properties specified in the -Unicode properties database. So C<\w> can be used to match an ideograph, -for instance. +As a side effect, when this pragma is used within the main package, +it also enables Unicode character semantics for the entire program. +See L<perlunicode> for more on that. -=item * - -Named Unicode properties and block ranges make be used as character -classes via the new C<\p{}> (matches property) and C<\P{}> (doesn't -match property) constructs. For instance, C<\p{Lu}> matches any -character with the Unicode uppercase property, while C<\p{M}> matches -any mark character. Single letter properties may omit the brackets, so -that can be written C<\pM> also. Many predefined character classes are -available, such as C<\p{IsMirrored}> and C<\p{InTibetan}>. +[XXX: split this out into separate "pragma" and/or -C command-line +switch?] =item * -The special pattern C<\X> match matches any extended Unicode sequence -(a "combining character sequence" in Standardese), where the first -character is a base character and subsequent characters are mark -characters that apply to the base character. It is equivalent to -C<(?:\PM\pM*)>. - -=item * +In the absence of inputs marked as UTF-8, regular expressions within the +scope of this pragma will default to using character semantics instead +of byte semantics. -The C<tr///> operator translates characters instead of bytes. It can also -be forced to translate between 8-bit codes and UTF-8 regardless of the -surrounding utf8 state. For instance, if you know your input in Latin-1, -you can say: - - use utf8; - while (<>) { - tr/\0-\xff//CU; # latin1 char to utf8 - ... + @bytes_or_chars = split //, $data; # may split to bytes if data + # $data isn't UTF-8 + { + use utf8; # force char semantics + @chars = split //, $data; # splits characters } -Similarly you could translate your output with - - tr/\0-\x{ff}//UC; # utf8 to latin1 char - -No, C<s///> doesn't take /U or /C (yet?). - -=item * - -Case translation operators use the Unicode case translation tables. -Note that C<uc()> translates to uppercase, while C<ucfirst> translates -to titlecase (for languages that make the distinction). Naturally -the corresponding backslash sequences have the same semantics. - -=item * - -Most operators that deal with positions or lengths in the string will -automatically switch to using character positions, including C<chop()>, -C<substr()>, C<pos()>, C<index()>, C<rindex()>, C<sprintf()>, -C<write()>, and C<length()>. Operators that specifically don't switch -include C<vec()>, C<pack()>, and C<unpack()>. Operators that really -don't care include C<chomp()>, as well as any other operator that -treats a string as a bucket of bits, such as C<sort()>, and the -operators dealing with filenames. - -=item * - -The C<pack()>/C<unpack()> letters "C<c>" and "C<C>" do I<not> change, -since they're often used for byte-oriented formats. (Again, think -"C<char>" in the C language.) However, there is a new "C<U>" specifier -that will convert between UTF-8 characters and integers. (It works -outside of the utf8 pragma too.) - -=item * - -The C<chr()> and C<ord()> functions work on characters. This is like -C<pack("U")> and C<unpack("U")>, not like C<pack("C")> and -C<unpack("C")>. In fact, the latter are how you now emulate -byte-oriented C<chr()> and C<ord()> under utf8. - -=item * - -And finally, C<scalar reverse()> reverses by character rather than by byte. - -=back - -=head1 CAVEATS - -As of yet, there is no method for automatically coercing input and -output to some encoding other than UTF-8. This is planned in the near -future, however. +[XXX: Should this should be enabled like chr()/sprintf("%c") by looking +at $^U instead?] -In any event, you'll need to keep track of whether interfaces to other -modules expect UTF-8 data or something else. The utf8 pragma does not -magically mark strings for you in order to remember their encoding, nor -will any automatic coercion happen (other than that eventually planned -for I/O). If you want such automatic coercion, you can build yourself -a set of pretty object-oriented modules. Expect it to run considerably -slower than than this low-level support. +=head1 SEE ALSO -Use of locales with utf8 may lead to odd results. Currently there is -some attempt to apply 8-bit locale info to characters in the range -0..255, but this is demonstrably incorrect for locales that use -characters above that range (when mapped into Unicode). It will also -tend to run slower. Avoidance of locales is strongly encouraged. +L<perlunicode>, L<byte> =cut @@ -2867,7 +2867,7 @@ Perl_pmruntime(pTHX_ OP *o, OP *expr, OP *repl) p = SvPV(pat, plen); pm->op_pmflags |= PMf_SKIPWHITE; } - else if (DO_UTF8(pat)) + if ((PL_hints & HINT_UTF8) || (SvUTF8(pat) && !(PL_hints & HINT_BYTE))) pm->op_pmdynflags |= PMdf_UTF8; pm->op_pmregexp = CALLREGCOMP(aTHX_ p, p + plen, pm); if (strEQ("\\s+", pm->op_pmregexp->precomp)) @@ -2875,6 +2875,8 @@ Perl_pmruntime(pTHX_ OP *o, OP *expr, OP *repl) op_free(expr); } else { + if (PL_hints & HINT_UTF8) + pm->op_pmdynflags |= PMdf_UTF8; if (pm->op_pmflags & PMf_KEEP || !(PL_hints & HINT_RE_EVAL)) expr = newUNOP((!(PL_hints & HINT_RE_EVAL) ? OP_REGCRESET @@ -3401,7 +3403,7 @@ Perl_newSTATEOP(pTHX_ I32 flags, char *label, OP *o) cop->op_ppaddr = PL_ppaddr[ OP_NEXTSTATE ]; } cop->op_flags = flags; - cop->op_private = (PL_hints & (HINT_UTF8|HINT_BYTE)); + cop->op_private = (PL_hints & HINT_BYTE); #ifdef NATIVE_HINTS cop->op_private |= NATIVE_HINTS; #endif @@ -2467,8 +2467,8 @@ enum { /* pass one of these to get_vtbl */ #define HINT_INTEGER 0x00000001 #define HINT_STRICT_REFS 0x00000002 /* #define HINT_notused4 0x00000004 */ -#define HINT_UTF8 0x00000008 -#define HINT_BYTE 0x00000010 +#define HINT_BYTE 0x00000008 +/* #define HINT_notused10 0x00000010 */ /* Note: 20,40,80 used for NATIVE_HINTS */ #define HINT_BLOCK_SCOPE 0x00000100 @@ -2487,6 +2487,7 @@ enum { /* pass one of these to get_vtbl */ #define HINT_RE_EVAL 0x00200000 #define HINT_FILETEST_ACCESS 0x00400000 +#define HINT_UTF8 0x00800000 /* Various states of an input record separator SV (rs, nrs) */ #define RsSNARF(sv) (! SvOK(sv)) diff --git a/pod/perldelta.pod b/pod/perldelta.pod index cba167f652..2b5e4e7233 100644 --- a/pod/perldelta.pod +++ b/pod/perldelta.pod @@ -378,8 +378,9 @@ building and installing from source, the defaults should be fine. =head2 Unicode and UTF-8 support Perl can optionally use UTF-8 as its internal representation for character -strings. The C<utf8> pragma enables this support in the current lexical -scope. See L<utf8> for more information. +strings. The C<utf8> and C<byte> pragmas are used to control this support +in the current lexical scope. See L<perlunicode>, L<utf8> and L<byte> for +more information. =head2 Interpreter cloning, threads, and concurrency @@ -1653,6 +1654,10 @@ A tutorial that introduces the essentials of references. A tutorial on managing class data for object modules. +=item perlunicode.pod + +An introduction to Unicode support features in Perl. + =back =head1 New or Changed Diagnostics diff --git a/pod/perlunicode.pod b/pod/perlunicode.pod new file mode 100644 index 0000000000..b0efcca8df --- /dev/null +++ b/pod/perlunicode.pod @@ -0,0 +1,202 @@ +=head1 NAME + +perlunicode - Unicode support in Perl + +=head1 DESCRIPTION + +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! + +Beginning with version 5.6, Perl uses logically wide characters to +represent strings internally. This internal representation of strings +uses the UTF-8 encoding. + +In future, Perl-level operations will expect to work with characters +rather than bytes, in general. + +However, Perl v5.6 aims to provide a safe migration path from byte +semantics to character semantics for programs. To preserve compatibility +with earlier versions of Perl which allowed byte semantics in Perl +operations (owing to the fact that the internal representation for +characters was in bytes) byte semantics will continue to be in effect +until a the C<utf8> pragma is used in the C<main> package, or the C<$^U> +global flag is explicitly set. + +Under character semantics, many operations that formerly operated on +bytes change to operating on characters. For ASCII data this makes +no difference, because UTF-8 stores ASCII in single bytes, but for +any character greater than C<chr(127)>, the character is stored in +a sequence of two or more bytes, all of which have the high bit set. +But by and large, the user need not worry about this, because Perl +hides it from the user. A character in Perl is logically just a number +ranging from 0 to 2**32 or so. Larger characters encode to longer +sequences of bytes internally, but again, this is just an internal +detail which is hidden at the Perl level. + +The C<byte> pragma can be used to force byte semantics in a particular +lexical scope. See L<byte>. + +The C<utf8> pragma is a compatibility device to enables recognition +of UTF-8 in literals encountered by the parser. It is also used +for enabling some experimental Unicode support features. Note that +this pragma is only required until a future version of Perl in which +character semantics will become the default. This pragma may then +become a no-op. See L<utf8>. + +Character semantics have the following effects: + +=over 4 + +=item * + +Strings and patterns may contain characters that have an ordinal value +larger than 255. In Perl v5.6, this is only enabled if the lexical +scope has a C<use utf8> declaration (due to compatibility needs) but +future versions may enable this by default. + +Presuming you use a Unicode editor to edit your program, such characters +will typically occur directly within the literal strings as UTF-8 +characters, but you can also specify a particular character with an +extension of the C<\x> notation. UTF-8 characters are specified by +putting the hexadecimal code within curlies after the C<\x>. For instance, +a Unicode smiley face is C<\x{263A}>. A character in the Latin-1 range +(128..255) should be written C<\x{ab}> rather than C<\xab>, since the +former will turn into a two-byte UTF-8 code, while the latter will +continue to be interpreted as generating a 8-bit byte rather than a +character. In fact, if C<-w> is turned on, it will produce a warning +that you might be generating invalid UTF-8. + +=item * + +Identifiers within the Perl script may contain Unicode alphanumeric +characters, including ideographs. (You are currently on your own when +it comes to using the canonical forms of characters--Perl doesn't (yet) +attempt to canonicalize variable names for you.) + +This also needs C<use utf8> currently. [XXX: Why? High-bit chars were +syntax errors when they occurred within identifiers in previous versions, +so this should be enabled by default.] + +=item * + +Regular expressions match characters instead of bytes. For instance, +"." matches a character instead of a byte. (However, the C<\C> pattern +is provided to force a match a single byte ("C<char>" in C, hence +C<\C>).) + +Unicode support in regular expressions needs C<use utf8> currently. +[XXX: Because the SWASH routines need to be loaded. And the RE engine +appears to need an overhaul to Unicode by default anyway.] + +=item * + +Character classes in regular expressions match characters instead of +bytes, and match against the character properties specified in the +Unicode properties database. So C<\w> can be used to match an ideograph, +for instance. + +C<use utf8> is needed to enable this. See above. + +=item * + +Named Unicode properties and block ranges make be used as character +classes via the new C<\p{}> (matches property) and C<\P{}> (doesn't +match property) constructs. For instance, C<\p{Lu}> matches any +character with the Unicode uppercase property, while C<\p{M}> matches +any mark character. Single letter properties may omit the brackets, so +that can be written C<\pM> also. Many predefined character classes are +available, such as C<\p{IsMirrored}> and C<\p{InTibetan}>. + +C<use utf8> is needed to enable this. See above. + +=item * + +The special pattern C<\X> match matches any extended Unicode sequence +(a "combining character sequence" in Standardese), where the first +character is a base character and subsequent characters are mark +characters that apply to the base character. It is equivalent to +C<(?:\PM\pM*)>. + +C<use utf8> is needed to enable this. See above. + +=item * + +The C<tr///> operator translates characters instead of bytes. It can also +be forced to translate between 8-bit codes and UTF-8 regardless of the +surrounding utf8 state. For instance, if you know your input in Latin-1, +you can say: + + use utf8; + while (<>) { + tr/\0-\xff//CU; # latin1 char to utf8 + ... + } + +Similarly you could translate your output with + + tr/\0-\x{ff}//UC; # utf8 to latin1 char + +No, C<s///> doesn't take /U or /C (yet?). + +C<use utf8> is needed to enable this. See above. + +=item * + +Case translation operators use the Unicode case translation tables +when provided character input. Note that C<uc()> translates to +uppercase, while C<ucfirst> translates to titlecase (for languages +that make the distinction). Naturally the corresponding backslash +sequences have the same semantics. + +=item * + +Most operators that deal with positions or lengths in the string will +automatically switch to using character positions, including C<chop()>, +C<substr()>, C<pos()>, C<index()>, C<rindex()>, C<sprintf()>, +C<write()>, and C<length()>. Operators that specifically don't switch +include C<vec()>, C<pack()>, and C<unpack()>. Operators that really +don't care include C<chomp()>, as well as any other operator that +treats a string as a bucket of bits, such as C<sort()>, and the +operators dealing with filenames. + +=item * + +The C<pack()>/C<unpack()> letters "C<c>" and "C<C>" do I<not> change, +since they're often used for byte-oriented formats. (Again, think +"C<char>" in the C language.) However, there is a new "C<U>" specifier +that will convert between UTF-8 characters and integers. (It works +outside of the utf8 pragma too.) + +=item * + +The C<chr()> and C<ord()> functions work on characters. This is like +C<pack("U")> and C<unpack("U")>, not like C<pack("C")> and +C<unpack("C")>. In fact, the latter are how you now emulate +byte-oriented C<chr()> and C<ord()> under utf8. + +=item * + +And finally, C<scalar reverse()> reverses by character rather than by byte. + +=back + +=head1 CAVEATS + +As of yet, there is no method for automatically coercing input and +output to some encoding other than UTF-8. This is planned in the near +future, however. + +Whether a piece of data will be treated as "characters" or "bytes" +by internal operations cannot be divined at the current time. + +Use of locales with utf8 may lead to odd results. Currently there is +some attempt to apply 8-bit locale info to characters in the range +0..255, but this is demonstrably incorrect for locales that use +characters above that range (when mapped into Unicode). It will also +tend to run slower. Avoidance of locales is strongly encouraged. + +=head1 SEE ALSO + +L<byte>, L<utf8>, L<perlvar/"$^U"> + +=cut @@ -116,8 +116,6 @@ PP(pp_regcomp) pm->op_pmflags = pm->op_pmpermflags; /* reset case sensitivity */ if (DO_UTF8(tmpstr)) pm->op_pmdynflags |= PMdf_UTF8; - else - pm->op_pmdynflags &= ~PMdf_UTF8; pm->op_pmregexp = CALLREGCOMP(aTHX_ t, t + len, pm); PL_reginterp_cnt = 0; /* XXXX Be extra paranoid - needed inside tie/overload accessors. */ @@ -1339,9 +1339,8 @@ Perl_pregcomp(pTHX_ char *exp, char *xend, PMOP *pm) if (exp == NULL) FAIL("NULL regexp argument"); - if (PL_curcop == &PL_compiling ? (PL_hints & HINT_UTF8) : IN_UTF8) { + if (pm->op_pmdynflags & PMdf_UTF8) { PL_reg_flags |= RF_utf8; - pm->op_pmdynflags |= PMdf_UTF8; } else PL_reg_flags = 0; @@ -2335,6 +2335,7 @@ S_regmatch(pTHX_ regnode *prog) I32 onpar = PL_regnpar; pm.op_pmflags = 0; + pm.op_pmdynflags = (UTF||DO_UTF8(ret) ? PMdf_UTF8 : 0); re = CALLREGCOMP(aTHX_ t, t + len, &pm); if (!(SvFLAGS(ret) & (SVs_TEMP | SVs_PADTMP | SVf_READONLY))) @@ -26,7 +26,7 @@ EXTCONST unsigned char PL_utf8skip[]; END_EXTERN_C -#define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) +/*#define IN_UTF8 (PL_curcop->op_private & HINT_UTF8)*/ #define IN_BYTE (PL_curcop->op_private & HINT_BYTE) #define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE) |