diff options
| author | M. J. T. Guy <mjtg@cus.cam.ac.uk> | 2000-07-25 13:52:45 +0100 |
|---|---|---|
| committer | Jarkko Hietaniemi <jhi@iki.fi> | 2000-07-25 13:59:28 +0000 |
| commit | 4ac469dc3d5bdda48d1fc1e1ea92a82232cc5603 (patch) | |
| tree | 245564d3a87f911b5bb3077960f5b71e5685f21f | |
| parent | 297b36dcf845fc5195afb0a1d7e83d539bb1d4ed (diff) | |
| download | perl-4ac469dc3d5bdda48d1fc1e1ea92a82232cc5603.tar.gz | |
Get UTF16 BOMs working. Patch from
Subject: Re: [ID 20000719.001] Problem with bleadperl of 7/18/00
Date: Tue, 25 Jul 2000 12:52:45 +0100
Message-Id: <E13H3GP-0004MR-00@libra.cus.cam.ac.uk>
and notes from
Subject: Re: [ID 20000719.001] Problem with bleadperl of 7/18/00
From: "M.J.T. Guy" <mjtg@cus.cam.ac.uk>
Date: Tue, 25 Jul 2000 11:43:25 +0100
Message-Id: <E13H2BJ-0002nG-00@libra.cus.cam.ac.uk>
p4raw-id: //depot/perl@6435
| -rwxr-xr-x | t/comp/require.t | 3 | ||||
| -rw-r--r-- | toke.c | 6 | ||||
| -rw-r--r-- | utf8.c | 9 |
3 files changed, 16 insertions, 2 deletions
diff --git a/t/comp/require.t b/t/comp/require.t index 48e3e0038b..51f513f5c8 100755 --- a/t/comp/require.t +++ b/t/comp/require.t @@ -132,7 +132,8 @@ $i++; do_require("$utf8\nprint \"ok $i\n\"; 1;\n"); $i++; do_require("$utf16\n1;"); -print "ok $i\n" if $@ =~ /Unsupported script encoding/; +print "not " unless $@ =~ /^Unrecognized character /; +print "ok $i\n"; END { 1 while unlink 'bleah.pm'; 1 while unlink 'bleah.do'; } @@ -2519,7 +2519,13 @@ Perl_yylex(pTHX) } } if (bof) + { + PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr); + /* Shouldn't this wsallow_bom() be earlier, e.g. + * immediately after where bof is set? Currently you can't + * have e.g. a UTF16 sharpbang line. --Mike Guy */ s = swallow_bom((U8*)s); + } incline(s); } while (PL_doextract); PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = s; @@ -320,13 +320,20 @@ Perl_bytes_to_utf8(pTHX_ U8* s, STRLEN *len) return dst; } -/* XXX NOTHING CALLS THE FOLLOWING TWO ROUTINES YET!!! */ /* * Convert native or reversed UTF-16 to UTF-8. * * Destination must be pre-extended to 3/2 source. Do not use in-place. * We optimize for native, for obvious reasons. */ +/* There are several problems with utf16_to_utf8(). + * (1) U16 is not necessarily *exactly* two bytes. + * (2) Secondly, no check is made for odd length. + * (3) Thirdly, the "Malformed UTF-16 surrogate" should probably be + * a hard error (and it should be listed in perldiag). + * (4) The tests (in comp/t/require.t) are a joke: the UTF16 BOM + * really ought to be followed by valid UTF16 characters. + * --Mike Guy */ U8* Perl_utf16_to_utf8(pTHX_ U16* p, U8* d, I32 bytelen) { |