Integrate perlio:

[  8055]
Make write() (i.e. formats) utf8-aware by calling do_print() rather
than raw PerlIO_write() - does not seem to break any tests.

[  8054]
Make print, syswrite, send, readline, getc honour utf8-ness of PerlIO.
(sysread, recv and write i.e. formats still to do...)
Allow :utf8 or :bytes in PerlIO_apply_layers() so that
open($fh,">:utf8","name")
etc. work. - "applying" those just sets/clears the UTF8 bit of the top layer,
so no extra overhead is involved.
Tweak t/comp/require.t to add a 'use bytes' to permit its dubious writing of BOM
to a non-utf8 stream.
Add initial io/utf8.t
Fix SvPVutf8() - sv_2pv() was not expecting to be called with something
that was already SvPOK() - (we just fossiked with SvUTF8 bit). Fix that
and also just use the SvPV macro in sv_2pvutf8() to avoid the issue/overhead.

[  8053]
Did not get that has_utf8/this_utf8 fix right last time, another spot
was only testing this_utf8.

[  8052]
Typo/thinko in S_scan_const() - seeing high bit sets has_utf8 not this_utf8
i.e. the output string has one, but don't mess with source assumption.

p4raw-link: @8055 on //depot/perlio: d75029d0f4549b2fe0cff3cd80934a1b6bbb54eb
p4raw-link: @8054 on //depot/perlio: 7d59b7e40bca518078f3e97c802950b76d52efa2
p4raw-link: @8053 on //depot/perlio: bbc28b27949817e8e7461c0a92c6108632259a4b
p4raw-link: @8052 on //depot/perlio: 4e553d734e1cb450bb69a562e02eb0c12ecd8995

p4raw-id: //depot/perl@8058

1 files changed, 30 insertions, 4 deletions

diff --git a/sv.c b/sv.c
index 56e9f8d264..c6b718c995 100644
--- a/sv.c
+++ b/sv.c
@@ -2192,7 +2192,11 @@ Perl_sv_2pv(pTHX_ register SV *sv, STRLEN *lp)
 	    return "";
 	}
     }
-    if (SvNOKp(sv)) {			/* See note in sv_2uv() */
+    if (SvPOK(sv)) {
+	*lp = SvCUR(sv);
+	return SvPVX(sv);
+    }
+    else if (SvNOKp(sv)) {			/* See note in sv_2uv() */
 	/* XXXX 64-bit?  IV may have better precision... */
 	/* I tried changing this to be 64-bit-aware and
 	 * the t/op/numconvert.t became very, very, angry.
@@ -2328,7 +2332,7 @@ char *
 Perl_sv_2pvutf8(pTHX_ register SV *sv, STRLEN *lp)
 {
     sv_utf8_upgrade(sv);
-    return sv_2pv(sv,lp);
+    return SvPV(sv,*lp);
 }
 
 /* This function is only called on magical items */
@@ -4330,14 +4334,31 @@ Perl_sv_gets(pTHX_ register SV *sv, register PerlIO *fp, I32 append)
 #endif
       SvCUR_set(sv, bytesread);
       buffer[bytesread] = '\0';
+      if (PerlIO_isutf8(fp))
+	SvUTF8_on(sv);
+      else
+	SvUTF8_off(sv);
       return(SvCUR(sv) ? SvPVX(sv) : Nullch);
     }
     else if (RsPARA(PL_rs)) {
 	rsptr = "\n\n";
 	rslen = 2;
     }
-    else
-	rsptr = SvPV(PL_rs, rslen);
+    else {
+	/* Get $/ i.e. PL_rs into same encoding as stream wants */
+	if (PerlIO_isutf8(fp)) {
+	    rsptr = SvPVutf8(PL_rs, rslen);
+	}
+	else {
+	    if (SvUTF8(PL_rs)) {
+		if (!sv_utf8_downgrade(PL_rs, TRUE)) {
+		    Perl_croak(aTHX_ "Wide character in $/");
+		}
+	    }
+	    rsptr = SvPV(PL_rs, rslen);
+	}
+    }
+
     rslast = rslen ? rsptr[rslen - 1] : '\0';
 
     if (RsPARA(PL_rs)) {		/* have to do this both before and after */
@@ -4556,6 +4577,11 @@ screamer2:
 	}
     }
 
+    if (PerlIO_isutf8(fp))
+	SvUTF8_on(sv);
+    else
+	SvUTF8_off(sv);
+
     return (SvCUR(sv) - append) ? SvPVX(sv) : Nullch;
 }