Make print, syswrite, send, readline, getc honour utf8-ness of PerlIO.

(sysread, recv and write i.e. formats still to do...)
Allow :utf8 or :bytes in PerlIO_apply_layers() so that
   open($fh,">:utf8","name")
etc. work. - "applying" those just sets/clears the UTF8 bit of the top layer,
so no extra overhead is involved.
Tweak t/comp/require.t to add a 'use bytes' to permit its dubious writing of BOM
to a non-utf8 stream.
Add initial io/utf8.t
Fix SvPVutf8() - sv_2pv() was not expecting to be called with something
that was already SvPOK() - (we just fossiked with SvUTF8 bit). Fix that
and also just use the SvPV macro in sv_2pvutf8() to avoid the issue/overhead.

p4raw-id: //depot/perlio@8054

1 files changed, 30 insertions, 4 deletions

diff --git a/sv.c b/sv.c
index 87da8f7a3c..3d25a2eede 100644
--- a/sv.c
+++ b/sv.c
@@ -2192,7 +2192,11 @@ Perl_sv_2pv(pTHX_ register SV *sv, STRLEN *lp)
 	    return "";
 	}
     }
-    if (SvNOKp(sv)) {			/* See note in sv_2uv() */
+    if (SvPOK(sv)) {
+	*lp = SvCUR(sv);
+	return SvPVX(sv);
+    }
+    else if (SvNOKp(sv)) {			/* See note in sv_2uv() */
 	/* XXXX 64-bit?  IV may have better precision... */
 	/* I tried changing this to be 64-bit-aware and
 	 * the t/op/numconvert.t became very, very, angry.
@@ -2328,7 +2332,7 @@ char *
 Perl_sv_2pvutf8(pTHX_ register SV *sv, STRLEN *lp)
 {
     sv_utf8_upgrade(sv);
-    return sv_2pv(sv,lp);
+    return SvPV(sv,*lp);
 }
 
 /* This function is only called on magical items */
@@ -4330,14 +4334,31 @@ Perl_sv_gets(pTHX_ register SV *sv, register PerlIO *fp, I32 append)
 #endif
       SvCUR_set(sv, bytesread);
       buffer[bytesread] = '\0';
+      if (PerlIO_isutf8(fp))
+	SvUTF8_on(sv);
+      else
+	SvUTF8_off(sv);
       return(SvCUR(sv) ? SvPVX(sv) : Nullch);
     }
     else if (RsPARA(PL_rs)) {
 	rsptr = "\n\n";
 	rslen = 2;
     }
-    else
-	rsptr = SvPV(PL_rs, rslen);
+    else {
+	/* Get $/ i.e. PL_rs into same encoding as stream wants */
+	if (PerlIO_isutf8(fp)) {
+	    rsptr = SvPVutf8(PL_rs, rslen);
+	}
+	else {
+	    if (SvUTF8(PL_rs)) {
+		if (!sv_utf8_downgrade(PL_rs, TRUE)) {
+		    Perl_croak(aTHX_ "Wide character in $/");
+		}
+	    }
+	    rsptr = SvPV(PL_rs, rslen);
+	}
+    }
+
     rslast = rslen ? rsptr[rslen - 1] : '\0';
 
     if (RsPARA(PL_rs)) {		/* have to do this both before and after */
@@ -4556,6 +4577,11 @@ screamer2:
 	}
     }
 
+    if (PerlIO_isutf8(fp))
+	SvUTF8_on(sv);
+    else
+	SvUTF8_off(sv);
+
     return (SvCUR(sv) - append) ? SvPVX(sv) : Nullch;
 }