Integrate perlio:

[  9417]
PerlIO.pm / perlio.pm conflict.
Update some pods.
Make t/op/utf8decode.t a text file.

p4raw-link: @9417 on //depot/perlio: 7d3b96bbd83d17f17b26b4c05ef623881c8477be

p4raw-id: //depot/perl@9418

7 files changed, 154 insertions, 140 deletions

diff --git a/MANIFEST b/MANIFEST
index 61346fb89e..056cbaf92c 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -851,7 +851,6 @@ lib/open2.pl		Open a two-ended pipe (uses IPC::Open2)
 lib/open3.pl		Open a three-ended pipe (uses IPC::Open3)
 lib/overload.pm		Module for overloading perl operators
 lib/perl5db.pl		Perl debugging routines
-lib/perlio.pm		Perl IO interface pragma
 lib/pwd.pl		Routines to keep track of PWD environment variable
 lib/shellwords.pl	Perl library to split into words with shell quoting
 lib/sigtrap.pm		For trapping an abort and giving traceback
diff --git a/lib/PerlIO.pm b/lib/PerlIO.pm
index 65b7ec3530..04cd4cfcc4 100644
--- a/lib/PerlIO.pm
+++ b/lib/PerlIO.pm
@@ -27,11 +27,15 @@ __END__
 
 =head1 NAME
 
-PerlIO - On demand loader for PerlIO::* name space
+PerlIO - On demand loader for PerlIO layers and root of PerlIO::* name space
 
 =head1 SYNOPSIS
 
-  open($fh,">:foo",...)
+  open($fh,">:crlf","my.txt")
+  open($fh,">:raw","his.jpg")
+
+  Shell:
+    PERLIO=perlio perl ....
 
 =head1 DESCRIPTION
 
@@ -47,7 +51,95 @@ The perl code in PerlIO.pm then attempts to locate a layer by doing
 Otherwise the C<PerlIO> package is a place holder for additional PerLIO related
 functions.
 
+The following layers are currently defined:
 
-=cut
+=over 4
+
+=item unix
+
+Low level layer which calls C<read>, C<write> and C<lseek> etc.
+
+=item stdio
+
+Layer which calls C<fread>, C<fwrite> and C<fseek>/C<ftell> etc.
+Note that as this is "real" stdio it will ignore any layers beneath it and
+got straight to the operating system via the C library as usual.
+
+=item perlio
+
+This is a re-implementation of "stdio-like" buffering written as a PerlIO "layer".
+As such it will call whatever layer is below it for its operations.
+
+=item crlf
+
+A layer which does CRLF to "\n" translation distinguishing "text" and "binary"
+files in the manner of MS-DOS and similar operating systems.
+
+=item utf8
+
+Declares that the stream accepts perl's internal encoding of characters.
+(Which really is UTF-8 on ASCII machines, but is UTF-EBCDIC on EBCDIC machines.)
+This allows any character perl can represent to be read from or written to the
+stream. The UTF-X encoding is chosen to render simple text parts (i.e.
+non-accented letters, digits and common punctuation) human readable in the
+encoded file.
+
+=item raw
+
+A pseudo-layer which performs two functions (which is messy, but necessary to
+maintain compatibility with non-PerLIO builds of perl and they way things
+have been documented elsewhere).
+
+Firstly it forces the file handle to be considered binary at that point
+in the layer stack,
+
+Secondly in prevents the IO system seaching back before it in the layer specification.
+Thus:
+
+    open($fh,":raw:perlio"),...)
+
+Forces the use of C<perlio> layer even if the platform default, or C<use open> default
+is something else (such as ":encoding(iso-8859-7)" ) which would interfere with
+binary nature of the stream.
 
+=back
+
+=head2 Defaults and how to override them
+
+If the platform is MS-DOS like and normally does CRLF to "\n" translation
+for text files then the default layers are :
+
+  unix crlf
+
+(The low level "unix" layer may be replaced by a platform specific low level layer.)
+
+Otherwise if C<Configure> found out how to do "fast" IO using system's stdio, then
+the default layers are :
+
+  unix stdio
+
+Otherwise the default layers are
+
+  unix perlio
+
+These defaults may change once perlio has been better tested and tuned.
+
+The default can be overridden by setting the environment variable PERLIO
+to a space separated list of layers (unix or platform low level layer is
+always pushed first).
+This can be used to see the effect of/bugs in the various layers e.g.
+
+  cd .../perl/t
+  PERLIO=stdio  ./perl harness
+  PERLIO=perlio ./perl harness
+
+=head1 AUTHOR
+
+Nick Ing-Simmons E<lt>nick@ing-simmons.netE<gt>
+
+=head1 SEE ALSO
+
+L<perlfunc/"binmode">, L<perlfunc/"open">, L<perlunicode>, L<open>
+
+=cut
 
diff --git a/lib/open.pm b/lib/open.pm
index 53ae308310..1aef9040b5 100644
--- a/lib/open.pm
+++ b/lib/open.pm
@@ -2,6 +2,7 @@ package open;
 use Carp;
 $open::hint_bits = 0x20000;
 
+# layers array and hash mainly manipulated by C code in perlio.c
 use vars qw(%layers @layers);
 
 # Populate hash in non-PerlIO case
@@ -58,44 +59,63 @@ open - perl pragma to set default disciplines for input and output
 
 =head1 DESCRIPTION
 
-The open pragma is used to declare one or more default disciplines for
-I/O operations.  Any open() and readpipe() (aka qx//) operators found
-within the lexical scope of this pragma will use the declared defaults.
-Neither open() with an explicit set of disciplines, nor sysopen() are
-influenced by this pragma.
+Full-fledged support for I/O disciplines is now implemented provided perl is
+configured to use PerlIO as its IO system (which is now the default).
 
-Only the two pseudo-disciplines ":raw" and ":crlf" are currently
-available.
+The C<open> pragma serves as one of the interfaces to declare default
+"layers" (aka disciplines) for all I/O.
+
+The C<open> pragma is used to declare one or more default layers for
+I/O operations.  Any open(), readpipe() (aka qx//) and similar operators
+found within the lexical scope of this pragma will use the declared defaults.
+
+When open() is given an explicit list of layers they are appended to the
+list declared using this pragma.
+
+Directory handles may also support disciplines in future.
+
+=head1 NONPERLIO FUNCTIONALITY
+
+If perl is not built to use PerlIO as its IO system then only the two pseudo-disciplines
+":raw" and ":crlf" are available.
 
 The ":raw" discipline corresponds to "binary mode" and the ":crlf"
 discipline corresponds to "text mode" on platforms that distinguish
 between the two modes when opening files (which is many DOS-like
-platforms, including Windows).  These two disciplines are currently
-no-ops on platforms where binmode() is a no-op, but will be
-supported everywhere in future.
+platforms, including Windows).  These two disciplines are
+no-ops on platforms where binmode() is a no-op, but perform their
+functions everywhere if PerlIO is enabled.
+
+=head1 IMPLEMENTATION DETAILS
 
-=head1 UNIMPLEMENTED FUNCTIONALITY
+There are two package variables C<%layers> and C<@layers> which
+are mainly manipulated by C code in F<perlio.c>, but are visible
+to the nosy:
 
-Full-fledged support for I/O disciplines is currently unimplemented.
-When they are eventually supported, this pragma will serve as one of
-the interfaces to declare default disciplines for all I/O.
+  print "Have ",join(',',keys %open::layers),"\n";
+  print "Using ",join(',',@open::layers),"\n";
 
-In future, any default disciplines declared by this pragma will be
-available by the special discipline name ":DEFAULT", and could be used
-within handle constructors that allow disciplines to be specified.
-This would make it possible to stack new disciplines over the default
-ones.
+The C<%open::layers> hash is a record of the available "layers" that may be pushed
+onto a C<PerlIO> stream. The values of the hash are perl objects, of class C<PerlIO::Layer>
+which are created by the C code in F<perlio.c>. As yet there is nothing useful you
+can do with the objects at the perl level.
 
-    open FH, "<:para :DEFAULT", $file or die "can't open $file: $!";
+The C<@open::layers> array is the current set of layers and their arguments.
+The array consists of layer => argument pairs and I<must> always have even number of
+entries and the even entries I<must> be C<PerlIO::Layer> objects or perl will "die"
+when it attempts to open a filehandle. In most cases the odd entry will be C<undef>,
+but in the case of (say) ":encoding(iso-8859-1)" it will be 'iso-8859-1'. These
+argument entries are currently restricted to being strings.
 
-Socket and directory handles will also support disciplines in
-future.
+When a new C<PerlIO> stream is opened, the C code looks at the
+array to determine the default layers to be pushed. So with care it is possible
+to manipulate the default layer "stack":
 
-Full support for I/O disciplines will enable all of the supported
-disciplines to work on all platforms.
+    splice(@PerlIO::layers,-2,2);
+    push(@PerlIO::layers,$PerlIO::layers{'stdio'} => undef);
 
 =head1 SEE ALSO
 
-L<perlfunc/"binmode">, L<perlfunc/"open">, L<perlunicode>
+L<perlfunc/"binmode">, L<perlfunc/"open">, L<perlunicode>, L<PerlIO>
 
 =cut
diff --git a/lib/perlio.pm b/lib/perlio.pm
deleted file mode 100644
index 48acfbbf0b..0000000000
--- a/lib/perlio.pm
+++ /dev/null
@@ -1,87 +0,0 @@
-package perlio;
-1;
-__END__
-
-=head1 NAME
-
-perlio - perl pragma to configure C level IO
-
-=head1 SYNOPSIS
-
-  Shell:
-    PERLIO=perlio perl ....
-
-    print "Have ",join(',',keys %perlio::layers),"\n";
-    print "Using ",join(',',@perlio::layers),"\n";
-
-
-=head1 DESCRIPTION
-
-Mainly a Place holder for now.
-
-The C<%perlio::layers> hash is a record of the available "layers" that may be pushed
-onto a C<PerlIO> stream.
-
-The C<@perlio::layers> array is the current set of layers that are used when
-a new C<PerlIO> stream is opened. The C code looks are the array each time
-a stream is opened so the "stack" can be manipulated by messing with the array :
-
-    pop(@perlio::layers);
-    push(@perlio::layers,$perlio::layers{'stdio'});
-
-The values if both the hash and the array are perl objects, of class C<perlio::Layer>
-which are created by the C code in C<perlio.c>. As yet there is nothing useful you
-can do with the objects at the perl level.
-
-There are three layers currently defined:
-
-=over 4
-
-=item unix
-
-Low level layer which calls C<read>, C<write> and C<lseek> etc.
-
-=item stdio
-
-Layer which calls C<fread>, C<fwrite> and C<fseek>/C<ftell> etc.
-Note that as this is "real" stdio it will ignore any layers beneath it and
-got straight to the operating system via the C library as usual.
-
-=item perlio
-
-This is a re-implementation of "stdio-like" buffering written as a PerlIO "layer".
-As such it will call whatever layer is below it for its operations.
-
-=back
-
-=head2 Defaults and how to override them
-
-If C<Configure> found out how to do "fast" IO using system's stdio, then
-the default layers are :
-
-  unix stdio
-
-Otherwise the default layers are
-
-  unix perlio
-
-(STDERR will have just unix in this case as that is optimal way to make it
-"unbuffered" - do not add a buffering layer!)
-
-The default may change once perlio has been better tested and tuned.
-
-The default can be overridden by setting the environment variable PERLIO
-to a space separated list of layers (unix is always pushed first).
-This can be used to see the effect of/bugs in the various layers e.g.
-
-  cd .../perl/t
-  PERLIO=stdio  ./perl harness
-  PERLIO=perlio ./perl harness
-
-=head1 AUTHOR
-
-Nick Ing-Simmons E<lt>nick@ing-simmons.netE<gt>
-
-=cut
-
-
diff --git a/perlio.c b/perlio.c
index 194c14fb35..6c4a3981b8 100644
--- a/perlio.c
+++ b/perlio.c
@@ -365,26 +365,6 @@ PerlIO_pop(pTHX_ PerlIO *f)
 /*--------------------------------------------------------------------------------------*/
 /* XS Interface for perl code */
 
-XS(XS_perlio_import)
-{
- dXSARGS;
- GV *gv = CvGV(cv);
- char *s = GvNAME(gv);
- STRLEN l = GvNAMELEN(gv);
- PerlIO_debug("%.*s\n",(int) l,s);
- XSRETURN_EMPTY;
-}
-
-XS(XS_perlio_unimport)
-{
- dXSARGS;
- GV *gv = CvGV(cv);
- char *s = GvNAME(gv);
- STRLEN l = GvNAMELEN(gv);
- PerlIO_debug("%.*s\n",(int) l,s);
- XSRETURN_EMPTY;
-}
-
 SV *
 PerlIO_find_layer(pTHX_ const char *name, STRLEN len)
 {
@@ -499,7 +479,7 @@ XS(XS_io_MODIFY_SCALAR_ATTRIBUTES)
 SV *
 PerlIO_tab_sv(pTHX_ PerlIO_funcs *tab)
 {
- HV *stash = gv_stashpv("perlio::Layer", TRUE);
+ HV *stash = gv_stashpv("PerlIO::Layer", TRUE);
  SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(tab))),stash);
  return sv;
 }
@@ -648,8 +628,6 @@ PerlIO_default_layers(pTHX)
   {
    const char *s  = (PL_tainting) ? Nullch : PerlEnv_getenv("PERLIO");
    PerlIO_layer_av = get_av("open::layers",GV_ADD|GV_ADDMULTI);
-   newXS("perlio::import",XS_perlio_import,__FILE__);
-   newXS("perlio::unimport",XS_perlio_unimport,__FILE__);
 #if 0
    newXS("io::MODIFY_SCALAR_ATTRIBUTES",XS_io_MODIFY_SCALAR_ATTRIBUTES,__FILE__);
 #endif
diff --git a/t/lib/b.t b/t/lib/b.t
index b10479ddff..0589d16d75 100755
--- a/t/lib/b.t
+++ b/t/lib/b.t
@@ -140,7 +140,7 @@ ok;
 
 chomp($a = `$^X $path "-MB::Stash" "-Mwarnings" -e1`);
 $a = join ',', sort split /,/, $a;
-$a =~ s/-u(perlio|open)(?:::\w+)?,//g if defined $Config{'useperlio'} and $Config{'useperlio'} eq 'define';
+$a =~ s/-u(PerlIO|open)(?:::\w+)?,//g if defined $Config{'useperlio'} and $Config{'useperlio'} eq 'define';
 $a =~ s/-uWin32,// if $^O eq 'MSWin32';
 $a =~ s/-u(Cwd|File|File::Copy|OS2),//g if $^O eq 'os2';
 $a =~ s/-uCwd,// if $^O eq 'cygwin';
diff --git a/t/op/utf8decode.t b/t/op/utf8decode.t
index cc2b26aaf3..04656ec9f6 100644
--- a/t/op/utf8decode.t
+++ b/t/op/utf8decode.t
@@ -8,6 +8,18 @@ BEGIN {
 {
     my $wide = v256;
     use bytes;
+    if (ord($wide) == 140) {
+	print "1..0 # Skip: UTF-EBCDIC (not UTF-8) used here\n";
+	exit 0;
+    }
+    elsif (ord($wide) != 196) {
+	warn sprintf("v256 starts with %02X\n",ord($wide));
+    }
+}
+
+{
+    my $wide = v256;
+    use bytes;
     my $ordwide = ord($wide);
     printf "# under use bytes ord(v256) = 0x%02x\n", $ordwide;
     if ($ordwide == 140) {