ExtUtils::Constant 0.13

Message-ID: <20020823182111.GA281@Bagpuss.unfortu.net>

p4raw-id: //depot/perl@17776

2 files changed, 211 insertions, 59 deletions

diff --git a/lib/ExtUtils/Constant.pm b/lib/ExtUtils/Constant.pm
index 1268ce02ba..0772ee8517 100644
--- a/lib/ExtUtils/Constant.pm
+++ b/lib/ExtUtils/Constant.pm
@@ -1,6 +1,6 @@
 package ExtUtils::Constant;
 use vars qw (@ISA $VERSION %XS_Constant %XS_TypeSet @EXPORT_OK %EXPORT_TAGS);
-$VERSION = '0.12';
+$VERSION = '0.13';
 
 =head1 NAME
 
@@ -94,8 +94,11 @@ if ($] >= 5.006) {
   eval "use warnings; 1" or die $@;
 }
 use strict;
+use vars '$is_perl56';
 use Carp;
 
+$is_perl56 = ($] < 5.007 && $] > 5.005_50);
+
 use Exporter;
 use Text::Wrap;
 $Text::Wrap::huge = 'overflow';
@@ -150,7 +153,12 @@ characters.
 sub C_stringify {
   local $_ = shift;
   return unless defined $_;
-  confess "Wide character in '$_' intended as a C identifier" if tr/\0-\377//c;
+  # grr 5.6.1
+  confess "Wide character in '$_' intended as a C identifier"
+    if tr/\0-\377// != length;
+  # grr 5.6.1 moreso because its regexps will break on data that happens to
+  # be utf8, which includes my 8 bit test cases.
+  $_ = pack 'C*', unpack 'U*', $_ . pack 'U*' if $is_perl56;
   s/\\/\\\\/g;
   s/([\"\'])/\\$1/g;	# Grr. fix perl mode.
   s/\n/\\n/g;		# Ensure newlines don't end up in octal
@@ -189,13 +197,27 @@ sub perl_stringify {
   s/\t/\\t/g;
   s/\f/\\f/g;
   s/\a/\\a/g;
-  s/([^\0-\177])/sprintf "\\x{%X}", ord $1/ge;
   unless ($] < 5.006) {
+    if ($] > 5.007) {
+      s/([^\0-\177])/sprintf "\\x{%X}", ord $1/ge;
+    } else {
+      # Grr 5.6.1. And I don't think I can use utf8; to force the regexp
+      # because 5.005_03 will fail.
+      # This is grim, but I also can't split on //
+      my $copy;
+      foreach my $index (0 .. length ($_) - 1) {
+        my $char = substr ($_, $index, 1);
+        $copy .= ($char le "\177") ? $char : sprintf "\\x{%X}", ord $char;
+      }
+      $_ = $copy;
+    }
     # This will elicit a warning on 5.005_03 about [: :] being reserved unless
     # I cheat
     my $cheat = '([[:^print:]])';
     s/$cheat/sprintf "\\%03o", ord $1/ge;
   } else {
+    # Turns out "\x{}" notation only arrived with 5.6
+    s/([^\0-\177])/sprintf "\\x%02X", ord $1/ge;
     require POSIX;
     s/([^A-Za-z0-9_])/POSIX::isprint($1) ? $1 : sprintf "\\%03o", ord $1/ge;
   }
@@ -433,7 +455,7 @@ sub match_clause {
       $body .= $indent . "  } else {\n";
       $body .= return_clause ($no, 4 + length $indent); 
     }
-    $body .= $indent . "  }";
+    $body .= $indent . "  }\n";
   } else {
     $body .= return_clause ($item, 2 + length $indent);
   }
@@ -466,7 +488,11 @@ sub switch_clause {
   }
   my @safe_names = @names;
   foreach (@safe_names) {
-    next unless tr/A-Za-z0-9_//c;
+    confess sprintf "Name '$_' is length %d, not $namelen", length
+      unless length == $namelen;
+    # Argh. 5.6.1
+    # next unless tr/A-Za-z0-9_//c;
+    next if tr/A-Za-z0-9_// == length;
     $_ = '"' . perl_stringify ($_) . '"';
     # Ensure that the enclosing C comment doesn't end
     # by turning */  into *" . "/
@@ -481,9 +507,15 @@ sub switch_clause {
   foreach my $i (0 .. ($namelen - 1)) {
     my ($min, $max) = (~0, 0);
     my %spread;
+    if ($is_perl56) {
+      # Need proper Unicode preserving hash keys for bytes in range 128-255
+      # here too, for some reason. grr 5.6.1 yet again.
+      tie %spread, 'ExtUtils::Constant::Aaargh56Hash';
+    }
     foreach (@names) {
       my $char = substr $_, $i, 1;
       my $ord = ord $char;
+      confess "char $ord is out of range" if $ord > 255;
       $max = $ord if $ord > $max;
       $min = $ord if $ord < $min;
       push @{$spread{$char}}, $_;
@@ -515,6 +547,9 @@ sub switch_clause {
   $body .= $indent . "/* Offset $offset gives the best switch position.  */\n";
   $body .= $indent . "switch (name[$offset]) {\n";
   foreach my $char (sort keys %$best) {
+    confess sprintf "'$char' is %d bytes long, not 1", length $char
+      if length ($char) != 1;
+    confess sprintf "char %#X is out of range", ord $char if ord ($char) > 255;
     $body .= $indent . "case '" . C_stringify ($char) . "':\n";
     foreach my $name (sort @{$best->{$char}}) {
       my $thisone = $items->{$name};
@@ -581,7 +616,11 @@ sub dump_names {
         next if $_->{utf8} eq 'no';
         # Copy the hashref, as we don't want to mess with the caller's hashref.
         $_ = {%$_};
-        utf8::decode ($_->{name});
+        unless ($is_perl56) {
+          utf8::decode ($_->{name});
+        } else {
+          $_->{name} = pack 'U*', unpack 'U0U*', $_->{name};
+        }
         delete $_->{utf8};
       }
     } else {
@@ -589,7 +628,9 @@ sub dump_names {
       $type = $default_type;
     }
     $used_types{$type}++;
-    if ($type eq $default_type and 0 == ($_->{name} =~ tr/A-Za-z0-9_//c)
+    if ($type eq $default_type
+        # grr 5.6.1
+        and length $_->{name} == ($_->{name} =~ tr/A-Za-z0-9_//)
         and !defined ($_->{macro}) and !defined ($_->{value})
         and !defined ($_->{default}) and !defined ($_->{pre})
         and !defined ($_->{post}) and !defined ($_->{def_pre})
@@ -836,6 +877,11 @@ sub C_constant {
     # be a hashref, and pinch %$items from our parent to save recalculation.
     ($namelen, $items) = @$breakout;
   } else {
+    if ($is_perl56) {
+      # Need proper Unicode preserving hash keys.
+      $items = {};
+      tie %$items, 'ExtUtils::Constant::Aaargh56Hash';
+    }
     $breakout ||= 3;
     $default_type ||= 'IV';
     if (!ref $what) {
@@ -870,7 +916,10 @@ sub C_constant {
         $what->{$default_type} = 1;
       }
       warn "ExtUtils::Constant doesn't know how to handle values of type $_ used in macro $name" unless defined $XS_Constant{$item->{type}};
-      if ($name !~ tr/\0-\177//c) {
+      # tr///c is broken on 5.6.1 for utf8, so my original tr/\0-\177//c
+      # doesn't work. Upgrade to 5.8
+      # if ($name !~ tr/\0-\177//c || $] < 5.005_50) {
+      if ($name =~ tr/\0-\177// == length $name || $] < 5.005_50) {
         # No characters outside 7 bit ASCII.
         if (exists $items->{$name}) {
           die "Multiple definitions for macro $name";
@@ -881,7 +930,12 @@ sub C_constant {
         if (exists $items->{$name} and ref $items->{$name} ne 'ARRAY') {
           confess "Unexpected ASCII definition for macro $name";
         }
-        if ($name !~ tr/\0-\377//c) {
+        # Again, 5.6.1 tr broken, so s/5\.6.*/5\.8\.0/;
+        # if ($name !~ tr/\0-\377//c) {
+        if ($name =~ tr/\0-\377// == length $name) {
+#          if ($] < 5.007) {
+#            $name = pack "C*", unpack "U*", $name;
+#          }
           $item->{utf8} = 'no';
           $items->{$name}[1] = $item;
           push @new_items, $item;
@@ -889,7 +943,13 @@ sub C_constant {
           $item = {%$item};
         }
         # Encode the name as utf8 bytes.
-        utf8::encode($name);
+        unless ($is_perl56) {
+          utf8::encode($name);
+        } else {
+#          warn "Was >$name< " . length ${name};
+          $name = pack 'C*', unpack 'C*', $name . pack 'U*';
+#          warn "Now '${name}' " . length ${name};
+        }
         if ($items->{$name}[0]) {
           die "Multiple definitions for macro $name";
         }
@@ -936,7 +996,18 @@ sub C_constant {
       next unless $by_length[$i];	# None of this length
       $body .= "  case $i:\n";
       if (@{$by_length[$i]} == 1) {
-        $body .= match_clause ($by_length[$i]->[0]);
+        my $only_thing = $by_length[$i]->[0];
+        if ($only_thing->{utf8}) {
+          if ($only_thing->{utf8} eq 'yes') {
+            # With utf8 on flag item is passed in element 0
+            $body .= match_clause ([$only_thing]);
+          } else {
+            # With utf8 off flag item is passed in element 1
+            $body .= match_clause ([undef, $only_thing]);
+          }
+        } else {
+          $body .= match_clause ($only_thing);
+        }
       } elsif (@{$by_length[$i]} < $breakout) {
         $body .= switch_clause (4, '', $i, $items, @{$by_length[$i]});
       } else {
@@ -1224,7 +1295,7 @@ EOT
 
 Writes a file of C code and a file of XS code which you should C<#include>
 and C<INCLUDE> in the C and XS sections respectively of your module's XS
-code.  You probaby want to do this in your C<Makefile.PL>, so that you can
+code.  You probably want to do this in your C<Makefile.PL>, so that you can
 easily edit the list of constants without touching the rest of your module.
 The attributes supported are
 
@@ -1301,7 +1372,7 @@ sub WriteConstants {
   print $c_fh constant_types(); # macro defs
   print $c_fh "\n";
 
-  # indent is still undef. Until anyone implents indent style rules with it.
+  # indent is still undef. Until anyone implements indent style rules with it.
   foreach (C_constant ($ARGS{NAME}, $ARGS{C_SUBNAME}, $ARGS{DEFAULT_TYPE},
                        $types, undef, $ARGS{BREAKOUT_AT}, @{$ARGS{NAMES}})) {
     print $c_fh $_, "\n"; # C constant subs
@@ -1313,6 +1384,28 @@ sub WriteConstants {
   close $xs_fh or warn "Error closing $ARGS{XS_FILE}: $!";
 }
 
+package ExtUtils::Constant::Aaargh56Hash;
+# A support module (hack) to provide sane Unicode hash keys on 5.6.x perl
+use strict;
+require Tie::Hash if $ExtUtils::Constant::is_perl56;
+use vars '@ISA';
+@ISA = 'Tie::StdHash';
+
+#my $a;
+# Storing the values as concatenated BER encoded numbers is actually going to
+# be terser than using UTF8 :-)
+# And the tests are slightly faster. Ops are bad, m'kay
+sub to_key {pack "w*", unpack "U*", ($_[0] . pack "U*")};
+sub from_key {defined $_[0] ? pack "U*", unpack 'w*', $_[0] : undef};
+
+sub STORE    { $_[0]->{to_key($_[1])} = $_[2] }
+sub FETCH    { $_[0]->{to_key($_[1])} }
+sub FIRSTKEY { my $a = scalar keys %{$_[0]}; from_key (each %{$_[0]}) }
+sub NEXTKEY  { from_key (each %{$_[0]}) }
+sub EXISTS   { exists $_[0]->{to_key($_[1])} }
+sub DELETE   { delete $_[0]->{to_key($_[1])} }
+
+#END {warn "$a accesses";}
 1;
 __END__
 
diff --git a/lib/ExtUtils/t/Constant.t b/lib/ExtUtils/t/Constant.t
index 25d705585e..6356ab45a6 100644
--- a/lib/ExtUtils/t/Constant.t
+++ b/lib/ExtUtils/t/Constant.t
@@ -1,6 +1,6 @@
 #!/usr/bin/perl -w
 
-print "1..51\n";
+print "1..52\n";
 
 BEGIN {
     if( $ENV{PERL_CORE} ) {
@@ -14,10 +14,18 @@ use strict;
 use ExtUtils::MakeMaker;
 use ExtUtils::Constant qw (constant_types C_constant XS_constant autoload);
 use Config;
-use File::Spec::Functions qw(catfile rel2abs);
+use File::Spec;
+
+my $do_utf_tests = $] > 5.006;
+my $better_than_56 = $] > 5.007;
+
 # Because were are going to be changing directory before running Makefile.PL
-my $perl;
-$perl = rel2abs( $^X ) unless $] < 5.006; # Hack. Until 5.00503 has rel2abs
+my $perl = $^X;
+# 5.005 doesn't have new enough File::Spec to have rel2abs. But actually we
+# only need it when $^X isn't absolute, which is going to be 5.8.0 or later
+# (where ExtUtils::Constant is in the core, and tests against the uninstalled
+# perl
+$perl = File::Spec->rel2abs ($perl) unless $] < 5.006;
 # ExtUtils::Constant::C_constant uses $^X inside a comment, and we want to
 # compare output to ensure that it is the same. We were probably run as ./perl
 # whereas we will run the child with the full path in $perl. So make $^X for
@@ -25,7 +33,9 @@ $perl = rel2abs( $^X ) unless $] < 5.006; # Hack. Until 5.00503 has rel2abs
 $^X = $perl;
 
 print "# perl=$perl\n";
-my $runperl = "$perl \"-I../../lib\"";
+
+my $lib = $ENV{PERL_CORE} ? '../../lib' : '../blib/lib';
+my $runperl = "$perl \"-I$lib\"";
 
 $| = 1;
 
@@ -62,11 +72,23 @@ if (ord('A') == 193) {  # EBCDIC platform
 } else { # ASCII platform
     $pound = chr 163; # A pound sign. (Currency)
 }
-my $inf = chr 0x221E;
-# Check that we can distiguish the pathological case of a string, and the
-# utf8 representation of that string.
-my $pound_bytes = my $pound_utf8 = $pound . '1';
-utf8::encode ($pound_bytes);
+
+my ($inf, $pound_bytes, $pound_utf8);
+if ($do_utf_tests) {
+  $inf = chr 0x221E;
+  # Check that we can distiguish the pathological case of a string, and the
+  # utf8 representation of that string.
+  $pound_utf8 = $pound . '1';
+  if ($better_than_56) {
+    $pound_bytes = $pound_utf8;
+    utf8::encode ($pound_bytes);
+  } else {
+    # Must have that "U*" to generate a zero length UTF string that forces
+    # top bit set chars (such as the pound sign) into UTF8, so that the
+    # unpack 'C*' then gets the byte form of the UTF8.
+    $pound_bytes =  pack 'C*', unpack 'C*', $pound_utf8 . pack "U*";
+  }
+}
 
 my @names = ("FIVE", {name=>"OK6", type=>"PV",},
              {name=>"OK7", type=>"PVN",
@@ -99,12 +121,16 @@ my @names_only = map {(ref $_) ? $_->{name} : $_} @names;
 push @names, ({name=>"*/", type=>"PV", value=>'"CLOSE"', macro=>1},
               {name=>"/*", type=>"PV", value=>'"OPEN"', macro=>1},
               {name=>$pound, type=>"PV", value=>'"Sterling"', macro=>1},
-              {name=>$inf, type=>"PV", value=>'"Infinity"', macro=>1},
-              {name=>$pound_utf8, type=>"PV", value=>'"1 Pound"', macro=>1},
-              {name=>$pound_bytes, type=>"PV", value=>'"1 Pound (as bytes)"',
-               macro=>1},
              );
 
+if ($do_utf_tests) {
+  push @names, ({name=>$inf, type=>"PV", value=>'"Infinity"', macro=>1},
+                {name=>$pound_utf8, type=>"PV", value=>'"1 Pound"', macro=>1},
+                {name=>$pound_bytes, type=>"PV", value=>'"1 Pound (as bytes)"',
+                 macro=>1},
+               );
+}
+
 =pod
 
 The above set of names seems to produce a suitably bad set of compile
@@ -125,6 +151,8 @@ ok 3
 
 =cut
 
+# Grr `
+
 my $types = {};
 my $constant_types = constant_types(); # macro defs
 my $C_constant = join "\n",
@@ -132,7 +160,7 @@ my $C_constant = join "\n",
 my $XS_constant = XS_constant ($package, $types); # XS for ExtTest::constant
 
 ################ Header
-my $header = catfile($dir, "test.h");
+my $header = File::Spec->catdir($dir, "test.h");
 push @files, "test.h";
 open FH, ">$header" or die "open >$header: $!\n";
 print FH <<"EOT";
@@ -155,7 +183,7 @@ while (my ($point, $bearing) = each %compass) {
 close FH or die "close $header: $!\n";
 
 ################ XS
-my $xs = catfile($dir, "$package.xs");
+my $xs = File::Spec->catdir($dir, "$package.xs");
 push @files, "$package.xs";
 open FH, ">$xs" or die "open >$xs: $!\n";
 
@@ -174,7 +202,7 @@ print FH $XS_constant;
 close FH or die "close $xs: $!\n";
 
 ################ PM
-my $pm = catfile($dir, "$package.pm");
+my $pm = File::Spec->catdir($dir, "$package.pm");
 push @files, "$package.pm";
 open FH, ">$pm" or die "open >$pm: $!\n";
 print FH "package $package;\n";
@@ -206,15 +234,16 @@ print FH "bootstrap $package \$VERSION;\n1;\n__END__\n";
 close FH or die "close $pm: $!\n";
 
 ################ test.pl
-my $testpl = catfile($dir, "test.pl");
+my $testpl = File::Spec->catdir($dir, "test.pl");
 push @files, "test.pl";
 open FH, ">$testpl" or die "open >$testpl: $!\n";
 
 print FH "use strict;\n";
-print FH "use $package qw(@names_only);\n";
-print FH <<"EOT";
+print FH "use $package qw(@names_only);\n\n";
+
+print FH "use utf8\n\n" if $do_utf_tests;
 
-use utf8;
+print FH <<"EOT";
 
 print "1..1\n";
 if (open OUTPUT, ">$output") {
@@ -228,6 +257,8 @@ EOT
 
 print FH << 'EOT';
 
+my $better_than_56 = $] > 5.007;
+
 # What follows goes to the temporary file.
 # IV
 my $five = FIVE;
@@ -403,25 +434,26 @@ if ($open eq '/*') {
 }
 EOT
 
-# Do this in 7 bit in case someone is testing with some settings that cause
-# 8 bit files incapable of storing this character.
-my @values
- = map {"'" . join (",", unpack "U*", $_) . "'"}
- ($pound, $inf, $pound_bytes, $pound_utf8);
-# Values is a list of strings, such as ('194,163,49', '163,49')
+if ($do_utf_tests) {
+  # Do this in 7 bit in case someone is testing with some settings that cause
+  # 8 bit files incapable of storing this character.
+  my @values
+    = map {"'" . join (",", unpack "U*", $_ . pack "U*") . "'"}
+      ($pound, $inf, $pound_bytes, $pound_utf8);
+  # Values is a list of strings, such as ('194,163,49', '163,49')
 
-print FH <<'EOT';
+  print FH <<'EOT';
 
-# I can see that this child test program might be about to use parts of
-# Test::Builder
+  # I can see that this child test program might be about to use parts of
+  # Test::Builder
 
-my $test = 23;
-my ($pound, $inf, $pound_bytes, $pound_utf8) = map {eval "pack 'U*', $_"}
+  my $test = 23;
+  my ($pound, $inf, $pound_bytes, $pound_utf8) = map {eval "pack 'U*', $_"}
 EOT
 
-print FH join ",", @values;
+  print FH join ",", @values;
 
-print FH << 'EOT';
+  print FH << 'EOT';
 ;
 
 foreach (["perl", "rules", "rules"],
@@ -437,12 +469,19 @@ foreach (["perl", "rules", "rules"],
   (my $name = $string) =~ s/([^ -~])/sprintf '\x{%X}', ord $1/ges;
   print "# \"$name\" => \'$expect\'\n";
   # Try to force this to be bytes if possible.
-  utf8::downgrade ($string, 1);
+  if ($better_than_56) {
+    utf8::downgrade ($string, 1);
+  } else {
+    if ($string =~ tr/0-\377// == length $string) {
+      # No chars outside range 0-255
+      $string = pack 'C*', unpack 'U*', ($string . pack 'U*');
+    }
+  }
 EOT
 
-print FH  "my (\$error, \$got) = ${package}::constant (\$string);\n";
+  print FH  "my (\$error, \$got) = ${package}::constant (\$string);\n";
 
-print FH <<'EOT';
+  print FH <<'EOT';
   if ($error or $got ne $expect) {
     print "not ok $test # error '$error', got '$got'\n";
   } else {
@@ -450,12 +489,16 @@ print FH <<'EOT';
   }
   $test++;
   print "# Now upgrade '$name' to utf8\n";
-  utf8::upgrade ($string);
+  if ($better_than_56) {
+    utf8::upgrade ($string);
+  } else {
+    $string = pack ('U*') . $string;
+  }
 EOT
 
-print FH  "my (\$error, \$got) = ${package}::constant (\$string);\n";
+  print FH  "my (\$error, \$got) = ${package}::constant (\$string);\n";
 
-print FH <<'EOT';
+  print FH <<'EOT';
   if ($error or $got ne $expect) {
     print "not ok $test # error '$error', got '$got'\n";
   } else {
@@ -465,12 +508,16 @@ print FH <<'EOT';
   if (defined $expect_bytes) {
     print "# And now with the utf8 byte sequence for name\n";
     # Try the encoded bytes.
-    utf8::encode ($string);
+    if ($better_than_56) {
+      utf8::encode ($string);
+    } else {
+      $string = pack 'C*', unpack 'C*', $string . pack "U*";
+    }
 EOT
 
-print FH "my (\$error, \$got) = ${package}::constant (\$string);\n";
+    print FH "my (\$error, \$got) = ${package}::constant (\$string);\n";
 
-print FH <<'EOT';
+    print FH <<'EOT';
     if (ref $expect_bytes) {
       # Error expected.
       if ($error) {
@@ -487,6 +534,12 @@ print FH <<'EOT';
   }
 }
 EOT
+} else {
+  # Don't utf tests;
+  print FH <<'EOT';
+print "ok $_ # Skipped on non Unicode perl\n" foreach 23..43;
+EOT
+}
 
 close FH or die "close $testpl: $!\n";
 
@@ -497,7 +550,7 @@ my $test = 44;
 ################ Makefile.PL
 # We really need a Makefile.PL because make test for a no dynamic linking perl
 # will run Makefile.PL again as part of the "make perl" target.
-my $makefilePL = catfile($dir, "Makefile.PL");
+my $makefilePL = File::Spec->catdir($dir, "Makefile.PL");
 push @files, "Makefile.PL";
 open FH, ">$makefilePL" or die "open >$makefilePL: $!\n";
 print FH <<"EOT";
@@ -516,7 +569,7 @@ close FH or die "close $makefilePL: $!\n";
 
 ################ MANIFEST
 # We really need a MANIFEST because make distclean checks it.
-my $manifest = catfile($dir, "MANIFEST");
+my $manifest = File::Spec->catdir($dir, "MANIFEST");
 push @files, "MANIFEST";
 open FH, ">$manifest" or die "open >$manifest: $!\n";
 print FH "$_\n" foreach @files;
@@ -525,7 +578,8 @@ close FH or die "close $manifest: $!\n";
 chdir $dir or die $!; push @INC,  '../../lib';
 END {chdir ".." or warn $!};
 
-my @perlout = `$runperl Makefile.PL PERL_CORE=1`;
+my $core = $ENV{PERL_CORE} ? ' PERL_CORE=1' : '';
+my @perlout = `$runperl Makefile.PL $core`;
 if ($?) {
   print "not ok 1 # $runperl Makefile.PL failed: $?\n";
   print "# $_" foreach @perlout;
@@ -701,3 +755,8 @@ unless ($keep_files) {
 }
 
 check_for_bonus_files ('.', '.', '..');
+
+# This was causing an assertion failure (a C<confess>ion)
+C_constant ($package, undef, undef, undef, undef, undef, chr 255);
+
+print "ok $test\n"; $test++;