Add support for Bison versions up to 3.7

This requires copying the `YY_CAST` and `YY_ATTRIBUTE_UNUSED` macros
from the generated code, and extracting the `yysymbol_kind_t` enum if
it's defined.

We must also handle token type names with escaped double-quotes in
them, since it now names the `YYEOF` and `YYUNDEF` tokens `"end of
file"` and `"invalid token"` instead of `$end` and `$undefined`,
respectively.

6 files changed, 39 insertions, 9 deletions

diff --git a/perly.act b/perly.act
index e2c91d4851..465ac4f712 100644
--- a/perly.act
+++ b/perly.act
@@ -2130,5 +2130,5 @@ case 2:
 
 /* Generated from:
  * f83d884147747f2d8f5a62eebc4ccd07d71b6b34e5ba1a8d7559526ad864dc97 perly.y
- * 1b401b34e1842dd8814919ea427f1b13abc479699495e6e88f8fe4c7ab2f838f regen_perly.pl
+ * 40aee1e96522879ab043fac10286df9e97b2db0197ad63dc6946ae7a61444afb regen_perly.pl
  * ex: set ro: */
diff --git a/perly.c b/perly.c
index 2b152c7d87..091371937d 100644
--- a/perly.c
+++ b/perly.c
@@ -53,6 +53,24 @@ typedef signed char yysigned_char;
 # define YY_NULLPTR NULL
 #endif
 
+#ifndef YY_CAST
+# ifdef __cplusplus
+#  define YY_CAST(Type, Val) static_cast<Type> (Val)
+#  define YY_REINTERPRET_CAST(Type, Val) reinterpret_cast<Type> (Val)
+# else
+#  define YY_CAST(Type, Val) ((Type) (Val))
+#  define YY_REINTERPRET_CAST(Type, Val) ((Type) (Val))
+# endif
+#endif
+
+#ifndef YY_ATTRIBUTE_UNUSED
+# if defined __GNUC__ && 2 < __GNUC__ + (7 <= __GNUC_MINOR__)
+#  define YY_ATTRIBUTE_UNUSED __attribute__ ((__unused__))
+# else
+#  define YY_ATTRIBUTE_UNUSED
+# endif
+#endif
+
 /* contains all the parser state tables; auto-generated from perly.y */
 #include "perly.tab"
 
diff --git a/perly.h b/perly.h
index 5a3cef7cf5..c86560aada 100644
--- a/perly.h
+++ b/perly.h
@@ -196,5 +196,5 @@ int yyparse (void);
 
 /* Generated from:
  * f83d884147747f2d8f5a62eebc4ccd07d71b6b34e5ba1a8d7559526ad864dc97 perly.y
- * 1b401b34e1842dd8814919ea427f1b13abc479699495e6e88f8fe4c7ab2f838f regen_perly.pl
+ * 40aee1e96522879ab043fac10286df9e97b2db0197ad63dc6946ae7a61444afb regen_perly.pl
  * ex: set ro: */
diff --git a/perly.tab b/perly.tab
index f4efa7a739..d8111ba086 100644
--- a/perly.tab
+++ b/perly.tab
@@ -1190,5 +1190,5 @@ static const toketypes yy_type_tab[] =
 
 /* Generated from:
  * f83d884147747f2d8f5a62eebc4ccd07d71b6b34e5ba1a8d7559526ad864dc97 perly.y
- * 1b401b34e1842dd8814919ea427f1b13abc479699495e6e88f8fe4c7ab2f838f regen_perly.pl
+ * 40aee1e96522879ab043fac10286df9e97b2db0197ad63dc6946ae7a61444afb regen_perly.pl
  * ex: set ro: */
diff --git a/pod/perldelta.pod b/pod/perldelta.pod
index 782457c697..d0c6acc444 100644
--- a/pod/perldelta.pod
+++ b/pod/perldelta.pod
@@ -254,7 +254,7 @@ L</Platform Support> section, instead.
 
 =item *
 
-The minimum supported Bison version is now 2.4.
+The minimum supported Bison version is now 2.4, and the maxiumm is 3.7.
 
 =back
 
diff --git a/regen_perly.pl b/regen_perly.pl
index c8df5a3d15..7c2dc8c12c 100644
--- a/regen_perly.pl
+++ b/regen_perly.pl
@@ -76,10 +76,10 @@ EOF
 
 # Don't change this to add new bison versions without testing that the generated
 # files actually work :-) Win32 in particular may not like them. :-(
-unless ($version =~ /\b(2\.[4567]|3\.[0-4])\b/) { die <<EOF; }
+unless ($version =~ /\b(2\.[4567]|3\.[0-7])\b/) { die <<EOF; }
 
 You have the wrong version of bison in your path; currently versions
-2.4-2.7 or 3.0-3.4 are known to work.  Try installing
+2.4-2.7 or 3.0-3.7 are known to work.  Try installing
     http://ftp.gnu.org/gnu/bison/bison-3.3.tar.gz
 or similar.  Your bison identifies itself as:
 
@@ -187,13 +187,25 @@ foreach ($act_fh, $tab_fh, $h_fh) {
 exit 0;
 
 
-# extract the tables and actions from the generated .c file
+# extract the symbol kinds, tables and actions from the generated .c file
 
 sub extract {
     my $clines = shift;
     my $tablines;
     my $actlines;
 
+    # extract the symbol kind table if it exists
+    $clines =~ m@
+        (?:
+            ^/\* \s* Symbol \s+ kind\. \s* \*/\n
+        )?
+        enum \s+ yysymbol_kind_t \s* \{
+        .*?
+        \} \s* ;\n
+        typedef \s+ enum \s+ \w+ \s+ \w+ ; \n+
+    @xms
+        and $tablines .= $&;
+
     my $last_table = $version >= 3 ? 'yyr2' : 'yystos';
     $clines =~ m@
 	(?:
@@ -206,7 +218,7 @@ sub extract {
 	}\s*;				# end of last table
     @xms
 	or die "Can't extract tables from $tmpc_file\n";
-    $tablines = $&;
+    $tablines .= $&;
 
 
     # extract all the cases in the big action switch statement
@@ -307,7 +319,7 @@ sub make_type_tab {
 	    /xsm
 	or die "Can't extract yytname[] from table string\n";
     my $fields = $1;
-    $fields =~ s{"([^"]+)"}
+    $fields =~ s{"((?:[^"\\]|\\.)+)"}
 		{ "toketype_" .
 		    (defined $tokens{$1} ? $tokens{$1} : $default_token)
 		}ge;