yysyntax_error: fix for consistent error with lookahead.

* NEWS (2.5): Document.
* data/yacc.c (yysyntax_error): In a verbose syntax error
message while in a consistent state with a default action (which
must be an error action given that yysyntax_error is being
invoked), continue to drop the expected token list, but don't
drop the unexpected token unless there actually is no lookahead.
Moreover, handle that internally instead of returning 1 to tell
the caller to do it.  With that meaning of 1 gone, renumber
return codes more usefully.
(yyparse, yypush_parse): Update yysyntax_error usage.  Most
importantly, set yytoken to YYEMPTY when there's no lookahead.
* data/glr.c (yyreportSyntaxError): As in yacc.c, don't drop the
unexpected token unless there actually is no lookahead.
* data/lalr1.cc (yy::parser::parse): If there's no lookahead,
pass yyempty_ not yyla.type to yysyntax_error_.
(yy::parser::yysyntax_error_): Again, don't drop the unexpected
token unless there actually is no lookahead.
* data/lalr1.java (YYParser::parse): If there's no lookahead,
set yytoken to yyempty_ before invoking yysyntax_error.
(YYParser::yysyntax_error): Again, don't drop the unexpected
token unless there actually is no lookahead.
* tests/conflicts.at (parse.error=verbose and consistent
errors): Extend test group to further reveal how the previous
use of the simple "syntax error" message was too general.  Test
yacc.c, glr.c, lalr1.cc, and lalr1.java.  No longer an expected
failure.
* tests/java.at (AT_JAVA_COMPILE, AT_JAVA_PARSER_CHECK): Move
to...
* tests/local.at: ... here.
(_AT_BISON_OPTION_PUSHDEFS): Push AT_SKEL_JAVA_IF definition.
(AT_BISON_OPTION_POPDEFS): Pop it.
(AT_FULL_COMPILE): Extend to handle Java.

3 files changed, 211 insertions, 55 deletions

diff --git a/tests/conflicts.at b/tests/conflicts.at
index 72f56963..655a666c 100644
--- a/tests/conflicts.at
+++ b/tests/conflicts.at
@@ -147,15 +147,30 @@ AT_SETUP([[parse.error=verbose and consistent errors]])
 
 m4_pushdef([AT_CONSISTENT_ERRORS_CHECK], [
 
-AT_DATA_GRAMMAR([input.y],
-[[%code {
+AT_BISON_OPTION_PUSHDEFS([$1])
+
+m4_pushdef([AT_YYLEX_PROTOTYPE],
+[AT_SKEL_CC_IF([[int yylex (yy::parser::semantic_type *lvalp)]],
+               [[int yylex (YYSTYPE *lvalp)]])])
+
+AT_SKEL_JAVA_IF([AT_DATA], [AT_DATA_GRAMMAR])([input.y],
+[AT_SKEL_JAVA_IF([[
+
+%code imports {
+  import java.io.IOException;
+}]], [[
+
+%code {]AT_SKEL_CC_IF([[
+  #include <string>]], [[
   #include <assert.h>
   #include <stdio.h>
-  int yylex (void);
-  void yyerror (char const *);
+  void yyerror (char const *msg);]])[
+  ]AT_YYLEX_PROTOTYPE[;
   #define USE(Var)
 }
 
+]AT_SKEL_CC_IF([[%defines]], [[%define api.pure]])])[
+
 ]$1[
 
 %define parse.error verbose
@@ -164,63 +179,193 @@ AT_DATA_GRAMMAR([input.y],
 
 ]$2[
 
-%%
+]AT_SKEL_JAVA_IF([[%code lexer {]], [[%%]])[
 
-int
-yylex (void)
+/*--------.
+| yylex.  |
+`--------*/]AT_SKEL_JAVA_IF([[
+
+public String input = "]$3[";
+public int index = 0;
+public int yylex ()
+{
+  if (index < input.length ())
+    return input.charAt (index++);
+  else
+    return 0;
+}
+public Object getLVal ()
+{
+  return new Integer(1);
+}]], [[
+
+]AT_YYLEX_PROTOTYPE[
 {
   static char const *input = "]$3[";
-  yylval = 1;
+  *lvalp = 1;
   return *input++;
+}]])[
+
+/*----------.
+| yyerror.  |
+`----------*/]AT_SKEL_JAVA_IF([[
+
+public void yyerror (String msg)
+{
+  System.err.println (msg);
 }
 
+};
+
+%%]], [AT_SKEL_CC_IF([[
+
+void
+yy::parser::error (std::string const &msg)
+{
+  std::cerr << msg << std::endl;
+}]], [[
+
 void
 yyerror (char const *msg)
 {
   fprintf (stderr, "%s\n", msg);
-}
+}]])])[
+
+/*-------.
+| main.  |
+`-------*/]AT_SKEL_JAVA_IF([[
+
+class input
+{
+  public static void main (String args[]) throws IOException
+  {
+    YYParser p = new YYParser ();
+    p.parse ();
+  }
+}]], [AT_SKEL_CC_IF([[
+
+int
+main (void)
+{
+  yy::parser parser;
+  return parser.parse ();
+}]], [[
 
 int
 main (void)
 {
   return yyparse ();
-}
+}]])])[
 ]])
-AT_BISON_CHECK([[-o input.c input.y]])
-AT_COMPILE([[input]])
+
+AT_FULL_COMPILE([[input]])
 
 m4_pushdef([AT_EXPECTING], [m4_if($5, [ab], [[, expecting 'a' or 'b']],
                                   $5, [a],  [[, expecting 'a']],
                                   $5, [b],  [[, expecting 'b']])])
 
-AT_PARSER_CHECK([[./input]], [[1]], [[]],
+AT_SKEL_JAVA_IF([AT_JAVA_PARSER_CHECK([[input]], [[0]]],
+                [AT_PARSER_CHECK([[./input]], [[1]]]),
+[[]],
 [[syntax error, unexpected ]$4[]AT_EXPECTING[
 ]])
 
 m4_popdef([AT_EXPECTING])
+m4_popdef([AT_YYLEX_PROTOTYPE])
+AT_BISON_OPTION_POPDEFS
 
 ])
 
+m4_pushdef([AT_PREVIOUS_STATE_GRAMMAR],
+[[%nonassoc 'a';
+
+start: consistent-error-on-a-a 'a' ;
+
+consistent-error-on-a-a:
+    'a' default-reduction
+  | 'a' default-reduction 'a'
+  | 'a' shift
+  ;
+
+default-reduction: /*empty*/ ;
+shift: 'b' ;
+
+// Provide another context in which all rules are useful so that this
+// test case looks a little more realistic.
+start: 'b' consistent-error-on-a-a 'c' ;
+]])
+
+m4_pushdef([AT_PREVIOUS_STATE_INPUT], [[a]])
+
+# Unfortunately, no expected tokens are reported even though 'b' can be
+# accepted.  Nevertheless, the main point of this test is to make sure
+# that at least the unexpected token is reported.  In a previous version
+# of Bison, it wasn't reported because the error is detected in a
+# consistent state with an error action, and that case always triggered
+# the simple "syntax error" message.
+#
+# The point isn't to test IELR here, but state merging happens to
+# complicate this example.
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[none]])
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr
+                             %glr-parser]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[none]])
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr
+                             %language "c++"]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[none]])
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr
+                             %language "java"]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[end of input]], [[none]])
+
+# Even canonical LR doesn't foresee the error for 'a'!
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr
+                             %define lr.default-reductions consistent]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[ab]])
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type ielr
+                             %define lr.default-reductions accepting]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[ab]])
+AT_CONSISTENT_ERRORS_CHECK([[%define lr.type canonical-lr]],
+                           [AT_PREVIOUS_STATE_GRAMMAR],
+                           [AT_PREVIOUS_STATE_INPUT],
+                           [[$end]], [[ab]])
+
+m4_popdef([AT_PREVIOUS_STATE_GRAMMAR])
+m4_popdef([AT_PREVIOUS_STATE_INPUT])
+
 m4_pushdef([AT_USER_ACTION_GRAMMAR],
 [[%nonassoc 'a';
 
-// If yylval=0 here, then we know that the 'a' destructor is being
-// invoked incorrectly for the 'b' set in the semantic action below.
-// All 'a' tokens are returned by yylex, which sets yylval=1.
+// If $$ = 0 here, then we know that the 'a' destructor is being invoked
+// incorrectly for the 'b' set in the semantic action below.  All 'a'
+// tokens are returned by yylex, which sets $$ = 1.
 %destructor {
   if (!$$)
     fprintf (stderr, "Wrong destructor.\n");
 } 'a';
 
-// The lookahead assigned by the semantic action isn't needed before
-// either error action is encountered.  In a previous version of Bison,
-// this was a problem as it meant yychar was not translated into yytoken
-// before either error action.  The second error action thus invoked a
+// Rather than depend on an inconsistent state to induce reading a
+// lookahead as in the previous grammar, just assign the lookahead in a
+// semantic action.  That lookahead isn't needed before either error
+// action is encountered.  In a previous version of Bison, this was a
+// problem as it meant yychar was not translated into yytoken before
+// either error action.  The second error action thus invoked a
 // destructor that it selected according to the incorrect yytoken.  The
 // first error action would have reported an incorrect unexpected token
-// except that, due to another bug, the unexpected token is not reported
-// at all because the error action is the default action in a consistent
-// state.  That bug still needs to be fixed.
+// except that, due to the bug described in the previous grammar, the
+// unexpected token was not reported at all.
 start: error-reduce consistent-error 'a' { USE ($][3); } ;
 
 error-reduce:
@@ -247,13 +392,16 @@ start: 'b' consistent-error 'b' ;
 ]])
 m4_pushdef([AT_USER_ACTION_INPUT], [[aa]])
 
-# See comments in grammar for why this test doesn't succeed.
-AT_XFAIL_IF([[:]])
-
 AT_CONSISTENT_ERRORS_CHECK([[]],
                            [AT_USER_ACTION_GRAMMAR],
                            [AT_USER_ACTION_INPUT],
                            [['b']], [[none]])
+AT_CONSISTENT_ERRORS_CHECK([[%glr-parser]],
+                           [AT_USER_ACTION_GRAMMAR],
+                           [AT_USER_ACTION_INPUT],
+                           [['b']], [[none]])
+# No C++ or Java test because yychar cannot be manipulated by users.
+
 AT_CONSISTENT_ERRORS_CHECK([[%define lr.default-reductions consistent]],
                            [AT_USER_ACTION_GRAMMAR],
                            [AT_USER_ACTION_INPUT],
diff --git a/tests/java.at b/tests/java.at
index 9ea703fe..dfd42523 100644
--- a/tests/java.at
+++ b/tests/java.at
@@ -219,25 +219,6 @@ m4_define([AT_DATA_JAVA_CALC_Y],
 ])
 
 
-
-# AT_JAVA_COMPILE(SOURCE)
-# -----------------------
-# Compile SOURCES into Java class files.  Skip the test if java or javac is
-# not installed.
-m4_define([AT_JAVA_COMPILE],
-[AT_KEYWORDS(java)
-AT_CHECK([test -n "$CONF_JAVA" || exit 77
-test -n "$CONF_JAVAC" || exit 77])
-AT_CHECK([$SHELL ../../../javacomp.sh $1],
-         0, [ignore], [ignore])])
-
-
-# AT_JAVA_PARSER_CHECK(COMMAND, EXIT-STATUS, EXPOUT, EXPERR, [PRE])
-# -----------------------------------------------------------------
-m4_define([AT_JAVA_PARSER_CHECK],
-[AT_CHECK([$5 $SHELL ../../../javaexec.sh $1], [$2], [$3], [$4])])
-
-
 # _AT_CHECK_JAVA_CALC_ERROR(BISON-OPTIONS, INPUT,
 #                           [VERBOSE-AND-LOCATED-ERROR-MESSAGE])
 # ---------------------------------------------------------
diff --git a/tests/local.at b/tests/local.at
index d3b245c2..bdcbfc49 100644
--- a/tests/local.at
+++ b/tests/local.at
@@ -80,6 +80,8 @@ m4_pushdef([AT_DEFINES_IF],
 [m4_bmatch([$3], [%defines], [$1], [$2])])
 m4_pushdef([AT_SKEL_CC_IF],
 [m4_bmatch([$3], [%language "[Cc]\+\+"\|%skeleton "[a-z0-9]+\.cc"], [$1], [$2])])
+m4_pushdef([AT_SKEL_JAVA_IF],
+[m4_bmatch([$3], [%language "[Jj][Aa][Vv][Aa]"\|%skeleton "[a-z0-9]+\.java"], [$1], [$2])])
 m4_pushdef([AT_GLR_IF],
 [m4_bmatch([$3], [%glr-parser\|%skeleton "glr\.], [$1], [$2])])
 m4_pushdef([AT_LALR1_CC_IF],
@@ -189,6 +191,7 @@ m4_popdef([AT_LEXPARAM_IF])
 m4_popdef([AT_YACC_IF])
 m4_popdef([AT_GLR_IF])
 m4_popdef([AT_SKEL_CC_IF])
+m4_popdef([AT_SKEL_JAVA_IF])
 m4_popdef([AT_GLR_CC_IF])
 m4_popdef([AT_LALR1_CC_IF])
 m4_popdef([AT_DEFINES_IF])
@@ -404,19 +407,38 @@ AT_CHECK([$BISON_CXX_WORKS], 0, ignore, ignore)
 AT_CHECK([$CXX $CXXFLAGS $CPPFLAGS m4_bmatch([$1], [[.]], [], [$LDFLAGS ])-o $1 m4_default([$2], [$1.cc])[]m4_bmatch([$1], [[.]], [], [ $LIBS])],
 	 0, [ignore], [ignore])])
 
+# AT_JAVA_COMPILE(SOURCES)
+# ------------------------
+# Compile SOURCES into Java class files.  Skip the test if java or javac
+# is not installed.
+m4_define([AT_JAVA_COMPILE],
+[AT_KEYWORDS(java)
+AT_CHECK([[test -n "$CONF_JAVA" || exit 77
+           test -n "$CONF_JAVAC" || exit 77]])
+AT_CHECK([[$SHELL ../../../javacomp.sh ]$1],
+         [[0]], [ignore], [ignore])])
 
 # AT_FULL_COMPILE(OUTPUT, [OTHER])
 # --------------------------------
-# Compile OUTPUT.y to OUTPUT.c or OUTPUT.cc, and compile it to OUTPUT.
-# If OTHER is specified, compile OUTPUT-OTHER.c or OUTPUT-OTHER.cc to OUTPUT
-# along with it.
-# Relies on AT_SKEL_CC_IF.
-m4_define([AT_FULL_COMPILE],
-[AT_SKEL_CC_IF(
-  [AT_BISON_CHECK([-o $1.cc $1.y])
-   AT_COMPILE_CXX([$1]m4_ifval($2, [, [$1.cc $1-$2.cc]]))],
-  [AT_BISON_CHECK([-o $1.c $1.y])
-   AT_COMPILE([$1]m4_ifval($2, [, [$1.c $1-$2.c]]))])
+# Compile OUTPUT.y to OUTPUT.c, OUTPUT.cc, or OUTPUT.java, and then
+# compile it to OUTPUT or OUTPUT.class.  If OTHER is specified, compile
+# OUTPUT-OTHER.c, OUTPUT-OTHER.cc, or OUTPUT-OTHER.java to OUTPUT or
+# OUTPUT.java along with it.  Relies on AT_SKEL_CC_IF and
+# AT_SKEL_JAVA_IF.
+m4_define([AT_FULL_COMPILE], [
+  AT_SKEL_JAVA_IF([
+    AT_BISON_CHECK([[-o ]$1[.java ]$1[.y]])
+    AT_JAVA_COMPILE([$1[.java]]m4_ifval($2,
+                                        [[$1[.java ]$1[-]$2[.java]]]))
+  ], [
+    AT_SKEL_CC_IF([
+      AT_BISON_CHECK([[-o ]$1[.cc ]$1[.y]])
+      AT_COMPILE_CXX([$1]m4_ifval($2, [, [$1[.cc ]$1[-]$2[.cc]]]))
+    ], [
+      AT_BISON_CHECK([[-o ]$1[.c ]$1[.y]])
+      AT_COMPILE([$1]m4_ifval($2, [, [$1[.c ]$1[-]$2[.c]]]))
+    ])
+  ])
 ])
 
 
@@ -430,6 +452,11 @@ m4_define([AT_FULL_COMPILE],
 m4_define([AT_PARSER_CHECK],
 [AT_CHECK([$5 $PREPARSER $1], [$2], [$3], [$4])])
 
+# AT_JAVA_PARSER_CHECK(COMMAND, EXIT-STATUS, EXPOUT, EXPERR, [PRE])
+# -----------------------------------------------------------------
+m4_define([AT_JAVA_PARSER_CHECK],
+[AT_CHECK([$5[ $SHELL ../../../javaexec.sh ]$1], [$2], [$3], [$4])])
+
 # AT_TEST_TABLES_AND_PARSE(TITLE, COND-VALUE, TEST-SPEC,
 #                          DECLS, GRAMMAR, INPUT,
 #                          BISON-STDERR, TABLES-OR-LAST-STATE,