From 9bb220d50391eda55bc45c29f16cd69859bc95a8 Mon Sep 17 00:00:00 2001 From: David Beazley Date: Sat, 25 Apr 2015 09:21:40 -0500 Subject: IOError handling improvements. More tests. Doc update --- CHANGES | 8 +++ doc/ply.html | 124 +++++++++++++++++++++++++++++++----- ply/lex.py | 5 +- ply/yacc.py | 20 ++++-- test/pkg_test4/__init__.py | 9 +++ test/pkg_test4/parsing/__init__.py | 0 test/pkg_test4/parsing/calclex.py | 47 ++++++++++++++ test/pkg_test4/parsing/calcparse.py | 66 +++++++++++++++++++ test/testyacc.py | 8 +++ 9 files changed, 265 insertions(+), 22 deletions(-) create mode 100644 test/pkg_test4/__init__.py create mode 100644 test/pkg_test4/parsing/__init__.py create mode 100644 test/pkg_test4/parsing/calclex.py create mode 100644 test/pkg_test4/parsing/calcparse.py diff --git a/CHANGES b/CHANGES index 3dcdef9..fdda63f 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,13 @@ Version 3.6 --------------------- +04/25/15: beazley + If PLY is unable to create the 'parser.out' or 'parsetab.py' files due + to permission issues, it now just issues a warning message and + continues to operate. This could happen if a module using PLY + is installed in a funny way where tables have to be regenerated, but + for whatever reason, the user doesn't have write permission on + the directory where PLY wants to put them. + 04/24/15: beazley Fixed some issues related to use of packages and table file modules. Just to emphasize, PLY now generates its special diff --git a/doc/ply.html b/doc/ply.html index 631d76f..808ed1d 100644 --- a/doc/ply.html +++ b/doc/ply.html @@ -74,6 +74,7 @@ dave@dabeaz.com
  • Debugging the lex() and yacc() commands
  • Run-time Debugging +
  • Packaging Advice
  • Where to go from here? @@ -82,6 +83,7 @@ dave@dabeaz.com
    +

    1. Preface and Requirements

    @@ -778,7 +780,7 @@ approach substantially improves the startup time of the lexer and it works in Python's optimized mode.

    -To change the name of the lexer-generated file, use the lextab keyword argument. For example: +To change the name of the lexer-generated module, use the lextab keyword argument. For example:

    @@ -787,15 +789,6 @@ lexer = lex.lex(optimize=1,lextab="footab")
    -

    To change the output directory of the file, use the outputdir keyword argument. For example: -

    - -
    -
    -lexer = lex.lex(optimize=1, outputdir="/some/directory")
-
    -
    - When running in optimized mode, it is important to note that lex disables most error checking. Thus, this is really only recommended if you're sure everything is working correctly and you're ready to start releasing production code. @@ -1748,12 +1741,17 @@ executions, yacc will reload the table from parsetab.py unless it has detected a change in the underlying grammar (in which case the tables and parsetab.py file are regenerated). Both of these files are written to the same directory -as the module in which the parser is specified. The output directory -can be changed by giving an outputdir keyword argument to yacc(). -The name of the parsetab module can also be changed using the -tabmodule keyword argument to yacc(). +as the module in which the parser is specified. +The name of the parsetab module can be changed using the +tabmodule keyword argument to yacc(). For example:

    +
    +
    +parser = yacc.yacc(tabmodule='fooparsetab')
+
    +
    +

    If any errors are detected in your grammar specification, yacc.py will produce diagnostic messages and possibly raise an exception. Some of the errors that can be detected include: @@ -3104,6 +3102,13 @@ parser = yacc.yacc(tabmodule="foo") +

    +Normally, the parsetab.py file is placed into the same directory as +the module where the parser is defined. If you want it to go somewhere else, you can +given an absolute package name for tabmodule instead. In that case, the +tables will be written there. +

    +

  • To change the directory in which the parsetab.py file (and other output files) are written, use:
    @@ -3112,6 +3117,12 @@ parser = yacc.yacc(tabmodule="foo",outputdir="somedirectory")
    +

    +Note: Be aware that unless the directory specified is also on Python's path (sys.path), subsequent +imports of the table file will fail. As a general rule, it's better to specify a destination using the +tabmodule argument instead of directly specifying a directory using the outputdir argument. +

    +

  • To prevent yacc from generating any kind of parser table file, use:
    @@ -3348,7 +3359,90 @@ For very complicated problems, you should pass in a logging object that redirects to a file where you can more easily inspect the output after execution. -

    10. Where to go from here?

    +

    10. Packaging Advice

    + + +

    +If you are distributing a package that makes use of PLY, you should +spend a few moments thinking about how you want to handle the files +that are automatically generated. For example, the parsetab.py +file generated by the yacc() function.

    + +

    +Starting in PLY-3.6, the table files are created in the same directory +as the file where a parser is defined. This means that the +parsetab.py file will live side-by-side with your parser +specification. In terms of packaging, this is probably the easiest and +most sane approach to manage. You don't need to give yacc() +any extra arguments and it should just "work."

    + +

    +One concern is the management of the parsetab.py file itself. +For example, should you have this file checked into version control (e.g., GitHub), +should it be included in a package distribution as a normal file, or should you +just let PLY generate it automatically for the user when they install your package? +

    + +

    +As of PLY-3.6, the parsetab.py file should be compatible across all versions +of Python including Python 2 and 3. Thus, a table file generated in Python 2 should +work fine if it's used on Python 3. Because of this, it should be relatively harmless +to distribute the parsetab.py file yourself if you need to. However, be aware +that older/newer versions of PLY may try to regenerate the file if there are future +enhancements or changes to its format. +

    + +

    +To make the generation of table files easier for the purposes of installation, you might +way to make your parser files executable using the -m option or similar. For +example: +

    + +
    +
    +# calc.py
+...
+...
+def make_parser():
+    parser = yacc.yacc()
+    return parser
+
+if __name__ == '__main__':
+    make_parser()
+
    +
    + +

    +You can then use a command such as python -m calc.py to generate the tables. Alternatively, +a setup.py script, can import the module and use make_parser() to create the +parsing tables. +

    + +

    +If you're willing to sacrifice a little startup time, you can also instruct PLY to never write the +tables using yacc.yacc(write_tables=False, debug=False). In this mode, PLY will regenerate +the parsing tables from scratch each time. For a small grammar, you probably won't notice. For a +large grammar, you should probably reconsider--the parsing tables are meant to dramatically speed up this process. +

    + +

    +During operation, is is normal for PLY to produce diagnostic error +messages (usually printed to standard error). These are generated +entirely using the logging module. If you want to redirect +these messages or silence them, you can provide your own logging +object to yacc(). For example: +

    + +
    +
    +import logging
+log = logging.getLogger('ply')
+...
+parser = yacc.yacc(errorlog=log)
+
    +
    + +

    11. Where to go from here?

    The examples directory of the PLY distribution contains several simple examples. Please consult a diff --git a/ply/lex.py b/ply/lex.py index d93c91d..47cfa44 100644 --- a/ply/lex.py +++ b/ply/lex.py @@ -1029,7 +1029,10 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab', # If in optimize mode, we write the lextab if lextab and optimize: - lexobj.writetab(baselextab, outputdir) + try: + lexobj.writetab(baselextab, outputdir) + except IOError as e: + errorlog.warning("Couldn't write lextab module %r. %s" % (lextab, e)) return lexobj diff --git a/ply/yacc.py b/ply/yacc.py index fb2a9ef..5527fde 100644 --- a/ply/yacc.py +++ b/ply/yacc.py @@ -2801,9 +2801,7 @@ del _lr_goto_items f.close() except IOError as e: - sys.stderr.write('Unable to create %r\n' % filename) - sys.stderr.write(str(e)+'\n') - return + raise # ----------------------------------------------------------------------------- @@ -3259,7 +3257,11 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star if debuglog is None: if debug: - debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w')) + try: + debuglog = PlyLogger(open(os.path.join(outputdir, debugfile), 'w')) + except IOError as e: + errorlog.warning("Couldn't open %r. %s" % (debugfile, e)) + debuglog = NullLogger() else: debuglog = NullLogger() @@ -3429,11 +3431,17 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star # Write the table file if requested if write_tables: - lr.write_table(basetabmodule, outputdir, signature) + try: + lr.write_table(basetabmodule, outputdir, signature) + except IOError as e: + errorlog.warning("Couldn't create %r. %s" % (tabmodule, e)) # Write a pickled version of the tables if picklefile: - lr.pickle_table(picklefile, signature) + try: + lr.pickle_table(picklefile, signature) + except IOError as e: + errorlog.warning("Couldn't create %r. %s" % (picklefile, e)) # Build the parser lr.bind_callables(pinfo.pdict) diff --git a/test/pkg_test4/__init__.py b/test/pkg_test4/__init__.py new file mode 100644 index 0000000..0e19558 --- /dev/null +++ b/test/pkg_test4/__init__.py @@ -0,0 +1,9 @@ +# Tests proper handling of lextab and parsetab files in package structures + +# Here for testing purposes +import sys +if '..' not in sys.path: + sys.path.insert(0, '..') + +from .parsing.calcparse import parser + diff --git a/test/pkg_test4/parsing/__init__.py b/test/pkg_test4/parsing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/pkg_test4/parsing/calclex.py b/test/pkg_test4/parsing/calclex.py new file mode 100644 index 0000000..b3c1a4d --- /dev/null +++ b/test/pkg_test4/parsing/calclex.py @@ -0,0 +1,47 @@ +# ----------------------------------------------------------------------------- +# calclex.py +# ----------------------------------------------------------------------------- + +import ply.lex as lex + +tokens = ( + 'NAME','NUMBER', + 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', + 'LPAREN','RPAREN', + ) + +# Tokens + +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_EQUALS = r'=' +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + +def t_NUMBER(t): + r'\d+' + try: + t.value = int(t.value) + except ValueError: + print("Integer value too large %s" % t.value) + t.value = 0 + return t + +t_ignore = " \t" + +def t_newline(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +def t_error(t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + +# Build the lexer +lexer = lex.lex(optimize=True) + + + diff --git a/test/pkg_test4/parsing/calcparse.py b/test/pkg_test4/parsing/calcparse.py new file mode 100644 index 0000000..c058e9f --- /dev/null +++ b/test/pkg_test4/parsing/calcparse.py @@ -0,0 +1,66 @@ +# ----------------------------------------------------------------------------- +# yacc_simple.py +# +# A simple, properly specifier grammar +# ----------------------------------------------------------------------------- + +from .calclex import tokens +from ply import yacc + +# Parsing rules +precedence = ( + ('left','PLUS','MINUS'), + ('left','TIMES','DIVIDE'), + ('right','UMINUS'), + ) + +# dictionary of names +names = { } + +def p_statement_assign(t): + 'statement : NAME EQUALS expression' + names[t[1]] = t[3] + +def p_statement_expr(t): + 'statement : expression' + t[0] = t[1] + +def p_expression_binop(t): + '''expression : expression PLUS expression + | expression MINUS expression + | expression TIMES expression + | expression DIVIDE expression''' + if t[2] == '+' : t[0] = t[1] + t[3] + elif t[2] == '-': t[0] = t[1] - t[3] + elif t[2] == '*': t[0] = t[1] * t[3] + elif t[2] == '/': t[0] = t[1] / t[3] + +def p_expression_uminus(t): + 'expression : MINUS expression %prec UMINUS' + t[0] = -t[2] + +def p_expression_group(t): + 'expression : LPAREN expression RPAREN' + t[0] = t[2] + +def p_expression_number(t): + 'expression : NUMBER' + t[0] = t[1] + +def p_expression_name(t): + 'expression : NAME' + try: + t[0] = names[t[1]] + except LookupError: + print("Undefined name '%s'" % t[1]) + t[0] = 0 + +def p_error(t): + print("Syntax error at '%s'" % t.value) + +parser = yacc.yacc() + + + + + diff --git a/test/testyacc.py b/test/testyacc.py index e5223ae..90b0ebb 100644 --- a/test/testyacc.py +++ b/test/testyacc.py @@ -425,4 +425,12 @@ class YaccErrorWarningTests(unittest.TestCase): r = parser.parse('3+4+5') self.assertEqual(r, 12) + def test_pkg_test4(self): + from pkg_test4 import parser + self.assertFalse(os.path.exists('pkg_test4/parsing/parsetab.py')) + self.assertFalse(os.path.exists('pkg_test4/parsing/lextab.py')) + self.assertFalse(os.path.exists('pkg_test4/parsing/parser.out')) + r = parser.parse('3+4+5') + self.assertEqual(r, 12) + unittest.main() -- cgit v1.2.1