From daa1160e3b7e7bd94dc06da9e11ce9e188406fda Mon Sep 17 00:00:00 2001 From: David Beazley Date: Fri, 24 Apr 2015 13:06:34 -0500 Subject: Further refinement of table handling with packages. More unit tests. --- CHANGES | 28 ++++++++++----- README.md | 2 +- doc/ply.html | 2 +- ply/lex.py | 54 ++++++++++++++++------------- ply/yacc.py | 45 ++++++++++++++---------- setup.py | 2 +- test/pkg_test1/__init__.py | 9 +++++ test/pkg_test1/parsing/__init__.py | 0 test/pkg_test1/parsing/calclex.py | 47 +++++++++++++++++++++++++ test/pkg_test1/parsing/calcparse.py | 66 ++++++++++++++++++++++++++++++++++++ test/pkg_test2/__init__.py | 9 +++++ test/pkg_test2/parsing/__init__.py | 0 test/pkg_test2/parsing/calclex.py | 47 +++++++++++++++++++++++++ test/pkg_test2/parsing/calcparse.py | 66 ++++++++++++++++++++++++++++++++++++ test/pkg_test3/__init__.py | 9 +++++ test/pkg_test3/generated/__init__.py | 0 test/pkg_test3/parsing/__init__.py | 0 test/pkg_test3/parsing/calclex.py | 47 +++++++++++++++++++++++++ test/pkg_test3/parsing/calcparse.py | 66 ++++++++++++++++++++++++++++++++++++ test/testyacc.py | 24 +++++++++++++ 20 files changed, 470 insertions(+), 53 deletions(-) create mode 100644 test/pkg_test1/__init__.py create mode 100644 test/pkg_test1/parsing/__init__.py create mode 100644 test/pkg_test1/parsing/calclex.py create mode 100644 test/pkg_test1/parsing/calcparse.py create mode 100644 test/pkg_test2/__init__.py create mode 100644 test/pkg_test2/parsing/__init__.py create mode 100644 test/pkg_test2/parsing/calclex.py create mode 100644 test/pkg_test2/parsing/calcparse.py create mode 100644 test/pkg_test3/__init__.py create mode 100644 test/pkg_test3/generated/__init__.py create mode 100644 test/pkg_test3/parsing/__init__.py create mode 100644 test/pkg_test3/parsing/calclex.py create mode 100644 test/pkg_test3/parsing/calcparse.py diff --git a/CHANGES b/CHANGES index 08e4876..3dcdef9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,18 +1,28 @@ Version 3.6 --------------------- 04/24/15: beazley - Fixed some problems related to use of packages and table file - modules. Just to emphasize, if you use a command such as this: + Fixed some issues related to use of packages and table file + modules. Just to emphasize, PLY now generates its special + files such as 'parsetab.py' and 'lextab.py' in the *SAME* + directory as the source file that uses lex() and yacc(). - # lexer.py - parser = yacc.yacc(tabmodule='foo.bar.parsetab') + If for some reason, you want to change the name of the table + module, use the tabmodule and lextab options: - You need to make sure that the file 'lexer.py' is located at - foo/bar/lexer.py. If this is not the same location, then - you need to also include the outputdir option + lexer = lex.lex(lextab='spamlextab') + parser = yacc.yacc(tabmodule='spamparsetab') - parse = yacc.yacc(tabmodule='foo.bar.parsetab', - outputdir='foo/bar') + If you specify a simple name as shown, the module will still be + created in the same directory as the file invoking lex() or yacc(). + If you want the table files to be placed into a different package, + then give a fully qualified package name. For example: + + lexer = lex.lex(lextab='pkgname.files.lextab') + parser = yacc.yacc(tabmodule='pkgname.files.parsetab') + + For this to work, 'pkgname.files' must already exist as a valid + Python package (i.e., the directories must already exist and be + set up with the proper __init__.py files, etc.). Version 3.5 --------------------- diff --git a/README.md b/README.md index 0d01f7a..c630d81 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -PLY (Python Lex-Yacc) Version 3.5 +PLY (Python Lex-Yacc) Version 3.6 Copyright (C) 2001-2015, David M. Beazley (Dabeaz LLC) diff --git a/doc/ply.html b/doc/ply.html index 95ab9d0..631d76f 100644 --- a/doc/ply.html +++ b/doc/ply.html @@ -12,7 +12,7 @@ dave@dabeaz.com

-PLY Version: 3.5 +PLY Version: 3.6

diff --git a/ply/lex.py b/ply/lex.py index 28f99fe..d93c91d 100644 --- a/ply/lex.py +++ b/ply/lex.py @@ -171,21 +171,10 @@ class Lexer: # ------------------------------------------------------------ # writetab() - Write lexer information to a table file # ------------------------------------------------------------ - def writetab(self, tabfile, outputdir=''): - if isinstance(tabfile, types.ModuleType): - return - parts = tabfile.split('.') - basetabfilename = parts[-1] - if not outputdir and len(parts) > 1: - # If no explicit output directory was given, then set it to the location of the tabfile - packagename = '.'.join(parts[:-1]) - exec('import %s' % packagename) - package = sys.modules[packagename] - outputdir = os.path.dirname(package.__file__) - - filename = os.path.join(outputdir, basetabfilename) + '.py' + def writetab(self, basetabmodule, outputdir=''): + filename = os.path.join(outputdir, basetabmodule) + '.py' with open(filename, 'w') as tf: - tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (tabfile, __version__)) + tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__)) tf.write('_tabversion = %s\n' % repr(__tabversion__)) tf.write('_lextokens = %s\n' % repr(self.lextokens)) tf.write('_lexreflags = %s\n' % repr(self.lexreflags)) @@ -886,19 +875,38 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab', if object: module = object + # Get the module dictionary used for the parser if module: _items = [(k, getattr(module, k)) for k in dir(module)] ldict = dict(_items) - if outputdir is None: - srcfile = getattr(module, '__file__', None) - if srcfile is None: - if hasattr(module, '__module__'): - srcfile = getattr(sys.modules[module.__module__], '__file__', '') - outputdir = os.path.dirname(srcfile) + # If no __file__ attribute is available, try to obtain it from the __module__ instead + if '__file__' not in ldict: + ldict['__file__'] = sys.modules[ldict['__module__']].__file__ else: ldict = get_caller_module_dict(2) - if outputdir is None: - outputdir = os.path.dirname(ldict.get('__file__', '')) + + if outputdir is None: + # If no output directory is set, the location of the output files + # is determined according to the following rules: + # - If lextab specifies a package, files go into that package directory + # - Otherwise, files go in the same directory as the specifying module + if '.' not in lextab: + srcfile = ldict['__file__'] + else: + parts = lextab.split('.') + pkgname = '.'.join(parts[:-1]) + exec('import %s' % pkgname) + srcfile = getattr(sys.modules[pkgname], '__file__', '') + outputdir = os.path.dirname(srcfile) + + # Determine if the module is package of a package or not. + # If so, fix the tabmodule setting so that tables load correctly + pkg = ldict.get('__package__') + if pkg: + if '.' not in lextab: + lextab = pkg + '.' + lextab + + baselextab = lextab.split('.')[-1] # Collect parser information from the dictionary linfo = LexerReflect(ldict, log=errorlog, reflags=reflags) @@ -1021,7 +1029,7 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab', # If in optimize mode, we write the lextab if lextab and optimize: - lexobj.writetab(lextab, outputdir) + lexobj.writetab(baselextab, outputdir) return lexobj diff --git a/ply/yacc.py b/ply/yacc.py index 5bbe419..fb2a9ef 100644 --- a/ply/yacc.py +++ b/ply/yacc.py @@ -2692,15 +2692,7 @@ class LRGeneratedTable(LRTable): # This function writes the LR parsing tables to a file # ----------------------------------------------------------------------------- - def write_table(self, modulename, outputdir='', signature=''): - parts = modulename.split('.') - basemodulename = parts[-1] - if not outputdir and len(parts) > 1: - # If no explicit output directory was given, then set it to the location of the tabfile - packagename = '.'.join(parts[:-1]) - exec('import %s' % packagename) - package = sys.modules[packagename] - outputdir = os.path.dirname(package.__file__) + def write_table(self, basemodulename, outputdir='', signature=''): filename = os.path.join(outputdir, basemodulename) + '.py' try: f = open(filename, 'w') @@ -3203,16 +3195,33 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star if module: _items = [(k, getattr(module, k)) for k in dir(module)] pdict = dict(_items) - if outputdir is None: - srcfile = getattr(module, '__file__', None) - if srcfile is None: - if hasattr(module, '__module__'): - srcfile = getattr(sys.modules[module.__module__], '__file__', '') - outputdir = os.path.dirname(srcfile) + # If no __file__ attribute is available, try to obtain it from the __module__ instead + if '__file__' not in pdict: + pdict['__file__'] = sys.modules[pdict['__module__']].__file__ else: pdict = get_caller_module_dict(2) - if outputdir is None: - outputdir = os.path.dirname(pdict.get('__file__', '')) + + if outputdir is None: + # If no output directory is set, the location of the output files + # is determined according to the following rules: + # - If tabmodule specifies a package, files go into that package directory + # - Otherwise, files go in the same directory as the specifying module + if '.' not in tabmodule: + srcfile = pdict['__file__'] + else: + parts = tabmodule.split('.') + pkgname = '.'.join(parts[:-1]) + exec('import %s' % pkgname) + srcfile = getattr(sys.modules[pkgname], '__file__', '') + outputdir = os.path.dirname(srcfile) + + # Determine if the module is package of a package or not. + # If so, fix the tabmodule setting so that tables load correctly + pkg = pdict.get('__package__') + if pkg and '.' not in tabmodule: + tabmodule = pkg + '.' + tabmodule + + basetabmodule = tabmodule.split('.')[-1] # Set start symbol if it's specified directly using an argument if start is not None: @@ -3420,7 +3429,7 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star # Write the table file if requested if write_tables: - lr.write_table(tabmodule, outputdir, signature) + lr.write_table(basetabmodule, outputdir, signature) # Write a pickled version of the tables if picklefile: diff --git a/setup.py b/setup.py index 1e95122..c6f2b44 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ PLY is extremely easy to use and provides very extensive error checking. It is compatible with both Python 2 and Python 3. """, license="""BSD""", - version = "3.5", + version = "3.6", author = "David Beazley", author_email = "dave@dabeaz.com", maintainer = "David Beazley", diff --git a/test/pkg_test1/__init__.py b/test/pkg_test1/__init__.py new file mode 100644 index 0000000..0e19558 --- /dev/null +++ b/test/pkg_test1/__init__.py @@ -0,0 +1,9 @@ +# Tests proper handling of lextab and parsetab files in package structures + +# Here for testing purposes +import sys +if '..' not in sys.path: + sys.path.insert(0, '..') + +from .parsing.calcparse import parser + diff --git a/test/pkg_test1/parsing/__init__.py b/test/pkg_test1/parsing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/pkg_test1/parsing/calclex.py b/test/pkg_test1/parsing/calclex.py new file mode 100644 index 0000000..b3c1a4d --- /dev/null +++ b/test/pkg_test1/parsing/calclex.py @@ -0,0 +1,47 @@ +# ----------------------------------------------------------------------------- +# calclex.py +# ----------------------------------------------------------------------------- + +import ply.lex as lex + +tokens = ( + 'NAME','NUMBER', + 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', + 'LPAREN','RPAREN', + ) + +# Tokens + +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_EQUALS = r'=' +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + +def t_NUMBER(t): + r'\d+' + try: + t.value = int(t.value) + except ValueError: + print("Integer value too large %s" % t.value) + t.value = 0 + return t + +t_ignore = " \t" + +def t_newline(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +def t_error(t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + +# Build the lexer +lexer = lex.lex(optimize=True) + + + diff --git a/test/pkg_test1/parsing/calcparse.py b/test/pkg_test1/parsing/calcparse.py new file mode 100644 index 0000000..c058e9f --- /dev/null +++ b/test/pkg_test1/parsing/calcparse.py @@ -0,0 +1,66 @@ +# ----------------------------------------------------------------------------- +# yacc_simple.py +# +# A simple, properly specifier grammar +# ----------------------------------------------------------------------------- + +from .calclex import tokens +from ply import yacc + +# Parsing rules +precedence = ( + ('left','PLUS','MINUS'), + ('left','TIMES','DIVIDE'), + ('right','UMINUS'), + ) + +# dictionary of names +names = { } + +def p_statement_assign(t): + 'statement : NAME EQUALS expression' + names[t[1]] = t[3] + +def p_statement_expr(t): + 'statement : expression' + t[0] = t[1] + +def p_expression_binop(t): + '''expression : expression PLUS expression + | expression MINUS expression + | expression TIMES expression + | expression DIVIDE expression''' + if t[2] == '+' : t[0] = t[1] + t[3] + elif t[2] == '-': t[0] = t[1] - t[3] + elif t[2] == '*': t[0] = t[1] * t[3] + elif t[2] == '/': t[0] = t[1] / t[3] + +def p_expression_uminus(t): + 'expression : MINUS expression %prec UMINUS' + t[0] = -t[2] + +def p_expression_group(t): + 'expression : LPAREN expression RPAREN' + t[0] = t[2] + +def p_expression_number(t): + 'expression : NUMBER' + t[0] = t[1] + +def p_expression_name(t): + 'expression : NAME' + try: + t[0] = names[t[1]] + except LookupError: + print("Undefined name '%s'" % t[1]) + t[0] = 0 + +def p_error(t): + print("Syntax error at '%s'" % t.value) + +parser = yacc.yacc() + + + + + diff --git a/test/pkg_test2/__init__.py b/test/pkg_test2/__init__.py new file mode 100644 index 0000000..0e19558 --- /dev/null +++ b/test/pkg_test2/__init__.py @@ -0,0 +1,9 @@ +# Tests proper handling of lextab and parsetab files in package structures + +# Here for testing purposes +import sys +if '..' not in sys.path: + sys.path.insert(0, '..') + +from .parsing.calcparse import parser + diff --git a/test/pkg_test2/parsing/__init__.py b/test/pkg_test2/parsing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/pkg_test2/parsing/calclex.py b/test/pkg_test2/parsing/calclex.py new file mode 100644 index 0000000..789e13f --- /dev/null +++ b/test/pkg_test2/parsing/calclex.py @@ -0,0 +1,47 @@ +# ----------------------------------------------------------------------------- +# calclex.py +# ----------------------------------------------------------------------------- + +import ply.lex as lex + +tokens = ( + 'NAME','NUMBER', + 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', + 'LPAREN','RPAREN', + ) + +# Tokens + +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_EQUALS = r'=' +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + +def t_NUMBER(t): + r'\d+' + try: + t.value = int(t.value) + except ValueError: + print("Integer value too large %s" % t.value) + t.value = 0 + return t + +t_ignore = " \t" + +def t_newline(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +def t_error(t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + +# Build the lexer +lexer = lex.lex(optimize=True, lextab='calclextab') + + + diff --git a/test/pkg_test2/parsing/calcparse.py b/test/pkg_test2/parsing/calcparse.py new file mode 100644 index 0000000..f519338 --- /dev/null +++ b/test/pkg_test2/parsing/calcparse.py @@ -0,0 +1,66 @@ +# ----------------------------------------------------------------------------- +# yacc_simple.py +# +# A simple, properly specifier grammar +# ----------------------------------------------------------------------------- + +from .calclex import tokens +from ply import yacc + +# Parsing rules +precedence = ( + ('left','PLUS','MINUS'), + ('left','TIMES','DIVIDE'), + ('right','UMINUS'), + ) + +# dictionary of names +names = { } + +def p_statement_assign(t): + 'statement : NAME EQUALS expression' + names[t[1]] = t[3] + +def p_statement_expr(t): + 'statement : expression' + t[0] = t[1] + +def p_expression_binop(t): + '''expression : expression PLUS expression + | expression MINUS expression + | expression TIMES expression + | expression DIVIDE expression''' + if t[2] == '+' : t[0] = t[1] + t[3] + elif t[2] == '-': t[0] = t[1] - t[3] + elif t[2] == '*': t[0] = t[1] * t[3] + elif t[2] == '/': t[0] = t[1] / t[3] + +def p_expression_uminus(t): + 'expression : MINUS expression %prec UMINUS' + t[0] = -t[2] + +def p_expression_group(t): + 'expression : LPAREN expression RPAREN' + t[0] = t[2] + +def p_expression_number(t): + 'expression : NUMBER' + t[0] = t[1] + +def p_expression_name(t): + 'expression : NAME' + try: + t[0] = names[t[1]] + except LookupError: + print("Undefined name '%s'" % t[1]) + t[0] = 0 + +def p_error(t): + print("Syntax error at '%s'" % t.value) + +parser = yacc.yacc(tabmodule='calcparsetab') + + + + + diff --git a/test/pkg_test3/__init__.py b/test/pkg_test3/__init__.py new file mode 100644 index 0000000..0e19558 --- /dev/null +++ b/test/pkg_test3/__init__.py @@ -0,0 +1,9 @@ +# Tests proper handling of lextab and parsetab files in package structures + +# Here for testing purposes +import sys +if '..' not in sys.path: + sys.path.insert(0, '..') + +from .parsing.calcparse import parser + diff --git a/test/pkg_test3/generated/__init__.py b/test/pkg_test3/generated/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/pkg_test3/parsing/__init__.py b/test/pkg_test3/parsing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/pkg_test3/parsing/calclex.py b/test/pkg_test3/parsing/calclex.py new file mode 100644 index 0000000..6ca2c4f --- /dev/null +++ b/test/pkg_test3/parsing/calclex.py @@ -0,0 +1,47 @@ +# ----------------------------------------------------------------------------- +# calclex.py +# ----------------------------------------------------------------------------- + +import ply.lex as lex + +tokens = ( + 'NAME','NUMBER', + 'PLUS','MINUS','TIMES','DIVIDE','EQUALS', + 'LPAREN','RPAREN', + ) + +# Tokens + +t_PLUS = r'\+' +t_MINUS = r'-' +t_TIMES = r'\*' +t_DIVIDE = r'/' +t_EQUALS = r'=' +t_LPAREN = r'\(' +t_RPAREN = r'\)' +t_NAME = r'[a-zA-Z_][a-zA-Z0-9_]*' + +def t_NUMBER(t): + r'\d+' + try: + t.value = int(t.value) + except ValueError: + print("Integer value too large %s" % t.value) + t.value = 0 + return t + +t_ignore = " \t" + +def t_newline(t): + r'\n+' + t.lexer.lineno += t.value.count("\n") + +def t_error(t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + +# Build the lexer +lexer = lex.lex(optimize=True, lextab='pkg_test3.generated.lextab') + + + diff --git a/test/pkg_test3/parsing/calcparse.py b/test/pkg_test3/parsing/calcparse.py new file mode 100644 index 0000000..2dcb52b --- /dev/null +++ b/test/pkg_test3/parsing/calcparse.py @@ -0,0 +1,66 @@ +# ----------------------------------------------------------------------------- +# yacc_simple.py +# +# A simple, properly specifier grammar +# ----------------------------------------------------------------------------- + +from .calclex import tokens +from ply import yacc + +# Parsing rules +precedence = ( + ('left','PLUS','MINUS'), + ('left','TIMES','DIVIDE'), + ('right','UMINUS'), + ) + +# dictionary of names +names = { } + +def p_statement_assign(t): + 'statement : NAME EQUALS expression' + names[t[1]] = t[3] + +def p_statement_expr(t): + 'statement : expression' + t[0] = t[1] + +def p_expression_binop(t): + '''expression : expression PLUS expression + | expression MINUS expression + | expression TIMES expression + | expression DIVIDE expression''' + if t[2] == '+' : t[0] = t[1] + t[3] + elif t[2] == '-': t[0] = t[1] - t[3] + elif t[2] == '*': t[0] = t[1] * t[3] + elif t[2] == '/': t[0] = t[1] / t[3] + +def p_expression_uminus(t): + 'expression : MINUS expression %prec UMINUS' + t[0] = -t[2] + +def p_expression_group(t): + 'expression : LPAREN expression RPAREN' + t[0] = t[2] + +def p_expression_number(t): + 'expression : NUMBER' + t[0] = t[1] + +def p_expression_name(t): + 'expression : NAME' + try: + t[0] = names[t[1]] + except LookupError: + print("Undefined name '%s'" % t[1]) + t[0] = 0 + +def p_error(t): + print("Syntax error at '%s'" % t.value) + +parser = yacc.yacc(tabmodule='pkg_test3.generated.parsetab') + + + + + diff --git a/test/testyacc.py b/test/testyacc.py index 96d4b0d..e5223ae 100644 --- a/test/testyacc.py +++ b/test/testyacc.py @@ -401,4 +401,28 @@ class YaccErrorWarningTests(unittest.TestCase): "Precedence rule 'left' defined for unknown symbol '/'\n" )) + def test_pkg_test1(self): + from pkg_test1 import parser + self.assertTrue(os.path.exists('pkg_test1/parsing/parsetab.py')) + self.assertTrue(os.path.exists('pkg_test1/parsing/lextab.py')) + self.assertTrue(os.path.exists('pkg_test1/parsing/parser.out')) + r = parser.parse('3+4+5') + self.assertEqual(r, 12) + + def test_pkg_test2(self): + from pkg_test2 import parser + self.assertTrue(os.path.exists('pkg_test2/parsing/calcparsetab.py')) + self.assertTrue(os.path.exists('pkg_test2/parsing/calclextab.py')) + self.assertTrue(os.path.exists('pkg_test2/parsing/parser.out')) + r = parser.parse('3+4+5') + self.assertEqual(r, 12) + + def test_pkg_test3(self): + from pkg_test3 import parser + self.assertTrue(os.path.exists('pkg_test3/generated/parsetab.py')) + self.assertTrue(os.path.exists('pkg_test3/generated/lextab.py')) + self.assertTrue(os.path.exists('pkg_test3/generated/parser.out')) + r = parser.parse('3+4+5') + self.assertEqual(r, 12) + unittest.main() -- cgit v1.2.1