Further refinement of table handling with packages. More unit tests.

20 files changed, 470 insertions, 53 deletions

diff --git a/CHANGES b/CHANGES
index 08e4876..3dcdef9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,18 +1,28 @@
 Version 3.6
 ---------------------
 04/24/15: beazley
-          Fixed some problems related to use of packages and table file
-          modules.  Just to emphasize, if you use a command such as this:
+          Fixed some issues related to use of packages and table file
+          modules.  Just to emphasize, PLY now generates its special
+          files such as 'parsetab.py' and 'lextab.py' in the *SAME*
+          directory as the source file that uses lex() and yacc().
 
-              # lexer.py
-              parser = yacc.yacc(tabmodule='foo.bar.parsetab')
+	  If for some reason, you want to change the name of the table
+          module, use the tabmodule and lextab options:
 
-          You need to make sure that the file 'lexer.py' is located at
-          foo/bar/lexer.py.  If this is not the same location, then
-          you need to also include the outputdir option
+             lexer = lex.lex(lextab='spamlextab')
+             parser = yacc.yacc(tabmodule='spamparsetab')
 
-              parse = yacc.yacc(tabmodule='foo.bar.parsetab', 
-	                        outputdir='foo/bar')
+          If you specify a simple name as shown, the module will still be
+          created in the same directory as the file invoking lex() or yacc().
+          If you want the table files to be placed into a different package,
+          then give a fully qualified package name.  For example:
+
+             lexer = lex.lex(lextab='pkgname.files.lextab')
+             parser = yacc.yacc(tabmodule='pkgname.files.parsetab')
+
+          For this to work, 'pkgname.files' must already exist as a valid 
+          Python package (i.e., the directories must already exist and be
+          set up with the proper __init__.py files, etc.).
 
 Version 3.5
 ---------------------
diff --git a/README.md b/README.md
index 0d01f7a..c630d81 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-PLY (Python Lex-Yacc)                   Version 3.5
+PLY (Python Lex-Yacc)                   Version 3.6
 
 Copyright (C) 2001-2015,
 David M. Beazley (Dabeaz LLC)
diff --git a/doc/ply.html b/doc/ply.html
index 95ab9d0..631d76f 100644
--- a/doc/ply.html
+++ b/doc/ply.html
@@ -12,7 +12,7 @@ dave@dabeaz.com<br>
 </b>
 
 <p>
-<b>PLY Version: 3.5</b>
+<b>PLY Version: 3.6</b>
 <p>
 
 <!-- INDEX -->
diff --git a/ply/lex.py b/ply/lex.py
index 28f99fe..d93c91d 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -171,21 +171,10 @@ class Lexer:
     # ------------------------------------------------------------
     # writetab() - Write lexer information to a table file
     # ------------------------------------------------------------
-    def writetab(self, tabfile, outputdir=''):
-        if isinstance(tabfile, types.ModuleType):
-            return
-        parts = tabfile.split('.')
-        basetabfilename = parts[-1]
-        if not outputdir and len(parts) > 1:
-            # If no explicit output directory was given, then set it to the location of the tabfile
-            packagename = '.'.join(parts[:-1])
-            exec('import %s' % packagename)
-            package = sys.modules[packagename]
-            outputdir = os.path.dirname(package.__file__)
-            
-        filename = os.path.join(outputdir, basetabfilename) + '.py'
+    def writetab(self, basetabmodule, outputdir=''):
+        filename = os.path.join(outputdir, basetabmodule) + '.py'
         with open(filename, 'w') as tf:
-            tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (tabfile, __version__))
+            tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__))
             tf.write('_tabversion   = %s\n' % repr(__tabversion__))
             tf.write('_lextokens    = %s\n' % repr(self.lextokens))
             tf.write('_lexreflags   = %s\n' % repr(self.lexreflags))
@@ -886,19 +875,38 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab',
     if object:
         module = object
 
+    # Get the module dictionary used for the parser
     if module:
         _items = [(k, getattr(module, k)) for k in dir(module)]
         ldict = dict(_items)
-        if outputdir is None:
-            srcfile = getattr(module, '__file__', None)
-            if srcfile is None:
-                if hasattr(module, '__module__'):
-                    srcfile = getattr(sys.modules[module.__module__], '__file__', '')
-            outputdir = os.path.dirname(srcfile)
+        # If no __file__ attribute is available, try to obtain it from the __module__ instead
+        if '__file__' not in ldict:
+            ldict['__file__'] = sys.modules[ldict['__module__']].__file__
     else:
         ldict = get_caller_module_dict(2)
-        if outputdir is None:
-            outputdir = os.path.dirname(ldict.get('__file__', ''))
+
+    if outputdir is None:
+        # If no output directory is set, the location of the output files
+        # is determined according to the following rules:
+        #     - If lextab specifies a package, files go into that package directory
+        #     - Otherwise, files go in the same directory as the specifying module
+        if '.' not in lextab:
+            srcfile = ldict['__file__']
+        else:
+            parts = lextab.split('.')
+            pkgname = '.'.join(parts[:-1])
+            exec('import %s' % pkgname)
+            srcfile = getattr(sys.modules[pkgname], '__file__', '')
+        outputdir = os.path.dirname(srcfile)
+
+    # Determine if the module is package of a package or not.
+    # If so, fix the tabmodule setting so that tables load correctly
+    pkg = ldict.get('__package__')
+    if pkg:
+        if '.' not in lextab:
+            lextab = pkg + '.' + lextab
+
+    baselextab = lextab.split('.')[-1]
 
     # Collect parser information from the dictionary
     linfo = LexerReflect(ldict, log=errorlog, reflags=reflags)
@@ -1021,7 +1029,7 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab',
 
     # If in optimize mode, we write the lextab
     if lextab and optimize:
-        lexobj.writetab(lextab, outputdir)
+        lexobj.writetab(baselextab, outputdir)
 
     return lexobj
 
diff --git a/ply/yacc.py b/ply/yacc.py
index 5bbe419..fb2a9ef 100644
--- a/ply/yacc.py
+++ b/ply/yacc.py
@@ -2692,15 +2692,7 @@ class LRGeneratedTable(LRTable):
     # This function writes the LR parsing tables to a file
     # -----------------------------------------------------------------------------
 
-    def write_table(self, modulename, outputdir='', signature=''):
-        parts = modulename.split('.')
-        basemodulename = parts[-1]
-        if not outputdir and len(parts) > 1:
-            # If no explicit output directory was given, then set it to the location of the tabfile
-            packagename = '.'.join(parts[:-1])
-            exec('import %s' % packagename)
-            package = sys.modules[packagename]
-            outputdir = os.path.dirname(package.__file__)
+    def write_table(self, basemodulename, outputdir='', signature=''):
         filename = os.path.join(outputdir, basemodulename) + '.py'
         try:
             f = open(filename, 'w')
@@ -3203,16 +3195,33 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
     if module:
         _items = [(k, getattr(module, k)) for k in dir(module)]
         pdict = dict(_items)
-        if outputdir is None:
-            srcfile = getattr(module, '__file__', None)
-            if srcfile is None:
-                if hasattr(module, '__module__'):
-                    srcfile = getattr(sys.modules[module.__module__], '__file__', '')
-            outputdir = os.path.dirname(srcfile)
+        # If no __file__ attribute is available, try to obtain it from the __module__ instead
+        if '__file__' not in pdict:
+            pdict['__file__'] = sys.modules[pdict['__module__']].__file__
     else:
         pdict = get_caller_module_dict(2)
-        if outputdir is None:
-            outputdir = os.path.dirname(pdict.get('__file__', ''))
+
+    if outputdir is None:
+        # If no output directory is set, the location of the output files
+        # is determined according to the following rules:
+        #     - If tabmodule specifies a package, files go into that package directory
+        #     - Otherwise, files go in the same directory as the specifying module
+        if '.' not in tabmodule:
+            srcfile = pdict['__file__']
+        else:
+            parts = tabmodule.split('.')
+            pkgname = '.'.join(parts[:-1])
+            exec('import %s' % pkgname)
+            srcfile = getattr(sys.modules[pkgname], '__file__', '')
+        outputdir = os.path.dirname(srcfile)
+
+    # Determine if the module is package of a package or not.
+    # If so, fix the tabmodule setting so that tables load correctly
+    pkg = pdict.get('__package__')
+    if pkg and '.' not in tabmodule:
+        tabmodule = pkg + '.' + tabmodule
+
+    basetabmodule = tabmodule.split('.')[-1]
 
     # Set start symbol if it's specified directly using an argument
     if start is not None:
@@ -3420,7 +3429,7 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
 
     # Write the table file if requested
     if write_tables:
-        lr.write_table(tabmodule, outputdir, signature)
+        lr.write_table(basetabmodule, outputdir, signature)
 
     # Write a pickled version of the tables
     if picklefile:
diff --git a/setup.py b/setup.py
index 1e95122..c6f2b44 100644
--- a/setup.py
+++ b/setup.py
@@ -17,7 +17,7 @@ PLY is extremely easy to use and provides very extensive error checking.
 It is compatible with both Python 2 and Python 3.
 """,
             license="""BSD""",
-            version = "3.5",
+            version = "3.6",
             author = "David Beazley",
             author_email = "dave@dabeaz.com",
             maintainer = "David Beazley",
diff --git a/test/pkg_test1/__init__.py b/test/pkg_test1/__init__.py
new file mode 100644
index 0000000..0e19558
--- /dev/null
+++ b/test/pkg_test1/__init__.py
@@ -0,0 +1,9 @@
+# Tests proper handling of lextab and parsetab files in package structures
+
+# Here for testing purposes
+import sys
+if '..' not in sys.path:  
+    sys.path.insert(0, '..')
+
+from .parsing.calcparse import parser
+
diff --git a/test/pkg_test1/parsing/__init__.py b/test/pkg_test1/parsing/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/pkg_test1/parsing/__init__.py
diff --git a/test/pkg_test1/parsing/calclex.py b/test/pkg_test1/parsing/calclex.py
new file mode 100644
index 0000000..b3c1a4d
--- /dev/null
+++ b/test/pkg_test1/parsing/calclex.py
@@ -0,0 +1,47 @@
+# -----------------------------------------------------------------------------
+# calclex.py
+# -----------------------------------------------------------------------------
+
+import ply.lex as lex
+
+tokens = (
+    'NAME','NUMBER',
+    'PLUS','MINUS','TIMES','DIVIDE','EQUALS',
+    'LPAREN','RPAREN',
+    )
+
+# Tokens
+
+t_PLUS    = r'\+'
+t_MINUS   = r'-'
+t_TIMES   = r'\*'
+t_DIVIDE  = r'/'
+t_EQUALS  = r'='
+t_LPAREN  = r'\('
+t_RPAREN  = r'\)'
+t_NAME    = r'[a-zA-Z_][a-zA-Z0-9_]*'
+
+def t_NUMBER(t):
+    r'\d+'
+    try:
+        t.value = int(t.value)
+    except ValueError:
+        print("Integer value too large %s" % t.value)
+        t.value = 0
+    return t
+
+t_ignore = " \t"
+
+def t_newline(t):
+    r'\n+'
+    t.lexer.lineno += t.value.count("\n")
+    
+def t_error(t):
+    print("Illegal character '%s'" % t.value[0])
+    t.lexer.skip(1)
+    
+# Build the lexer
+lexer = lex.lex(optimize=True)
+
+
+
diff --git a/test/pkg_test1/parsing/calcparse.py b/test/pkg_test1/parsing/calcparse.py
new file mode 100644
index 0000000..c058e9f
--- /dev/null
+++ b/test/pkg_test1/parsing/calcparse.py
@@ -0,0 +1,66 @@
+# -----------------------------------------------------------------------------
+# yacc_simple.py
+#
+# A simple, properly specifier grammar
+# -----------------------------------------------------------------------------
+
+from .calclex import tokens
+from ply import yacc
+
+# Parsing rules
+precedence = (
+    ('left','PLUS','MINUS'),
+    ('left','TIMES','DIVIDE'),
+    ('right','UMINUS'),
+    )
+
+# dictionary of names
+names = { }
+
+def p_statement_assign(t):
+    'statement : NAME EQUALS expression'
+    names[t[1]] = t[3]
+
+def p_statement_expr(t):
+    'statement : expression'
+    t[0] = t[1]
+
+def p_expression_binop(t):
+    '''expression : expression PLUS expression
+                  | expression MINUS expression
+                  | expression TIMES expression
+                  | expression DIVIDE expression'''
+    if t[2] == '+'  : t[0] = t[1] + t[3]
+    elif t[2] == '-': t[0] = t[1] - t[3]
+    elif t[2] == '*': t[0] = t[1] * t[3]
+    elif t[2] == '/': t[0] = t[1] / t[3]
+
+def p_expression_uminus(t):
+    'expression : MINUS expression %prec UMINUS'
+    t[0] = -t[2]
+
+def p_expression_group(t):
+    'expression : LPAREN expression RPAREN'
+    t[0] = t[2]
+
+def p_expression_number(t):
+    'expression : NUMBER'
+    t[0] = t[1]
+
+def p_expression_name(t):
+    'expression : NAME'
+    try:
+        t[0] = names[t[1]]
+    except LookupError:
+        print("Undefined name '%s'" % t[1])
+        t[0] = 0
+
+def p_error(t):
+    print("Syntax error at '%s'" % t.value)
+
+parser = yacc.yacc()
+
+
+
+
+
diff --git a/test/pkg_test2/__init__.py b/test/pkg_test2/__init__.py
new file mode 100644
index 0000000..0e19558
--- /dev/null
+++ b/test/pkg_test2/__init__.py
@@ -0,0 +1,9 @@
+# Tests proper handling of lextab and parsetab files in package structures
+
+# Here for testing purposes
+import sys
+if '..' not in sys.path:  
+    sys.path.insert(0, '..')
+
+from .parsing.calcparse import parser
+
diff --git a/test/pkg_test2/parsing/__init__.py b/test/pkg_test2/parsing/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/pkg_test2/parsing/__init__.py
diff --git a/test/pkg_test2/parsing/calclex.py b/test/pkg_test2/parsing/calclex.py
new file mode 100644
index 0000000..789e13f
--- /dev/null
+++ b/test/pkg_test2/parsing/calclex.py
@@ -0,0 +1,47 @@
+# -----------------------------------------------------------------------------
+# calclex.py
+# -----------------------------------------------------------------------------
+
+import ply.lex as lex
+
+tokens = (
+    'NAME','NUMBER',
+    'PLUS','MINUS','TIMES','DIVIDE','EQUALS',
+    'LPAREN','RPAREN',
+    )
+
+# Tokens
+
+t_PLUS    = r'\+'
+t_MINUS   = r'-'
+t_TIMES   = r'\*'
+t_DIVIDE  = r'/'
+t_EQUALS  = r'='
+t_LPAREN  = r'\('
+t_RPAREN  = r'\)'
+t_NAME    = r'[a-zA-Z_][a-zA-Z0-9_]*'
+
+def t_NUMBER(t):
+    r'\d+'
+    try:
+        t.value = int(t.value)
+    except ValueError:
+        print("Integer value too large %s" % t.value)
+        t.value = 0
+    return t
+
+t_ignore = " \t"
+
+def t_newline(t):
+    r'\n+'
+    t.lexer.lineno += t.value.count("\n")
+    
+def t_error(t):
+    print("Illegal character '%s'" % t.value[0])
+    t.lexer.skip(1)
+    
+# Build the lexer
+lexer = lex.lex(optimize=True, lextab='calclextab')
+
+
+
diff --git a/test/pkg_test2/parsing/calcparse.py b/test/pkg_test2/parsing/calcparse.py
new file mode 100644
index 0000000..f519338
--- /dev/null
+++ b/test/pkg_test2/parsing/calcparse.py
@@ -0,0 +1,66 @@
+# -----------------------------------------------------------------------------
+# yacc_simple.py
+#
+# A simple, properly specifier grammar
+# -----------------------------------------------------------------------------
+
+from .calclex import tokens
+from ply import yacc
+
+# Parsing rules
+precedence = (
+    ('left','PLUS','MINUS'),
+    ('left','TIMES','DIVIDE'),
+    ('right','UMINUS'),
+    )
+
+# dictionary of names
+names = { }
+
+def p_statement_assign(t):
+    'statement : NAME EQUALS expression'
+    names[t[1]] = t[3]
+
+def p_statement_expr(t):
+    'statement : expression'
+    t[0] = t[1]
+
+def p_expression_binop(t):
+    '''expression : expression PLUS expression
+                  | expression MINUS expression
+                  | expression TIMES expression
+                  | expression DIVIDE expression'''
+    if t[2] == '+'  : t[0] = t[1] + t[3]
+    elif t[2] == '-': t[0] = t[1] - t[3]
+    elif t[2] == '*': t[0] = t[1] * t[3]
+    elif t[2] == '/': t[0] = t[1] / t[3]
+
+def p_expression_uminus(t):
+    'expression : MINUS expression %prec UMINUS'
+    t[0] = -t[2]
+
+def p_expression_group(t):
+    'expression : LPAREN expression RPAREN'
+    t[0] = t[2]
+
+def p_expression_number(t):
+    'expression : NUMBER'
+    t[0] = t[1]
+
+def p_expression_name(t):
+    'expression : NAME'
+    try:
+        t[0] = names[t[1]]
+    except LookupError:
+        print("Undefined name '%s'" % t[1])
+        t[0] = 0
+
+def p_error(t):
+    print("Syntax error at '%s'" % t.value)
+
+parser = yacc.yacc(tabmodule='calcparsetab')
+
+
+
+
+
diff --git a/test/pkg_test3/__init__.py b/test/pkg_test3/__init__.py
new file mode 100644
index 0000000..0e19558
--- /dev/null
+++ b/test/pkg_test3/__init__.py
@@ -0,0 +1,9 @@
+# Tests proper handling of lextab and parsetab files in package structures
+
+# Here for testing purposes
+import sys
+if '..' not in sys.path:  
+    sys.path.insert(0, '..')
+
+from .parsing.calcparse import parser
+
diff --git a/test/pkg_test3/generated/__init__.py b/test/pkg_test3/generated/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/pkg_test3/generated/__init__.py
diff --git a/test/pkg_test3/parsing/__init__.py b/test/pkg_test3/parsing/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/test/pkg_test3/parsing/__init__.py
diff --git a/test/pkg_test3/parsing/calclex.py b/test/pkg_test3/parsing/calclex.py
new file mode 100644
index 0000000..6ca2c4f
--- /dev/null
+++ b/test/pkg_test3/parsing/calclex.py
@@ -0,0 +1,47 @@
+# -----------------------------------------------------------------------------
+# calclex.py
+# -----------------------------------------------------------------------------
+
+import ply.lex as lex
+
+tokens = (
+    'NAME','NUMBER',
+    'PLUS','MINUS','TIMES','DIVIDE','EQUALS',
+    'LPAREN','RPAREN',
+    )
+
+# Tokens
+
+t_PLUS    = r'\+'
+t_MINUS   = r'-'
+t_TIMES   = r'\*'
+t_DIVIDE  = r'/'
+t_EQUALS  = r'='
+t_LPAREN  = r'\('
+t_RPAREN  = r'\)'
+t_NAME    = r'[a-zA-Z_][a-zA-Z0-9_]*'
+
+def t_NUMBER(t):
+    r'\d+'
+    try:
+        t.value = int(t.value)
+    except ValueError:
+        print("Integer value too large %s" % t.value)
+        t.value = 0
+    return t
+
+t_ignore = " \t"
+
+def t_newline(t):
+    r'\n+'
+    t.lexer.lineno += t.value.count("\n")
+    
+def t_error(t):
+    print("Illegal character '%s'" % t.value[0])
+    t.lexer.skip(1)
+    
+# Build the lexer
+lexer = lex.lex(optimize=True, lextab='pkg_test3.generated.lextab')
+
+
+
diff --git a/test/pkg_test3/parsing/calcparse.py b/test/pkg_test3/parsing/calcparse.py
new file mode 100644
index 0000000..2dcb52b
--- /dev/null
+++ b/test/pkg_test3/parsing/calcparse.py
@@ -0,0 +1,66 @@
+# -----------------------------------------------------------------------------
+# yacc_simple.py
+#
+# A simple, properly specifier grammar
+# -----------------------------------------------------------------------------
+
+from .calclex import tokens
+from ply import yacc
+
+# Parsing rules
+precedence = (
+    ('left','PLUS','MINUS'),
+    ('left','TIMES','DIVIDE'),
+    ('right','UMINUS'),
+    )
+
+# dictionary of names
+names = { }
+
+def p_statement_assign(t):
+    'statement : NAME EQUALS expression'
+    names[t[1]] = t[3]
+
+def p_statement_expr(t):
+    'statement : expression'
+    t[0] = t[1]
+
+def p_expression_binop(t):
+    '''expression : expression PLUS expression
+                  | expression MINUS expression
+                  | expression TIMES expression
+                  | expression DIVIDE expression'''
+    if t[2] == '+'  : t[0] = t[1] + t[3]
+    elif t[2] == '-': t[0] = t[1] - t[3]
+    elif t[2] == '*': t[0] = t[1] * t[3]
+    elif t[2] == '/': t[0] = t[1] / t[3]
+
+def p_expression_uminus(t):
+    'expression : MINUS expression %prec UMINUS'
+    t[0] = -t[2]
+
+def p_expression_group(t):
+    'expression : LPAREN expression RPAREN'
+    t[0] = t[2]
+
+def p_expression_number(t):
+    'expression : NUMBER'
+    t[0] = t[1]
+
+def p_expression_name(t):
+    'expression : NAME'
+    try:
+        t[0] = names[t[1]]
+    except LookupError:
+        print("Undefined name '%s'" % t[1])
+        t[0] = 0
+
+def p_error(t):
+    print("Syntax error at '%s'" % t.value)
+
+parser = yacc.yacc(tabmodule='pkg_test3.generated.parsetab')
+
+
+
+
+
diff --git a/test/testyacc.py b/test/testyacc.py
index 96d4b0d..e5223ae 100644
--- a/test/testyacc.py
+++ b/test/testyacc.py
@@ -401,4 +401,28 @@ class YaccErrorWarningTests(unittest.TestCase):
                                     "Precedence rule 'left' defined for unknown symbol '/'\n"
                                     ))
 
+    def test_pkg_test1(self):
+        from pkg_test1 import parser
+        self.assertTrue(os.path.exists('pkg_test1/parsing/parsetab.py'))
+        self.assertTrue(os.path.exists('pkg_test1/parsing/lextab.py'))
+        self.assertTrue(os.path.exists('pkg_test1/parsing/parser.out'))
+        r = parser.parse('3+4+5')
+        self.assertEqual(r, 12)
+
+    def test_pkg_test2(self):
+        from pkg_test2 import parser
+        self.assertTrue(os.path.exists('pkg_test2/parsing/calcparsetab.py'))
+        self.assertTrue(os.path.exists('pkg_test2/parsing/calclextab.py'))
+        self.assertTrue(os.path.exists('pkg_test2/parsing/parser.out'))
+        r = parser.parse('3+4+5')
+        self.assertEqual(r, 12)
+
+    def test_pkg_test3(self):
+        from pkg_test3 import parser
+        self.assertTrue(os.path.exists('pkg_test3/generated/parsetab.py'))
+        self.assertTrue(os.path.exists('pkg_test3/generated/lextab.py'))
+        self.assertTrue(os.path.exists('pkg_test3/generated/parser.out'))
+        r = parser.parse('3+4+5')
+        self.assertEqual(r, 12)
+
 unittest.main()