Fixed potential package problems with lextab and parsetab files

4 files changed, 42 insertions, 20 deletions

diff --git a/CHANGES b/CHANGES
index c63d99d..08e4876 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,3 +1,19 @@
+Version 3.6
+---------------------
+04/24/15: beazley
+          Fixed some problems related to use of packages and table file
+          modules.  Just to emphasize, if you use a command such as this:
+
+              # lexer.py
+              parser = yacc.yacc(tabmodule='foo.bar.parsetab')
+
+          You need to make sure that the file 'lexer.py' is located at
+          foo/bar/lexer.py.  If this is not the same location, then
+          you need to also include the outputdir option
+
+              parse = yacc.yacc(tabmodule='foo.bar.parsetab', 
+	                        outputdir='foo/bar')
+
 Version 3.5
 ---------------------
 04/21/15: beazley
diff --git a/ply/lex.py b/ply/lex.py
index 9bc9d09..28f99fe 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -174,7 +174,15 @@ class Lexer:
     def writetab(self, tabfile, outputdir=''):
         if isinstance(tabfile, types.ModuleType):
             return
-        basetabfilename = tabfile.split('.')[-1]
+        parts = tabfile.split('.')
+        basetabfilename = parts[-1]
+        if not outputdir and len(parts) > 1:
+            # If no explicit output directory was given, then set it to the location of the tabfile
+            packagename = '.'.join(parts[:-1])
+            exec('import %s' % packagename)
+            package = sys.modules[packagename]
+            outputdir = os.path.dirname(package.__file__)
+            
         filename = os.path.join(outputdir, basetabfilename) + '.py'
         with open(filename, 'w') as tf:
             tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (tabfile, __version__))
@@ -208,17 +216,12 @@ class Lexer:
     # ------------------------------------------------------------
     # readtab() - Read lexer information from a tab file
     # ------------------------------------------------------------
-    def readtab(self, outputdir, tabfile, fdict):
+    def readtab(self, tabfile, fdict):
         if isinstance(tabfile, types.ModuleType):
             lextab = tabfile
         else:
-            basetabname = tabfile.split('.')[-1]
-            oldpath = sys.path
-            sys.path = [outputdir]
-            try:
-                lextab = __import__(basetabname)
-            finally:
-                sys.path = oldpath
+            exec('import %s' % tabfile)
+            lextab = sys.modules[tabfile]
 
         if getattr(lextab, '_tabversion', '0.0') != __tabversion__:
             raise ImportError('Inconsistent PLY version')
@@ -906,7 +909,7 @@ def lex(module=None, object=None, debug=False, optimize=False, lextab='lextab',
 
     if optimize and lextab:
         try:
-            lexobj.readtab(outputdir, lextab, ldict)
+            lexobj.readtab(lextab, ldict)
             token = lexobj.token
             input = lexobj.input
             lexer = lexobj
diff --git a/ply/yacc.py b/ply/yacc.py
index 8fa5c64..5bbe419 100644
--- a/ply/yacc.py
+++ b/ply/yacc.py
@@ -1957,16 +1957,12 @@ class LRTable(object):
         self.lr_productions = None
         self.lr_method = None
 
-    def read_table(self, outputdir, module):
+    def read_table(self, module):
         if isinstance(module, types.ModuleType):
             parsetab = module
         else:
-            oldpath = sys.path
-            sys.path = [outputdir]
-            try:
-                parsetab = __import__(module)
-            finally:
-                sys.path = oldpath
+            exec('import %s' % module)
+            parsetab = sys.modules[module]
 
         if parsetab._tabversion != __tabversion__:
             raise VersionError('yacc table file version is out of date')
@@ -2697,7 +2693,14 @@ class LRGeneratedTable(LRTable):
     # -----------------------------------------------------------------------------
 
     def write_table(self, modulename, outputdir='', signature=''):
-        basemodulename = modulename.split('.')[-1]
+        parts = modulename.split('.')
+        basemodulename = parts[-1]
+        if not outputdir and len(parts) > 1:
+            # If no explicit output directory was given, then set it to the location of the tabfile
+            packagename = '.'.join(parts[:-1])
+            exec('import %s' % packagename)
+            package = sys.modules[packagename]
+            outputdir = os.path.dirname(package.__file__)
         filename = os.path.join(outputdir, basemodulename) + '.py'
         try:
             f = open(filename, 'w')
@@ -3231,7 +3234,7 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
         if picklefile:
             read_signature = lr.read_pickle(picklefile)
         else:
-            read_signature = lr.read_table(outputdir, tabmodule)
+            read_signature = lr.read_table(tabmodule)
         if optimize or (read_signature == signature):
             try:
                 lr.bind_callables(pinfo.pdict)
diff --git a/test/lex_optimize3.py b/test/lex_optimize3.py
index c6c8cce..b8df5aa 100644
--- a/test/lex_optimize3.py
+++ b/test/lex_optimize3.py
@@ -45,7 +45,7 @@ def t_error(t):
     t.lexer.skip(1)
     
 # Build the lexer
-lex.lex(optimize=1,lextab="lexdir.sub.calctab",outputdir="lexdir/sub")
+lex.lex(optimize=1,lextab="lexdir.sub.calctab" ,outputdir="lexdir/sub")
 lex.runmain(data="3+4")