Improvements to output file handling. Python2/3 compatibility. Table generation

5 files changed, 94 insertions, 36 deletions

diff --git a/CHANGES b/CHANGES
index 53651fa..95bbfb1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,24 @@
 Version 3.5
 ---------------------
+04/19/15: beazley
+          Import of the 'parsetab.py' file is now constrained to only consider the
+          directory specified by the outputdir argument to yacc().  If not supplied,
+          the import will only consider the directory in which the grammar is defined.
+          This should greatly reduce problems with the wrong parsetab.py file being
+          imported by mistake. For example, if it's found somewhere else on the path
+          by accident.
+
+04/19/15: beazley
+          Changed default output directory to be the same as that in which the
+          yacc grammar is defined.  If your grammar is in a file 'calc.py',
+          then the parsetab.py and parser.out files should be generated in the
+          same directory as that file.  
+
+04/19/15: beazley
+          Changed the parsetab.py file signature slightly so that the parsetab won't
+          regenerate if created on a different major version of Python (ie., a 
+          parsetab created on Python 2 will work with Python 3).
+
 04/16/15: beazley
           Fixed Issue #44 call_errorfunc() should return the result of errorfunc()
 
diff --git a/README.md b/README.md
index 43bd6c9..58507a8 100644
--- a/README.md
+++ b/README.md
@@ -96,7 +96,7 @@ A simple example is found at the end of this document
 
 Requirements
 ============
-PLY requires the use of Python 2.2 or greater.  However, you should
+PLY requires the use of Python 2.6 or greater.  However, you should
 use the latest Python release if possible.  It should work on just
 about any platform.  PLY has been tested with both CPython and Jython.
 It also seems to work with IronPython.
diff --git a/doc/ply.html b/doc/ply.html
index b0be3ab..13113f6 100644
--- a/doc/ply.html
+++ b/doc/ply.html
@@ -90,12 +90,8 @@ into a big development project with PLY.
 </p>
 
 <p>
-PLY-3.0 is compatible with both Python 2 and Python 3.  Be aware that
-Python 3 support is new and has not been extensively tested (although
-all of the examples and unit tests pass under Python 3.0).   If you are
-using Python 2, you should try to use Python 2.4 or newer.  Although PLY
-works with versions as far back as Python 2.2, some of its optional features
-require more modern library modules.
+PLY-3.5 is compatible with both Python 2 and Python 3.  If you are using
+Python 2, you should use Python 2.6 or newer.
 </p>
 
 <H2><a name="ply_nn1"></a>2. Introduction</H2>
diff --git a/ply/lex.py b/ply/lex.py
index d94654b..15ca924 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -31,8 +31,8 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 # -----------------------------------------------------------------------------
 
-__version__    = "3.5"
-__tabversion__ = "3.5"       # Version of table file used
+__version__    = '3.5'
+__tabversion__ = '3.5'       # Version of table file used
 
 import re, sys, types, copy, os, inspect
 
@@ -208,21 +208,38 @@ class Lexer:
              else:
                   taberr[key] = None
         tf.write("_lexstateerrorf = %s\n" % repr(taberr))
+
+        tabeof = { }
+        for key, ef in self.lexstateeoff.items():
+             if ef:
+                  tabeof[key] = ef.__name__
+             else:
+                  tabeof[key] = None
+        tf.write("_lexstateeoff = %s\n" % repr(tabeof))
         tf.close()
 
     # ------------------------------------------------------------
     # readtab() - Read lexer information from a tab file
     # ------------------------------------------------------------
-    def readtab(self,tabfile,fdict):
+    def readtab(self, outputdir, tabfile, fdict):
         if isinstance(tabfile,types.ModuleType):
             lextab = tabfile
         else:
+            basetabname = tabfile.split(".")[-1]
+            oldpath = sys.path
+            sys.path = [outputdir]
+            try:
+                lextab = __import__(basetabname)
+            finally:
+                sys.path = oldpath
+            '''
             if sys.version_info[0] < 3:
                 exec("import %s as lextab" % tabfile)
             else:
                 env = { }
                 exec("import %s as lextab" % tabfile, env,env)
                 lextab = env['lextab']
+                '''
 
         if getattr(lextab,"_tabversion","0.0") != __tabversion__:
             raise ImportError("Inconsistent PLY version")
@@ -245,6 +262,10 @@ class Lexer:
         self.lexstateerrorf = { }
         for key,ef in lextab._lexstateerrorf.items():
              self.lexstateerrorf[key] = fdict[ef]
+        self.lexstateeoff = { }
+        for key,ef in lextab._lexstateeoff.items():
+             self.lexstateeoff[key] = fdict[ef]
+
         self.begin('INITIAL')
 
     # ------------------------------------------------------------
@@ -885,13 +906,16 @@ class LexerReflect(object):
 #
 # Build all of the regular expression rules from definitions in the supplied module
 # -----------------------------------------------------------------------------
-def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,nowarn=0,outputdir="", debuglog=None, errorlog=None):
+
+def lex(module=None, object=None, debug=False, optimize=False, lextab="lextab",
+        reflags=0, nowarn=False, outputdir=None, debuglog=None, errorlog=None):
     global lexer
+
     ldict = None
     stateinfo  = { 'INITIAL' : 'inclusive'}
     lexobj = Lexer()
     lexobj.lexoptimize = optimize
-    global token,input
+    global token, input
 
     if errorlog is None:
         errorlog = PlyLogger(sys.stderr)
@@ -901,16 +925,25 @@ def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,now
             debuglog = PlyLogger(sys.stderr)
 
     # Get the module dictionary used for the lexer
-    if object: module = object
+    if object: 
+        module = object
 
     if module:
         _items = [(k,getattr(module,k)) for k in dir(module)]
         ldict = dict(_items)
+        if outputdir is None:
+            srcfile = getattr(module, '__file__', None)
+            if srcfile is None:
+                if hasattr(module, '__module__'):
+                    srcfile = getattr(sys.modules[module.__module__], '__file__', '')
+            outputdir = os.path.dirname(srcfile)
     else:
         ldict = get_caller_module_dict(2)
+        if outputdir is None:
+            outputdir = os.path.dirname(ldict.get('__file__', ''))
 
     # Collect parser information from the dictionary
-    linfo = LexerReflect(ldict,log=errorlog,reflags=reflags)
+    linfo = LexerReflect(ldict, log=errorlog, reflags=reflags)
     linfo.get_all()
     if not optimize:
         if linfo.validate_all():
@@ -918,7 +951,7 @@ def lex(module=None,object=None,debug=0,optimize=0,lextab="lextab",reflags=0,now
 
     if optimize and lextab:
         try:
-            lexobj.readtab(lextab,ldict)
+            lexobj.readtab(outputdir, lextab, ldict)
             token = lexobj.token
             input = lexobj.input
             lexer = lexobj
diff --git a/ply/yacc.py b/ply/yacc.py
index a2237e8..f80e060 100644
--- a/ply/yacc.py
+++ b/ply/yacc.py
@@ -59,8 +59,8 @@
 # own risk!
 # ----------------------------------------------------------------------------
 
-__version__    = "3.5"
-__tabversion__ = "3.2"       # Table version
+__version__    = '3.5'
+__tabversion__ = '3.5'       # Table version
 
 #-----------------------------------------------------------------------------
 #                     === User configurable parameters ===
@@ -84,7 +84,7 @@ resultlimit = 40               # Size limit of results when running in debug mod
 
 pickle_protocol = 0            # Protocol to use when writing pickle files
 
-import re, types, sys, os.path, inspect
+import re, types, sys, os.path, inspect, base64
 
 # Compatibility function for python 2.6/3.0
 if sys.version_info[0] < 3:
@@ -281,7 +281,6 @@ class YaccProduction:
     def error(self):
        raise SyntaxError
 
-
 # -----------------------------------------------------------------------------
 #                               == LRParser ==
 #
@@ -1881,16 +1880,16 @@ class LRTable(object):
         self.lr_productions = None
         self.lr_method = None
 
-    def read_table(self,module):
+    def read_table(self, outputdir, module):
         if isinstance(module,types.ModuleType):
             parsetab = module
         else:
-            if sys.version_info[0] < 3:
-                exec("import %s as parsetab" % module)
-            else:
-                env = { }
-                exec("import %s as parsetab" % module, env, env)
-                parsetab = env['parsetab']
+            oldpath = sys.path
+            sys.path = [outputdir]
+            try:
+                parsetab = __import__(module)
+            finally:
+                sys.path = oldpath
 
         if parsetab._tabversion != __tabversion__:
             raise VersionError("yacc table file version is out of date")
@@ -2742,7 +2741,7 @@ del _lr_goto_items
         outp = []
         for p in self.lr_productions:
             if p.func:
-                outp.append((p.str,p.name, p.len, p.func,p.file,p.line))
+                outp.append((p.str,p.name, p.len, p.func,os.path.basename(p.file),p.line))
             else:
                 outp.append((str(p),p.name,p.len,None,None,None))
         pickle.dump(outp,outf,pickle_protocol)
@@ -2875,7 +2874,11 @@ class ParserReflect(object):
                     sig.update(f[3].encode('latin-1'))
         except (TypeError,ValueError):
             pass
-        return sig.digest()
+
+        digest = base64.b16encode(sig.digest())
+        if sys.version_info[0] >= 3:
+            digest = digest.decode('latin-1')
+        return digest
 
     # -----------------------------------------------------------------------------
     # validate_modules()
@@ -3094,13 +3097,13 @@ class ParserReflect(object):
 # -----------------------------------------------------------------------------
 
 def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, start=None, 
-         check_recursion=1, optimize=0, write_tables=1, debugfile=debug_file,outputdir='',
-         debuglog=None, errorlog = None, picklefile=None):
+         check_recursion=True, optimize=False, write_tables=True, debugfile=debug_file, 
+         outputdir=None, debuglog=None, errorlog=None, picklefile=None):
 
-    global parse                 # Reference to the parsing method of the last built parser
+    # Reference to the parsing method of the last built parser
+    global parse
 
     # If pickling is enabled, table files are not created
-
     if picklefile:
         write_tables = 0
 
@@ -3111,8 +3114,16 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
     if module:
         _items = [(k,getattr(module,k)) for k in dir(module)]
         pdict = dict(_items)
+        if outputdir is None:
+            srcfile = getattr(module, '__file__', None)
+            if srcfile is None:
+                if hasattr(module, '__module__'):
+                    srcfile = getattr(sys.modules[module.__module__], '__file__', '')
+            outputdir = os.path.dirname(srcfile)
     else:
         pdict = get_caller_module_dict(2)
+        if outputdir is None:
+            outputdir = os.path.dirname(pdict.get('__file__', ''))
 
     # Set start symbol if it's specified directly using an argument
     if start is not None:
@@ -3134,7 +3145,7 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
         if picklefile:
             read_signature = lr.read_pickle(picklefile)
         else:
-            read_signature = lr.read_table(tabmodule)
+            read_signature = lr.read_table(outputdir, tabmodule)
         if optimize or (read_signature == signature):
             try:
                 lr.bind_callables(pinfo.pdict)
@@ -3146,19 +3157,18 @@ def yacc(method='LALR', debug=yaccdebug, module=None, tabmodule=tab_module, star
                 errorlog.warning("There was a problem loading the table file: %s", repr(e))
     except VersionError:
         e = sys.exc_info()
-        errorlog.warning(str(e))
+        errorlog.warning(str(e[1]))
     except Exception:
         pass
 
     if debuglog is None:
         if debug:
-            debuglog = PlyLogger(open(debugfile,"w"))
+            debuglog = PlyLogger(open(os.path.join(outputdir, debugfile),"w"))
         else:
             debuglog = NullLogger()
 
     debuglog.info("Created by PLY version %s (http://www.dabeaz.com/ply)", __version__)
 
-
     errors = 0
 
     # Validate the parser information