Merge pull request #33 from mdboom/load_unicode_file

Support defining lexers in source files with a magic coding comment

2 files changed, 31 insertions, 42 deletions

diff --git a/ply/lex.py b/ply/lex.py
index 137d28f..8f05537 100644
--- a/ply/lex.py
+++ b/ply/lex.py
@@ -34,7 +34,7 @@
 __version__    = "3.5"
 __tabversion__ = "3.5"       # Version of table file used
 
-import re, sys, types, copy, os
+import re, sys, types, copy, os, inspect
 
 # This tuple contains known string types
 try:
@@ -558,7 +558,7 @@ class LexerReflect(object):
         self.tokens     = []
         self.reflags    = reflags
         self.stateinfo  = { 'INITIAL' : 'inclusive'}
-        self.files      = {}
+        self.modules    = {}
         self.error      = 0
 
         if log is None:
@@ -740,7 +740,8 @@ class LexerReflect(object):
             for fname, f in self.funcsym[state]:
                 line = func_code(f).co_firstlineno
                 file = func_code(f).co_filename
-                self.files[file] = 1
+                module = inspect.getmodule(f)
+                self.modules[module] = 1
 
                 tokname = self.toknames[fname]
                 if isinstance(f, types.MethodType):
@@ -810,7 +811,8 @@ class LexerReflect(object):
                 f = efunc
                 line = func_code(f).co_firstlineno
                 file = func_code(f).co_filename
-                self.files[file] = 1
+                module = inspect.getmodule(f)
+                self.modules[module] = 1
 
                 if isinstance(f, types.MethodType):
                     reqargs = 2
@@ -825,35 +827,26 @@ class LexerReflect(object):
                     self.log.error("%s:%d: Rule '%s' requires an argument", file,line,f.__name__)
                     self.error = 1
 
-        for f in self.files:
-            self.validate_file(f)
+        for module in self.modules:
+            self.validate_module(module)
 
 
     # -----------------------------------------------------------------------------
-    # validate_file()
+    # validate_module()
     #
     # This checks to see if there are duplicated t_rulename() functions or strings
     # in the parser input file.  This is done using a simple regular expression
-    # match on each line in the given file.  
+    # match on each line in the source code of the given module.
     # -----------------------------------------------------------------------------
 
-    def validate_file(self,filename):
-        import os.path
-        base,ext = os.path.splitext(filename)
-        if ext != '.py': return         # No idea what the file is. Return OK
-
-        try:
-            f = open(filename)
-            lines = f.readlines()
-            f.close()
-        except IOError:
-            return                      # Couldn't find the file.  Don't worry about it
+    def validate_module(self, module):
+        lines, linen = inspect.getsourcelines(module)
 
         fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
         sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
 
         counthash = { }
-        linen = 1
+        linen += 1
         for l in lines:
             m = fre.match(l)
             if not m:
@@ -864,6 +857,7 @@ class LexerReflect(object):
                 if not prev:
                     counthash[name] = linen
                 else:
+                    filename = inspect.getsourcefile(module)
                     self.log.error("%s:%d: Rule %s redefined. Previously defined on line %d",filename,linen,name,prev)
                     self.error = 1
             linen += 1
diff --git a/ply/yacc.py b/ply/yacc.py
index 4966299..67eff06 100644
--- a/ply/yacc.py
+++ b/ply/yacc.py
@@ -84,7 +84,7 @@ resultlimit = 40               # Size limit of results when running in debug mod
 
 pickle_protocol = 0            # Protocol to use when writing pickle files
 
-import re, types, sys, os.path
+import re, types, sys, os.path, inspect
 
 # Compatibility function for python 2.6/3.0
 if sys.version_info[0] < 3:
@@ -2818,7 +2818,7 @@ class ParserReflect(object):
         self.start      = None
         self.error_func = None
         self.tokens     = None
-        self.files      = {}
+        self.modules    = {}
         self.grammar    = []
         self.error      = 0
 
@@ -2842,7 +2842,7 @@ class ParserReflect(object):
         self.validate_tokens()
         self.validate_precedence()
         self.validate_pfunctions()
-        self.validate_files()
+        self.validate_modules()
         return self.error
 
     # Compute a signature over the grammar
@@ -2867,7 +2867,7 @@ class ParserReflect(object):
         return sig.digest()
 
     # -----------------------------------------------------------------------------
-    # validate_file()
+    # validate_modules()
     #
     # This method checks to see if there are duplicated p_rulename() functions
     # in the parser module file.  Without this function, it is really easy for
@@ -2877,20 +2877,12 @@ class ParserReflect(object):
     # to try and detect duplicates.
     # -----------------------------------------------------------------------------
 
-    def validate_files(self):
+    def validate_modules(self):
         # Match def p_funcname(
         fre = re.compile(r'\s*def\s+(p_[a-zA-Z_0-9]*)\(')
 
-        for filename in self.files.keys():
-            base,ext = os.path.splitext(filename)
-            if ext != '.py': return 1          # No idea. Assume it's okay.
-
-            try:
-                f = open(filename)
-                lines = f.readlines()
-                f.close()
-            except IOError:
-                continue
+        for module in self.modules.keys():
+            lines, linen = inspect.getsourcelines(module)
 
             counthash = { }
             for linen,l in enumerate(lines):
@@ -2902,6 +2894,7 @@ class ParserReflect(object):
                     if not prev:
                         counthash[name] = linen
                     else:
+                        filename = inspect.getsourcefile(module)
                         self.log.warning("%s:%d: Function %s redefined. Previously defined on line %d", filename,linen,name,prev)
 
     # Get the start symbol
@@ -2932,7 +2925,8 @@ class ParserReflect(object):
 
             eline = func_code(self.error_func).co_firstlineno
             efile = func_code(self.error_func).co_filename
-            self.files[efile] = 1
+            module = inspect.getmodule(self.error_func)
+            self.modules[module] = 1
 
             argcount = func_code(self.error_func).co_argcount - ismethod
             if argcount != 1:
@@ -3016,8 +3010,8 @@ class ParserReflect(object):
             if name == 'p_error': continue
             if isinstance(item,(types.FunctionType,types.MethodType)):
                 line = func_code(item).co_firstlineno
-                file = func_code(item).co_filename
-                p_functions.append((line,file,name,item.__doc__))
+                module = inspect.getmodule(item)
+                p_functions.append((line,module,name,item.__doc__))
 
         # Sort all of the actions by line number
         p_functions.sort()
@@ -3031,9 +3025,10 @@ class ParserReflect(object):
         if len(self.pfuncs) == 0:
             self.log.error("no rules of the form p_rulename are defined")
             self.error = 1
-            return 
-        
-        for line, file, name, doc in self.pfuncs:
+            return
+
+        for line, module, name, doc in self.pfuncs:
+            file = inspect.getsourcefile(module)
             func = self.pdict[name]
             if isinstance(func, types.MethodType):
                 reqargs = 2
@@ -3059,7 +3054,7 @@ class ParserReflect(object):
 
                 # Looks like a valid grammar rule
                 # Mark the file in which defined.
-                self.files[file] = 1
+                self.modules[module] = 1
 
         # Secondary validation step that looks for p_ definitions that are not functions
         # or functions that look like they might be grammar rules.