Test improvements for Python >3.4 and PyPy

2 files changed, 63 insertions, 41 deletions

diff --git a/test/testlex.py b/test/testlex.py
index e436781..9b2cc90 100755
--- a/test/testlex.py
+++ b/test/testlex.py
@@ -8,8 +8,8 @@ except ImportError:
 
 import sys
 import os
-import imp
 import warnings
+import platform
 
 sys.path.insert(0,"..")
 sys.tracebacklimit = 0
@@ -21,7 +21,11 @@ def make_pymodule_path(filename):
     file = os.path.basename(filename)
     mod, ext = os.path.splitext(file)
 
-    if sys.hexversion >= 0x3020000:
+    if sys.hexversion >= 0x3040000:
+        import importlib.util
+        fullpath = importlib.util.cache_from_source(filename, ext=='.pyc')
+    elif sys.hexversion >= 0x3020000:
+        import imp
         modname = mod+"."+imp.get_tag()+ext
         fullpath = os.path.join(path,'__pycache__',modname)
     else:
@@ -34,7 +38,15 @@ def pymodule_out_exists(filename):
 def pymodule_out_remove(filename):
     os.remove(make_pymodule_path(filename))
 
-def check_expected(result, expected):
+def implementation():
+    if platform.system().startswith("Java"):
+        return "Jython"
+    elif hasattr(sys, "pypy_version_info"):
+        return "PyPy"
+    else:
+        return "CPython"
+
+def check_expected(result, expected, contains=False):
     if sys.version_info[0] >= 3:
         if isinstance(result,str):
             result = result.encode('ascii')
@@ -47,8 +59,12 @@ def check_expected(result, expected):
         return False
 
     for rline,eline in zip(resultlines,expectedlines):
-        if not rline.endswith(eline):
-            return False
+        if contains:
+            if eline not in rline:
+                return False
+        else:
+            if not rline.endswith(eline):
+                return False
     return True
 
 def run_import(module):
@@ -139,7 +155,8 @@ class LexErrorWarningTests(unittest.TestCase):
         self.assertRaises(SyntaxError,run_import,"lex_re1")
         result = sys.stderr.getvalue()
         self.assert_(check_expected(result,
-                                    "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n"))
+                                    "Invalid regular expression for rule 't_NUMBER'. unbalanced parenthesis\n",
+                                    contains=True))
 
     def test_lex_re2(self):
         self.assertRaises(SyntaxError,run_import,"lex_re2")
@@ -150,9 +167,14 @@ class LexErrorWarningTests(unittest.TestCase):
     def test_lex_re3(self):
         self.assertRaises(SyntaxError,run_import,"lex_re3")
         result = sys.stderr.getvalue()
+#        self.assert_(check_expected(result,
+#                                    "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
+#                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
+
         self.assert_(check_expected(result,
                                     "Invalid regular expression for rule 't_POUND'. unbalanced parenthesis\n"
-                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n"))
+                                    "Make sure '#' in rule 't_POUND' is escaped with '\\#'\n",
+                                    contains=True))
 
     def test_lex_rule1(self):
         self.assertRaises(SyntaxError,run_import,"lex_rule1")
@@ -318,6 +340,8 @@ class LexBuildOptionTests(unittest.TestCase):
                                     "(NUMBER,3,1,0)\n"
                                     "(PLUS,'+',1,1)\n"
                                     "(NUMBER,4,1,2)\n"))
+
+    @unittest.skipUnless(implementation() == 'CPython', 'CPython only')
     def test_lex_optimize(self):
         try:
             os.remove("lextab.py")
@@ -373,6 +397,7 @@ class LexBuildOptionTests(unittest.TestCase):
         except OSError:
             pass
 
+    @unittest.skipUnless(implementation() == 'CPython', 'CPython only')
     def test_lex_optimize2(self):
         try:
             os.remove("opt2tab.py")
@@ -424,6 +449,7 @@ class LexBuildOptionTests(unittest.TestCase):
         except OSError:
             pass
 
+    @unittest.skipUnless(implementation() == 'CPython', 'CPython only')
     def test_lex_optimize3(self):
         try:
             shutil.rmtree("lexdir")
@@ -464,6 +490,7 @@ class LexBuildOptionTests(unittest.TestCase):
         except OSError:
             pass
 
+    @unittest.skipUnless(implementation() == 'CPython', 'CPython only')
     def test_lex_opt_alias(self):
         try:
             os.remove("aliastab.py")
@@ -542,21 +569,22 @@ class LexBuildOptionTests(unittest.TestCase):
 
         self.assert_(os.path.exists("manytab.py"))
 
-        p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
-                             stdout=subprocess.PIPE)
-        result = p.stdout.read()
-        self.assert_(check_expected(result,
-                                    "(TOK34,'TOK34:',1,0)\n"
-                                    "(TOK143,'TOK143:',1,7)\n"
-                                    "(TOK269,'TOK269:',1,15)\n"
-                                    "(TOK372,'TOK372:',1,23)\n"
-                                    "(TOK452,'TOK452:',1,31)\n"
-                                    "(TOK561,'TOK561:',1,39)\n"
-                                    "(TOK999,'TOK999:',1,47)\n"
-                                    ))
-
-        self.assert_(pymodule_out_exists("manytab.pyo"))
-        pymodule_out_remove("manytab.pyo")
+        if implementation() == 'CPython':
+            p = subprocess.Popen([sys.executable,'-O','lex_many_tokens.py'],
+                                 stdout=subprocess.PIPE)
+            result = p.stdout.read()
+            self.assert_(check_expected(result,
+                                        "(TOK34,'TOK34:',1,0)\n"
+                                        "(TOK143,'TOK143:',1,7)\n"
+                                        "(TOK269,'TOK269:',1,15)\n"
+                                        "(TOK372,'TOK372:',1,23)\n"
+                                        "(TOK452,'TOK452:',1,31)\n"
+                                        "(TOK561,'TOK561:',1,39)\n"
+                                        "(TOK999,'TOK999:',1,47)\n"
+                                        ))
+
+            self.assert_(pymodule_out_exists("manytab.pyo"))
+            pymodule_out_remove("manytab.pyo")
         try:
             os.remove("manytab.py")
         except OSError:
diff --git a/test/testyacc.py b/test/testyacc.py
index 27d0f92..f8f5f4f 100644
--- a/test/testyacc.py
+++ b/test/testyacc.py
@@ -10,19 +10,23 @@ import sys
 import os
 import warnings
 import re
+import platform
 
 sys.path.insert(0,"..")
 sys.tracebacklimit = 0
 
 import ply.yacc
-import imp
 
 def make_pymodule_path(filename):
     path = os.path.dirname(filename)
     file = os.path.basename(filename)
     mod, ext = os.path.splitext(file)
 
-    if sys.hexversion >= 0x3020000:
+    if sys.hexversion >= 0x3040000:
+        import importlib.util
+        fullpath = importlib.util.cache_from_source(filename, ext=='.pyc')
+    elif sys.hexversion >= 0x3020000:
+        import imp
         modname = mod+"."+imp.get_tag()+ext
         fullpath = os.path.join(path,'__pycache__',modname)
     else:
@@ -35,23 +39,13 @@ def pymodule_out_exists(filename):
 def pymodule_out_remove(filename):
     os.remove(make_pymodule_path(filename))
 
-# Old implementation (not safe for Python 3.3)
-def check_expected(result,expected):
-    resultlines = []
-    for line in result.splitlines():
-        if line.startswith("WARNING: "):
-            line = line[9:]
-        elif line.startswith("ERROR: "):
-            line = line[7:]
-        resultlines.append(line)
-
-    expectedlines = expected.splitlines()
-    if len(resultlines) != len(expectedlines):
-        return False
-    for rline,eline in zip(resultlines,expectedlines):
-        if not rline.endswith(eline):
-            return False
-    return True
+def implementation():
+    if platform.system().startswith("Java"):
+        return "Jython"
+    elif hasattr(sys, "pypy_version_info"):
+        return "PyPy"
+    else:
+        return "CPython"
 
 # Check the output to see if it contains all of a set of expected output lines.
 # This alternate implementation looks weird, but is needed to properly handle