Cleanup test modules, more pytest-like testing.

14 files changed, 52 insertions, 64 deletions

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 00000000..a6c6adb3
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,3 @@
+[run]
+include =
+	pygments/*
diff --git a/.hgignore b/.hgignore
index 36b381c8..0fde8b3c 100644
--- a/.hgignore
+++ b/.hgignore
@@ -16,6 +16,6 @@ build/*
 dist/*
 doc/_build
 TAGS
-tests/.coverage
-tests/cover
+.coverage
+htmlcov/
 tests/examplefiles/output
diff --git a/Makefile b/Makefile
index e57e0507..405dd674 100644
--- a/Makefile
+++ b/Makefile
@@ -49,7 +49,7 @@ pylint:
 reindent:
 	@$(PYTHON) scripts/reindent.py -r -B .
 
-TEST = test
+TEST = tests
 
 test:
 	@$(PYTHON) `which py.test` $(TEST)
diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py
index 4c7385a7..0d20fc24 100644
--- a/tests/test_basic_api.py
+++ b/tests/test_basic_api.py
@@ -91,7 +91,7 @@ def test_lexer_options(cls):
     def ensure(tokens, output):
         concatenated = ''.join(token[1] for token in tokens)
         assert concatenated == output, \
-            '%s: %r != %r' % (lexer, concatenated, output)
+            '%s: %r != %r' % (cls, concatenated, output)
 
     inst = cls(stripnl=False)
     ensure(inst.get_tokens('a\nb'), 'a\nb\n')
@@ -111,18 +111,15 @@ def test_lexer_options(cls):
 
 def test_get_lexers():
     # test that the lexers functions work
-    def verify(func, args):
-        x = func(opt='val', *args)
-        assert isinstance(x, lexers.PythonLexer)
-        assert x.options["opt"] == "val"
-
     for func, args in [(lexers.get_lexer_by_name, ("python",)),
                        (lexers.get_lexer_for_filename, ("test.py",)),
                        (lexers.get_lexer_for_mimetype, ("text/x-python",)),
                        (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)),
                        (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
                        ]:
-        verify(func, args)
+        x = func(opt='val', *args)
+        assert isinstance(x, lexers.PythonLexer)
+        assert x.options["opt"] == "val"
 
     for cls, (_, lname, aliases, _, mimetypes) in lexers.LEXERS.items():
         assert cls == lexers.find_lexer_class(lname).__name__
diff --git a/tests/test_cfm.py b/tests/test_cfm.py
index 0ff1b167..401ac78c 100644
--- a/tests/test_cfm.py
+++ b/tests/test_cfm.py
@@ -8,7 +8,6 @@
 """
 
 import unittest
-import os
 
 from pygments.token import Token
 from pygments.lexers import ColdfusionHtmlLexer
diff --git a/tests/test_clexer.py b/tests/test_clexer.py
index 5095b797..8841bf3e 100644
--- a/tests/test_clexer.py
+++ b/tests/test_clexer.py
@@ -8,7 +8,6 @@
 """
 
 import unittest
-import os
 import textwrap
 
 from pygments.token import Text, Number, Token
diff --git a/tests/test_crystal.py b/tests/test_crystal.py
index 9a1588f2..efbab68d 100644
--- a/tests/test_crystal.py
+++ b/tests/test_crystal.py
@@ -10,7 +10,7 @@
 from __future__ import unicode_literals
 import unittest
 
-from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
+from pygments.token import Text, Operator, Keyword, Name, String, \
     Number, Punctuation, Error
 from pygments.lexers import CrystalLexer
 
@@ -122,7 +122,7 @@ class CrystalTest(unittest.TestCase):
             (Text, '\n')
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-    
+
     def testArrayAccess(self):
         fragment = '[5][5]?\n'
         tokens = [
diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py
index 835f3b1c..5251f2ff 100644
--- a/tests/test_examplefiles.py
+++ b/tests/test_examplefiles.py
@@ -20,7 +20,7 @@ from pygments.lexers import get_lexer_for_filename, get_lexer_by_name
 from pygments.token import Error
 from pygments.util import ClassNotFound
 
-STORE_OUTPUT = False
+STORE_OUTPUT = True
 
 STATS = {}
 
@@ -34,7 +34,7 @@ BAD_FILES_FOR_JYTHON = ('Object.st', 'all.nit', 'genclass.clj',
 
 
 def get_example_files():
-    # TODO: move this to a fixture
+    # TODO: move stats to a fixture
     # global STATS
     # STATS = {}
     outdir = os.path.join(TESTDIR, 'examplefiles', 'output')
@@ -53,28 +53,7 @@ def get_example_files():
             continue
 
         print(absfn)
-        with open(absfn, 'rb') as f:
-            code = f.read()
-        try:
-            code = code.decode('utf-8')
-        except UnicodeError:
-            code = code.decode('latin1')
-
-        lx = None
-        if '_' in fn:
-            try:
-                lx = get_lexer_by_name(fn.split('_')[0])
-            except ClassNotFound:
-                pass
-        if lx is None:
-            try:
-                lx = get_lexer_for_filename(absfn, code=code)
-            except ClassNotFound:
-                raise AssertionError('file %r has no registered extension, '
-                                     'nor is of the form <lexer>_filename '
-                                     'for overriding, thus no lexer found.'
-                                     % fn)
-        yield lx, fn
+        yield fn
 
     # N = 7
     # stats = list(STATS.items())
@@ -89,14 +68,34 @@ def get_example_files():
     #     print('%-30s  %6d chars  %8.2f ms  %7.3f ms/char' % ((fn,) + t))
 
 
-@pytest.mark.parametrize('what', get_example_files())
-def test_examplefile(what):
-    lx, fn = what
-    if os.name == 'java' and fn in BAD_FILES_FOR_JYTHON:
-        pytest.skip('%s is a known bad file on Jython' % fn)
-    absfn = os.path.join(TESTDIR, 'examplefiles', fn)
-    with open(absfn, 'rb') as fp:
-        text = fp.read()
+@pytest.mark.parametrize('filename', get_example_files())
+def test_examplefile(filename):
+    if os.name == 'java' and filename in BAD_FILES_FOR_JYTHON:
+        pytest.skip('%s is a known bad file on Jython' % filename)
+
+    absfn = os.path.join(TESTDIR, 'examplefiles', filename)
+    with open(absfn, 'rb') as f:
+        text = f.read()
+    try:
+        utext = text.decode('utf-8')
+    except UnicodeError:
+        utext = text.decode('latin1')
+
+    lx = None
+    if '_' in filename:
+        try:
+            lx = get_lexer_by_name(filename.split('_')[0])
+        except ClassNotFound:
+            pass
+    if lx is None:
+        try:
+            lx = get_lexer_for_filename(absfn, code=utext)
+        except ClassNotFound:
+            raise AssertionError('file %r has no registered extension, '
+                                 'nor is of the form <lexer>_filename '
+                                 'for overriding, thus no lexer found.'
+                                 % filename)
+
     text = text.replace(b'\r\n', b'\n')
     text = text.strip(b'\n') + b'\n'
     try:
@@ -126,7 +125,7 @@ def test_examplefile(what):
     # check output against previous run if enabled
     if STORE_OUTPUT:
         # no previous output -- store it
-        outfn = os.path.join(TESTDIR, 'examplefiles', 'output', fn)
+        outfn = os.path.join(TESTDIR, 'examplefiles', 'output', filename)
         if not os.path.isfile(outfn):
             with open(outfn, 'wb') as fp:
                 pickle.dump(tokens, fp)
diff --git a/tests/test_kotlin.py b/tests/test_kotlin.py
index 7c733ad9..bc27908a 100644
--- a/tests/test_kotlin.py
+++ b/tests/test_kotlin.py
@@ -9,15 +9,16 @@
 
 import unittest
 
-from pygments.token import Text, Name, Operator, Keyword, Number, Punctuation, String
+from pygments.token import Text, Name, Keyword, Punctuation, String
 from pygments.lexers import KotlinLexer
 
+
 class KotlinTest(unittest.TestCase):
 
     def setUp(self):
         self.lexer = KotlinLexer()
         self.maxDiff = None
-    
+
     def testCanCopeWithBackTickNamesInFunctions(self):
         fragment = u'fun `wo bble`'
         tokens = [
@@ -37,7 +38,7 @@ class KotlinTest(unittest.TestCase):
             (Text, u'\n')
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-    
+
     def testCanCopeWithDestructuring(self):
         fragment = u'val (a, b) = '
         tokens = [
@@ -55,7 +56,7 @@ class KotlinTest(unittest.TestCase):
             (Text, u'\n')
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-    
+
     def testCanCopeGenericsInDestructuring(self):
         fragment = u'val (a: List<Something>, b: Set<Wobble>) ='
         tokens = [
@@ -122,7 +123,7 @@ class KotlinTest(unittest.TestCase):
     def testShouldCopeWithMultilineComments(self):
         fragment = u'"""\nthis\nis\na\ncomment"""'
         tokens = [
-            (String, u'"""\nthis\nis\na\ncomment"""'), 
+            (String, u'"""\nthis\nis\na\ncomment"""'),
             (Text, u'\n')
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py
index aee7db66..2a5fbe21 100644
--- a/tests/test_objectiveclexer.py
+++ b/tests/test_objectiveclexer.py
@@ -8,7 +8,6 @@
 """
 
 import unittest
-import os
 
 from pygments.token import Token
 from pygments.lexers import ObjectiveCLexer
diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py
index 102f0a9f..9845d9d6 100644
--- a/tests/test_perllexer.py
+++ b/tests/test_perllexer.py
@@ -124,10 +124,6 @@ class RunawayRegexTest(unittest.TestCase):
         self.assert_single_token(r's<aaa>', String.Regex)
         self.assert_fast_tokenization('s<' + '\\'*999)
 
-    def test_substitution_with_angle_bracket(self):
-        self.assert_single_token(r's<aaa>', String.Regex)
-        self.assert_fast_tokenization('s<' + '\\'*999)
-
     def test_substitution_with_square_bracket(self):
         self.assert_single_token(r's[aaa]', String.Regex)
         self.assert_fast_tokenization('s[' + '\\'*999)
@@ -154,4 +150,3 @@ class RunawayRegexTest(unittest.TestCase):
         self.assert_tokens(['require', ' ', 'Foo'], [Keyword, Text, Name.Namespace])
         self.assert_tokens(['require', '  ', 'Foo::Bar'], [Keyword, Text, Name.Namespace])
         self.assert_tokens(['require', ' ', '"Foo/Bar.pm"'], [Keyword, Text, String])
-
diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py
index 0ea221a1..e6212d65 100644
--- a/tests/test_qbasiclexer.py
+++ b/tests/test_qbasiclexer.py
@@ -7,8 +7,6 @@
     :license: BSD, see LICENSE for details.
 """
 
-import glob
-import os
 import unittest
 
 from pygments.token import Token
diff --git a/tests/test_smarty.py b/tests/test_smarty.py
index e1e079d9..a9bce11c 100644
--- a/tests/test_smarty.py
+++ b/tests/test_smarty.py
@@ -9,7 +9,7 @@
 
 import unittest
 
-from pygments.token import Operator, Number, Text, Token
+from pygments.token import Token
 from pygments.lexers import SmartyLexer
 
 
@@ -37,4 +37,3 @@ class SmartyTest(unittest.TestCase):
             (Token.Other, u'\n'),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
-
diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py
index 453dd61f..57c2b61f 100644
--- a/tests/test_textfmts.py
+++ b/tests/test_textfmts.py
@@ -9,7 +9,7 @@
 
 import unittest
 
-from pygments.token import Operator, Number, Text, Token
+from pygments.token import Token
 from pygments.lexers.textfmts import HttpLexer
 
 
@@ -38,4 +38,3 @@ class RubyTest(unittest.TestCase):
         ]
         self.assertEqual(
             tokens, list(self.lexer.get_tokens(fragment))[-len(tokens):])
-