From 65873fba7b51171e35d4496fdf5bf38f54285dfa Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Mon, 6 May 2019 16:18:56 +0200 Subject: Cleanup test modules, more pytest-like testing. --- .coveragerc | 3 ++ .hgignore | 4 +-- Makefile | 2 +- tests/test_basic_api.py | 11 +++----- tests/test_cfm.py | 1 - tests/test_clexer.py | 1 - tests/test_crystal.py | 4 +-- tests/test_examplefiles.py | 65 +++++++++++++++++++++---------------------- tests/test_kotlin.py | 11 ++++---- tests/test_objectiveclexer.py | 1 - tests/test_perllexer.py | 5 ---- tests/test_qbasiclexer.py | 2 -- tests/test_smarty.py | 3 +- tests/test_textfmts.py | 3 +- 14 files changed, 52 insertions(+), 64 deletions(-) create mode 100644 .coveragerc diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 00000000..a6c6adb3 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,3 @@ +[run] +include = + pygments/* diff --git a/.hgignore b/.hgignore index 36b381c8..0fde8b3c 100644 --- a/.hgignore +++ b/.hgignore @@ -16,6 +16,6 @@ build/* dist/* doc/_build TAGS -tests/.coverage -tests/cover +.coverage +htmlcov/ tests/examplefiles/output diff --git a/Makefile b/Makefile index e57e0507..405dd674 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ pylint: reindent: @$(PYTHON) scripts/reindent.py -r -B . -TEST = test +TEST = tests test: @$(PYTHON) `which py.test` $(TEST) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 4c7385a7..0d20fc24 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -91,7 +91,7 @@ def test_lexer_options(cls): def ensure(tokens, output): concatenated = ''.join(token[1] for token in tokens) assert concatenated == output, \ - '%s: %r != %r' % (lexer, concatenated, output) + '%s: %r != %r' % (cls, concatenated, output) inst = cls(stripnl=False) ensure(inst.get_tokens('a\nb'), 'a\nb\n') @@ -111,18 +111,15 @@ def test_lexer_options(cls): def test_get_lexers(): # test that the lexers functions work - def verify(func, args): - x = func(opt='val', *args) - assert isinstance(x, lexers.PythonLexer) - assert x.options["opt"] == "val" - for func, args in [(lexers.get_lexer_by_name, ("python",)), (lexers.get_lexer_for_filename, ("test.py",)), (lexers.get_lexer_for_mimetype, ("text/x-python",)), (lexers.guess_lexer, ("#!/usr/bin/python -O\nprint",)), (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>")) ]: - verify(func, args) + x = func(opt='val', *args) + assert isinstance(x, lexers.PythonLexer) + assert x.options["opt"] == "val" for cls, (_, lname, aliases, _, mimetypes) in lexers.LEXERS.items(): assert cls == lexers.find_lexer_class(lname).__name__ diff --git a/tests/test_cfm.py b/tests/test_cfm.py index 0ff1b167..401ac78c 100644 --- a/tests/test_cfm.py +++ b/tests/test_cfm.py @@ -8,7 +8,6 @@ """ import unittest -import os from pygments.token import Token from pygments.lexers import ColdfusionHtmlLexer diff --git a/tests/test_clexer.py b/tests/test_clexer.py index 5095b797..8841bf3e 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -8,7 +8,6 @@ """ import unittest -import os import textwrap from pygments.token import Text, Number, Token diff --git a/tests/test_crystal.py b/tests/test_crystal.py index 9a1588f2..efbab68d 100644 --- a/tests/test_crystal.py +++ b/tests/test_crystal.py @@ -10,7 +10,7 @@ from __future__ import unicode_literals import unittest -from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ +from pygments.token import Text, Operator, Keyword, Name, String, \ Number, Punctuation, Error from pygments.lexers import CrystalLexer @@ -122,7 +122,7 @@ class CrystalTest(unittest.TestCase): (Text, '\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) - + def testArrayAccess(self): fragment = '[5][5]?\n' tokens = [ diff --git a/tests/test_examplefiles.py b/tests/test_examplefiles.py index 835f3b1c..5251f2ff 100644 --- a/tests/test_examplefiles.py +++ b/tests/test_examplefiles.py @@ -20,7 +20,7 @@ from pygments.lexers import get_lexer_for_filename, get_lexer_by_name from pygments.token import Error from pygments.util import ClassNotFound -STORE_OUTPUT = False +STORE_OUTPUT = True STATS = {} @@ -34,7 +34,7 @@ BAD_FILES_FOR_JYTHON = ('Object.st', 'all.nit', 'genclass.clj', def get_example_files(): - # TODO: move this to a fixture + # TODO: move stats to a fixture # global STATS # STATS = {} outdir = os.path.join(TESTDIR, 'examplefiles', 'output') @@ -53,28 +53,7 @@ def get_example_files(): continue print(absfn) - with open(absfn, 'rb') as f: - code = f.read() - try: - code = code.decode('utf-8') - except UnicodeError: - code = code.decode('latin1') - - lx = None - if '_' in fn: - try: - lx = get_lexer_by_name(fn.split('_')[0]) - except ClassNotFound: - pass - if lx is None: - try: - lx = get_lexer_for_filename(absfn, code=code) - except ClassNotFound: - raise AssertionError('file %r has no registered extension, ' - 'nor is of the form _filename ' - 'for overriding, thus no lexer found.' - % fn) - yield lx, fn + yield fn # N = 7 # stats = list(STATS.items()) @@ -89,14 +68,34 @@ def get_example_files(): # print('%-30s %6d chars %8.2f ms %7.3f ms/char' % ((fn,) + t)) -@pytest.mark.parametrize('what', get_example_files()) -def test_examplefile(what): - lx, fn = what - if os.name == 'java' and fn in BAD_FILES_FOR_JYTHON: - pytest.skip('%s is a known bad file on Jython' % fn) - absfn = os.path.join(TESTDIR, 'examplefiles', fn) - with open(absfn, 'rb') as fp: - text = fp.read() +@pytest.mark.parametrize('filename', get_example_files()) +def test_examplefile(filename): + if os.name == 'java' and filename in BAD_FILES_FOR_JYTHON: + pytest.skip('%s is a known bad file on Jython' % filename) + + absfn = os.path.join(TESTDIR, 'examplefiles', filename) + with open(absfn, 'rb') as f: + text = f.read() + try: + utext = text.decode('utf-8') + except UnicodeError: + utext = text.decode('latin1') + + lx = None + if '_' in filename: + try: + lx = get_lexer_by_name(filename.split('_')[0]) + except ClassNotFound: + pass + if lx is None: + try: + lx = get_lexer_for_filename(absfn, code=utext) + except ClassNotFound: + raise AssertionError('file %r has no registered extension, ' + 'nor is of the form _filename ' + 'for overriding, thus no lexer found.' + % filename) + text = text.replace(b'\r\n', b'\n') text = text.strip(b'\n') + b'\n' try: @@ -126,7 +125,7 @@ def test_examplefile(what): # check output against previous run if enabled if STORE_OUTPUT: # no previous output -- store it - outfn = os.path.join(TESTDIR, 'examplefiles', 'output', fn) + outfn = os.path.join(TESTDIR, 'examplefiles', 'output', filename) if not os.path.isfile(outfn): with open(outfn, 'wb') as fp: pickle.dump(tokens, fp) diff --git a/tests/test_kotlin.py b/tests/test_kotlin.py index 7c733ad9..bc27908a 100644 --- a/tests/test_kotlin.py +++ b/tests/test_kotlin.py @@ -9,15 +9,16 @@ import unittest -from pygments.token import Text, Name, Operator, Keyword, Number, Punctuation, String +from pygments.token import Text, Name, Keyword, Punctuation, String from pygments.lexers import KotlinLexer + class KotlinTest(unittest.TestCase): def setUp(self): self.lexer = KotlinLexer() self.maxDiff = None - + def testCanCopeWithBackTickNamesInFunctions(self): fragment = u'fun `wo bble`' tokens = [ @@ -37,7 +38,7 @@ class KotlinTest(unittest.TestCase): (Text, u'\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) - + def testCanCopeWithDestructuring(self): fragment = u'val (a, b) = ' tokens = [ @@ -55,7 +56,7 @@ class KotlinTest(unittest.TestCase): (Text, u'\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) - + def testCanCopeGenericsInDestructuring(self): fragment = u'val (a: List, b: Set) =' tokens = [ @@ -122,7 +123,7 @@ class KotlinTest(unittest.TestCase): def testShouldCopeWithMultilineComments(self): fragment = u'"""\nthis\nis\na\ncomment"""' tokens = [ - (String, u'"""\nthis\nis\na\ncomment"""'), + (String, u'"""\nthis\nis\na\ncomment"""'), (Text, u'\n') ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py index aee7db66..2a5fbe21 100644 --- a/tests/test_objectiveclexer.py +++ b/tests/test_objectiveclexer.py @@ -8,7 +8,6 @@ """ import unittest -import os from pygments.token import Token from pygments.lexers import ObjectiveCLexer diff --git a/tests/test_perllexer.py b/tests/test_perllexer.py index 102f0a9f..9845d9d6 100644 --- a/tests/test_perllexer.py +++ b/tests/test_perllexer.py @@ -124,10 +124,6 @@ class RunawayRegexTest(unittest.TestCase): self.assert_single_token(r's', String.Regex) self.assert_fast_tokenization('s<' + '\\'*999) - def test_substitution_with_angle_bracket(self): - self.assert_single_token(r's', String.Regex) - self.assert_fast_tokenization('s<' + '\\'*999) - def test_substitution_with_square_bracket(self): self.assert_single_token(r's[aaa]', String.Regex) self.assert_fast_tokenization('s[' + '\\'*999) @@ -154,4 +150,3 @@ class RunawayRegexTest(unittest.TestCase): self.assert_tokens(['require', ' ', 'Foo'], [Keyword, Text, Name.Namespace]) self.assert_tokens(['require', ' ', 'Foo::Bar'], [Keyword, Text, Name.Namespace]) self.assert_tokens(['require', ' ', '"Foo/Bar.pm"'], [Keyword, Text, String]) - diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py index 0ea221a1..e6212d65 100644 --- a/tests/test_qbasiclexer.py +++ b/tests/test_qbasiclexer.py @@ -7,8 +7,6 @@ :license: BSD, see LICENSE for details. """ -import glob -import os import unittest from pygments.token import Token diff --git a/tests/test_smarty.py b/tests/test_smarty.py index e1e079d9..a9bce11c 100644 --- a/tests/test_smarty.py +++ b/tests/test_smarty.py @@ -9,7 +9,7 @@ import unittest -from pygments.token import Operator, Number, Text, Token +from pygments.token import Token from pygments.lexers import SmartyLexer @@ -37,4 +37,3 @@ class SmartyTest(unittest.TestCase): (Token.Other, u'\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) - diff --git a/tests/test_textfmts.py b/tests/test_textfmts.py index 453dd61f..57c2b61f 100644 --- a/tests/test_textfmts.py +++ b/tests/test_textfmts.py @@ -9,7 +9,7 @@ import unittest -from pygments.token import Operator, Number, Text, Token +from pygments.token import Token from pygments.lexers.textfmts import HttpLexer @@ -38,4 +38,3 @@ class RubyTest(unittest.TestCase): ] self.assertEqual( tokens, list(self.lexer.get_tokens(fragment))[-len(tokens):]) - -- cgit v1.2.1