From f44f4394a82ca8d0e08c58850b4c08c67a981d61 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 09:25:38 +0200 Subject: Refactored formatter mapping to work like the lexer mapping. Thanks to Ilia Choly for the initial pull request. --- tests/test_basic_api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 893fa90c..e1f51d62 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -151,10 +151,10 @@ def test_formatter_public_api(): out = StringIO() # test that every formatter class has the correct public API def verify(formatter, info): - assert len(info) == 4 - assert info[0], "missing formatter name" - assert info[1], "missing formatter aliases" - assert info[3], "missing formatter docstring" + assert len(info) == 5 + assert info[1], "missing formatter name" + assert info[2], "missing formatter aliases" + assert info[4], "missing formatter docstring" if formatter.name == 'Raw tokens': # will not work with Unicode output file @@ -172,7 +172,9 @@ def test_formatter_public_api(): inst.format(ts, out) for formatter, info in formatters.FORMATTERS.items(): - yield verify, formatter, info + fmter = getattr(formatters, formatter) + yield verify, fmter, info + def test_formatter_encodings(): from pygments.formatters import HtmlFormatter @@ -223,7 +225,9 @@ def test_formatter_unicode_handling(): assert type(out) is bytes, '%s: %r' % (formatter, out) for formatter, info in formatters.FORMATTERS.items(): - yield verify, formatter + # this tests the automatic importing as well + fmter = getattr(formatters, formatter) + yield verify, fmter def test_get_formatters(): -- cgit v1.2.1 From 263571018b4057691369c7e8b567f36c66326dd0 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 10:45:31 +0200 Subject: Make each instantiation a single test --- tests/test_basic_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index e1f51d62..b8fb00d6 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -27,10 +27,12 @@ random.shuffle(test_content) test_content = ''.join(test_content) + '\n' -def test_lexer_import_all(): +def test_lexer_instantiate_all(): # instantiate every lexer, to see if the token type defs are correct + def verify(name): + getattr(lexers, name) for x in lexers.LEXERS: - c = getattr(lexers, x)() + yield verify, x def test_lexer_classes(): -- cgit v1.2.1 From 569a560c3875bdc589030413ed8b14d7c285670e Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 11:48:21 +0200 Subject: basic API test: use SkipTest; PEP8 --- tests/test_basic_api.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index b8fb00d6..8998858c 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -42,7 +42,7 @@ def test_lexer_classes(): for attr in 'aliases', 'filenames', 'alias_filenames', 'mimetypes': assert hasattr(cls, attr) assert type(getattr(cls, attr)) is list, \ - "%s: %s attribute wrong" % (cls, attr) + "%s: %s attribute wrong" % (cls, attr) result = cls.analyse_text("abc") assert isinstance(result, float) and 0.0 <= result <= 1.0 result = cls.analyse_text(".abc") @@ -78,7 +78,7 @@ def test_lexer_classes(): assert isinstance(token[1], text_type) txt += token[1] assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \ - (cls.name, test_content, txt) + (cls.name, test_content, txt) for lexer in lexers._iter_lexerclasses(): yield verify, lexer @@ -89,7 +89,8 @@ def test_lexer_options(): def ensure(tokens, output): concatenated = ''.join(token[1] for token in tokens) assert concatenated == output, \ - '%s: %r != %r' % (lexer, concatenated, output) + '%s: %r != %r' % (lexer, concatenated, output) + def verify(cls): inst = cls(stripnl=False) ensure(inst.get_tokens('a\nb'), 'a\nb\n') @@ -98,12 +99,12 @@ def test_lexer_options(): ensure(inst.get_tokens(' \n b\n\n\n'), 'b\n') # some lexers require full lines in input if cls.__name__ not in ( - 'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer', - 'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer', - 'BashSessionLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', - 'PostgresConsoleLexer', 'ElixirConsoleLexer', 'JuliaConsoleLexer', - 'RobotFrameworkLexer', 'DylanConsoleLexer', 'ShellSessionLexer', - 'LiterateIdrisLexer', 'LiterateCryptolLexer'): + 'PythonConsoleLexer', 'RConsoleLexer', 'RubyConsoleLexer', + 'SqliteConsoleLexer', 'MatlabSessionLexer', 'ErlangShellLexer', + 'BashSessionLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', + 'PostgresConsoleLexer', 'ElixirConsoleLexer', 'JuliaConsoleLexer', + 'RobotFrameworkLexer', 'DylanConsoleLexer', 'ShellSessionLexer', + 'LiterateIdrisLexer', 'LiterateCryptolLexer'): inst = cls(ensurenl=False) ensure(inst.get_tokens('a\nb'), 'a\nb') inst = cls(ensurenl=False, stripall=True) @@ -149,9 +150,10 @@ def test_get_lexers(): def test_formatter_public_api(): + # test that every formatter class has the correct public API ts = list(lexers.PythonLexer().get_tokens("def f(): pass")) out = StringIO() - # test that every formatter class has the correct public API + def verify(formatter, info): assert len(info) == 5 assert info[1], "missing formatter name" @@ -165,7 +167,7 @@ def test_formatter_public_api(): try: inst = formatter(opt1="val1") except (ImportError, FontNotFound): - return + raise support.SkipTest try: inst.get_style_defs() except NotImplementedError: @@ -209,7 +211,7 @@ def test_formatter_unicode_handling(): inst = formatter(encoding=None) except (ImportError, FontNotFound): # some dependency or font not installed - return + raise support.SkipTest if formatter.name != 'Raw tokens': out = format(tokens, inst) @@ -246,7 +248,7 @@ def test_get_formatters(): def test_styles(): # minimal style test from pygments.formatters import HtmlFormatter - fmt = HtmlFormatter(style="pastie") + HtmlFormatter(style="pastie") class FiltersTest(unittest.TestCase): -- cgit v1.2.1 From b4e544bcfff8ad69d3aafb54b1060ebb53e71df6 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 11:56:59 +0200 Subject: make test with -v a bit easier on the eyes --- tests/test_basic_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 8998858c..e6b9b325 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -154,7 +154,8 @@ def test_formatter_public_api(): ts = list(lexers.PythonLexer().get_tokens("def f(): pass")) out = StringIO() - def verify(formatter, info): + def verify(formatter): + info = formatters.FORMATTERS[formatter.__name__] assert len(info) == 5 assert info[1], "missing formatter name" assert info[2], "missing formatter aliases" @@ -175,9 +176,9 @@ def test_formatter_public_api(): pass inst.format(ts, out) - for formatter, info in formatters.FORMATTERS.items(): - fmter = getattr(formatters, formatter) - yield verify, fmter, info + for name in formatters.FORMATTERS: + formatter = getattr(formatters, name) + yield verify, formatter def test_formatter_encodings(): -- cgit v1.2.1 From f62d7598287b7f5026637811ed5913b51c649f3f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 11:57:59 +0200 Subject: Don't test lexer classes from arbitrary plugins. --- tests/test_basic_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index e6b9b325..6069f97c 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -80,7 +80,7 @@ def test_lexer_classes(): assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \ (cls.name, test_content, txt) - for lexer in lexers._iter_lexerclasses(): + for lexer in lexers._iter_lexerclasses(plugins=False): yield verify, lexer @@ -110,7 +110,7 @@ def test_lexer_options(): inst = cls(ensurenl=False, stripall=True) ensure(inst.get_tokens('a\nb\n\n'), 'a\nb') - for lexer in lexers._iter_lexerclasses(): + for lexer in lexers._iter_lexerclasses(plugins=False): if lexer.__name__ == 'RawTokenLexer': # this one is special continue -- cgit v1.2.1 From ccc25791eb7bd31d2c90ff52a747973615e087e2 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 20 Sep 2014 12:11:53 +0200 Subject: Sort out Unicode output issues with RTF and image formatters. --- tests/test_basic_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 6069f97c..37d170aa 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -217,7 +217,7 @@ def test_formatter_unicode_handling(): if formatter.name != 'Raw tokens': out = format(tokens, inst) if formatter.unicodeoutput: - assert type(out) is text_type + assert type(out) is text_type, '%s: %r' % (formatter, out) inst = formatter(encoding='utf-8') out = format(tokens, inst) -- cgit v1.2.1 From 900ce2be770ce98e7a93d5e687ae6a0184b2015c Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 4 Oct 2014 11:06:28 -0700 Subject: Fix minor problems noted by regexlint and 'make check' --- tests/test_basic_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 37d170aa..eff49383 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -68,7 +68,9 @@ def test_lexer_classes(): try: tokens = list(inst.get_tokens(test_content)) except KeyboardInterrupt: - raise KeyboardInterrupt('interrupted %s.get_tokens(): test_content=%r' % (cls.__name__, test_content)) + raise KeyboardInterrupt( + 'interrupted %s.get_tokens(): test_content=%r' % + (cls.__name__, test_content)) txt = "" for token in tokens: assert isinstance(token, tuple) -- cgit v1.2.1 From 9a51e6a6df8a56aebede133687e91e519a186122 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 7 Oct 2014 14:10:28 +0200 Subject: Closes #980: fix DeprecationWarnings (mostly due to files closed by __del__) on Py3. Also fix a bunch of other uses of open() to use the with statement. --- tests/test_basic_api.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index eff49383..0beb0171 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -264,11 +264,8 @@ class FiltersTest(unittest.TestCase): for x in filters.FILTERS: lx = lexers.PythonLexer() lx.add_filter(x, **filter_args.get(x, {})) - fp = open(TESTFILE, 'rb') - try: + with open(TESTFILE, 'rb') as fp: text = fp.read().decode('utf-8') - finally: - fp.close() tokens = list(lx.get_tokens(text)) roundtext = ''.join([t[1] for t in tokens]) if x not in ('whitespace', 'keywordcase'): @@ -284,22 +281,16 @@ class FiltersTest(unittest.TestCase): def test_whitespace(self): lx = lexers.PythonLexer() lx.add_filter('whitespace', spaces='%') - fp = open(TESTFILE, 'rb') - try: + with open(TESTFILE, 'rb') as fp: text = fp.read().decode('utf-8') - finally: - fp.close() lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))]) self.assertFalse(' ' in lxtext) def test_keywordcase(self): lx = lexers.PythonLexer() lx.add_filter('keywordcase', case='capitalize') - fp = open(TESTFILE, 'rb') - try: + with open(TESTFILE, 'rb') as fp: text = fp.read().decode('utf-8') - finally: - fp.close() lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))]) self.assertTrue('Def' in lxtext and 'Class' in lxtext) -- cgit v1.2.1 From b8100205450bf23cca7efdf569112391e355ee35 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 7 Oct 2014 16:29:05 +0200 Subject: Closes #1028: fix filters to return Unicode strings --- tests/test_basic_api.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 0beb0171..e0df3447 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -257,18 +257,27 @@ def test_styles(): class FiltersTest(unittest.TestCase): def test_basic(self): - filter_args = { - 'whitespace': {'spaces': True, 'tabs': True, 'newlines': True}, - 'highlight': {'names': ['isinstance', 'lexers', 'x']}, - } - for x in filters.FILTERS: + filters_args = [ + ('whitespace', {'spaces': True, 'tabs': True, 'newlines': True}), + ('whitespace', {'wstokentype': False, 'spaces': True}), + ('highlight', {'names': ['isinstance', 'lexers', 'x']}), + ('codetagify', {'codetags': 'API'}), + ('keywordcase', {'case': 'capitalize'}), + ('raiseonerror', {}), + ('gobble', {'n': 4}), + ('tokenmerge', {}), + ] + for x, args in filters_args: lx = lexers.PythonLexer() - lx.add_filter(x, **filter_args.get(x, {})) + lx.add_filter(x, **args) with open(TESTFILE, 'rb') as fp: text = fp.read().decode('utf-8') tokens = list(lx.get_tokens(text)) + self.assertTrue(all(isinstance(t[1], text_type) + for t in tokens), + '%s filter did not return Unicode' % x) roundtext = ''.join([t[1] for t in tokens]) - if x not in ('whitespace', 'keywordcase'): + if x not in ('whitespace', 'keywordcase', 'gobble'): # these filters change the text self.assertEqual(roundtext, text, "lexer roundtrip with %s filter failed" % x) -- cgit v1.2.1 From c4ab8b3b31e240e4575a7034e166b9d8140c9e2b Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Tue, 7 Oct 2014 16:36:28 +0200 Subject: Closes #908: fix test suite to test formatters with no Unicode output correctly. --- tests/test_basic_api.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index e0df3447..32d675b6 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -16,7 +16,7 @@ from pygments import lexers, formatters, filters, format from pygments.token import _TokenType, Text from pygments.lexer import RegexLexer from pygments.formatters.img import FontNotFound -from pygments.util import text_type, StringIO, xrange, ClassNotFound +from pygments.util import text_type, StringIO, BytesIO, xrange, ClassNotFound import support @@ -154,7 +154,8 @@ def test_get_lexers(): def test_formatter_public_api(): # test that every formatter class has the correct public API ts = list(lexers.PythonLexer().get_tokens("def f(): pass")) - out = StringIO() + string_out = StringIO() + bytes_out = BytesIO() def verify(formatter): info = formatters.FORMATTERS[formatter.__name__] @@ -163,20 +164,21 @@ def test_formatter_public_api(): assert info[2], "missing formatter aliases" assert info[4], "missing formatter docstring" - if formatter.name == 'Raw tokens': - # will not work with Unicode output file - return - try: inst = formatter(opt1="val1") except (ImportError, FontNotFound): raise support.SkipTest + try: inst.get_style_defs() except NotImplementedError: # may be raised by formatters for which it doesn't make sense pass - inst.format(ts, out) + + if formatter.unicodeoutput: + inst.format(ts, string_out) + else: + inst.format(ts, bytes_out) for name in formatters.FORMATTERS: formatter = getattr(formatters, name) -- cgit v1.2.1 From d9dcb23898e9bb97f4494b24f8723e85e8045bc4 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 8 Oct 2014 00:58:47 +0200 Subject: Dont print out all tokens to stdout. --- tests/test_basic_api.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'tests/test_basic_api.py') diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index 32d675b6..7485df1a 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -75,8 +75,6 @@ def test_lexer_classes(): for token in tokens: assert isinstance(token, tuple) assert isinstance(token[0], _TokenType) - if isinstance(token[1], str): - print(repr(token[1])) assert isinstance(token[1], text_type) txt += token[1] assert txt == test_content, "%s lexer roundtrip failed: %r != %r" % \ -- cgit v1.2.1