diff options
-rw-r--r-- | CHANGES | 4 | ||||
-rw-r--r-- | Makefile | 4 | ||||
-rw-r--r-- | pygments/__init__.py | 2 | ||||
-rw-r--r-- | pygments/cmdline.py | 70 | ||||
-rwxr-xr-x | setup.py | 2 | ||||
-rw-r--r-- | tests/test_cmdline.py | 190 | ||||
-rw-r--r-- | tests/test_util.py | 20 |
7 files changed, 194 insertions, 98 deletions
@@ -2,9 +2,9 @@ Pygments changelog ================== Issue numbers refer to the tracker at -<http://bitbucket.org/birkenfeld/pygments-main/issues>, +<https://bitbucket.org/birkenfeld/pygments-main/issues>, pull request numbers to the requests at -<http://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>. +<https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>. Version 2.0.1 @@ -50,7 +50,7 @@ reindent: @$(PYTHON) scripts/reindent.py -r -B . test: - @$(PYTHON) tests/run.py $(TEST) + @$(PYTHON) tests/run.py -d $(TEST) test-coverage: - @$(PYTHON) tests/run.py --with-coverage --cover-package=pygments $(TEST) + @$(PYTHON) tests/run.py -d --with-coverage --cover-package=pygments $(TEST) diff --git a/pygments/__init__.py b/pygments/__init__.py index dc377369..c1ea2af4 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -26,7 +26,7 @@ :license: BSD, see LICENSE for details. """ -__version__ = '2.0.1' +__version__ = '2.1a0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] diff --git a/pygments/cmdline.py b/pygments/cmdline.py index 0f629ece..40c1a599 100644 --- a/pygments/cmdline.py +++ b/pygments/cmdline.py @@ -19,7 +19,7 @@ from pygments import __version__, highlight from pygments.util import ClassNotFound, OptionError, docstring_headline, \ guess_decode, guess_decode_from_terminal, terminal_encoding from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \ - get_lexer_for_filename, find_lexer_class, TextLexer + get_lexer_for_filename, find_lexer_class_for_filename, TextLexer from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter from pygments.formatters import get_all_formatters, get_formatter_by_name, \ get_formatter_for_filename, find_formatter_class, \ @@ -30,7 +30,7 @@ from pygments.styles import get_all_styles, get_style_by_name USAGE = """\ Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>] - [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>] + [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>] %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>] %s -L [<which> ...] @@ -90,6 +90,9 @@ waiting to process the entire file. This only works for stdin, and is intended for streaming input such as you get from 'tail -f'. Example usage: "tail -f sql.log | pygmentize -s -l sql" +The -v option prints a detailed traceback on unhandled exceptions, +which is useful for debugging and bug reports. + The -h option prints this help. The -V option prints the package version. """ @@ -100,7 +103,7 @@ def _parse_options(o_strs): if not o_strs: return opts for o_str in o_strs: - if not o_str: + if not o_str.strip(): continue o_args = o_str.split(',') for o_arg in o_args: @@ -132,7 +135,7 @@ def _parse_filters(f_strs): def _print_help(what, name): try: if what == 'lexer': - cls = find_lexer_class(name) + cls = get_lexer_by_name(name) print("Help on the %s lexer:" % cls.name) print(dedent(cls.__doc__)) elif what == 'formatter': @@ -143,7 +146,7 @@ def _print_help(what, name): cls = find_filter_class(name) print("Help on the %s filter:" % name) print(dedent(cls.__doc__)) - except AttributeError: + except (AttributeError, ValueError): print("%s not found!" % what, file=sys.stderr) @@ -271,13 +274,9 @@ def main_inner(popts, args, usage): # handle ``pygmentize -N`` infn = opts.pop('-N', None) if infn is not None: - try: - lexer = get_lexer_for_filename(infn, **parsed_opts) - except ClassNotFound as err: - lexer = TextLexer() - except OptionError as err: - print('Error:', err, file=sys.stderr) - return 1 + lexer = find_lexer_class_for_filename(infn) + if lexer is None: + lexer = TextLexer print(lexer.aliases[0]) return 0 @@ -387,6 +386,20 @@ def main_inner(popts, args, usage): except ClassNotFound: lexer = TextLexer(**parsed_opts) + else: # -s option needs a lexer with -l + if not lexer: + print('Error: when using -s a lexer has to be selected with -l', + file=sys.stderr) + return 2 + + # process filters + for fname, fopts in F_opts: + try: + lexer.add_filter(fname, **fopts) + except ClassNotFound as err: + print('Error:', err, file=sys.stderr) + return 1 + # select formatter outfn = opts.pop('-o', None) fmter = opts.pop('-f', None) @@ -429,7 +442,7 @@ def main_inner(popts, args, usage): # provide coloring under Windows, if possible if not outfn and sys.platform in ('win32', 'cygwin') and \ - fmter.name in ('Terminal', 'Terminal256'): + fmter.name in ('Terminal', 'Terminal256'): # pragma: no cover # unfortunately colorama doesn't support binary streams on Py3 if sys.version_info > (3,): from pygments.util import UnclosingTextIOWrapper @@ -452,24 +465,12 @@ def main_inner(popts, args, usage): right = escapeinside[1] lexer = LatexEmbeddedLexer(left, right, lexer) - # process filters - for fname, fopts in F_opts: - try: - lexer.add_filter(fname, **fopts) - except ClassNotFound as err: - print('Error:', err, file=sys.stderr) - return 1 - # ... and do it! if '-s' not in opts: # process whole input as per normal... highlight(code, lexer, fmter, outfile) return 0 else: - if not lexer: - print('Error: when using -s a lexer has to be selected with -l', - file=sys.stderr) - return 1 # line by line processing of stdin (eg: for 'tail -f')... try: while 1: @@ -485,7 +486,8 @@ def main_inner(popts, args, usage): highlight(line, lexer, fmter, outfile) if hasattr(outfile, 'flush'): outfile.flush() - except KeyboardInterrupt: + return 0 + except KeyboardInterrupt: # pragma: no cover return 0 @@ -496,7 +498,7 @@ def main(args=sys.argv): usage = USAGE % ((args[0],) * 6) try: - popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs") + popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs") except getopt.GetoptError: print(usage, file=sys.stderr) return 2 @@ -504,6 +506,18 @@ def main(args=sys.argv): try: return main_inner(popts, args, usage) except Exception: + if '-v' in dict(popts): + print(file=sys.stderr) + print('*' * 65, file=sys.stderr) + print('An unhandled exception occurred while highlighting.', + file=sys.stderr) + print('Please report the whole traceback to the issue tracker at', + file=sys.stderr) + print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.', + file=sys.stderr) + print('*' * 65, file=sys.stderr) + print(file=sys.stderr) + raise import traceback info = traceback.format_exception(*sys.exc_info()) msg = info[-1].strip() @@ -513,4 +527,6 @@ def main(args=sys.argv): print(file=sys.stderr) print('*** Error while highlighting:', file=sys.stderr) print(msg, file=sys.stderr) + print('*** If this is a bug you want to report, please rerun with -v.', + file=sys.stderr) return 1 @@ -54,7 +54,7 @@ else: setup( name = 'Pygments', - version = '2.0.1', + version = '2.1a0', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py index 64c4245e..cf935152 100644 --- a/tests/test_cmdline.py +++ b/tests/test_cmdline.py @@ -11,32 +11,44 @@ from __future__ import print_function import io import os +import re import sys import tempfile import unittest -from pygments import highlight, cmdline -from pygments.util import StringIO, BytesIO - import support +from pygments import cmdline, highlight +from pygments.util import BytesIO, StringIO + TESTFILE, TESTDIR = support.location(__file__) +TESTCODE = '''\ +def func(args): + pass +''' -def run_cmdline(*args): +def run_cmdline(*args, **kwds): + saved_stdin = sys.stdin saved_stdout = sys.stdout saved_stderr = sys.stderr if sys.version_info > (3,): + stdin_buffer = BytesIO() stdout_buffer = BytesIO() stderr_buffer = BytesIO() + new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8') new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8') new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8') else: + stdin_buffer = new_stdin = sys.stdin = StringIO() stdout_buffer = new_stdout = sys.stdout = StringIO() stderr_buffer = new_stderr = sys.stderr = StringIO() + new_stdin.write(kwds.get('stdin', '')) + new_stdin.seek(0, 0) try: ret = cmdline.main(["pygmentize"] + list(args)) finally: + sys.stdin = saved_stdin sys.stdout = saved_stdout sys.stderr = saved_stderr new_stdout.flush() @@ -48,88 +60,127 @@ def run_cmdline(*args): class CmdLineTest(unittest.TestCase): + def check_success(self, *cmdline, **kwds): + code, out, err = run_cmdline(*cmdline, **kwds) + self.assertEqual(code, 0) + self.assertEqual(err, '') + return out + + def check_failure(self, *cmdline, **kwds): + expected_code = kwds.pop('code', 1) + code, out, err = run_cmdline(*cmdline, **kwds) + self.assertEqual(code, expected_code) + self.assertEqual(out, '') + return err + + def test_normal(self): + # test that cmdline gives the same output as library api + from pygments.lexers import PythonLexer + from pygments.formatters import HtmlFormatter + filename = TESTFILE + with open(filename, 'rb') as fp: + code = fp.read() + + output = highlight(code, PythonLexer(), HtmlFormatter()) + + o = self.check_success("-lpython", "-fhtml", filename) + self.assertEqual(o, output) + + def test_stdin(self): + o = self.check_success('-lpython', '-fhtml', stdin=TESTCODE) + o = re.sub('<[^>]*>', '', o) + # rstrip is necessary since HTML inserts a \n after the last </div> + self.assertEqual(o.rstrip(), TESTCODE.rstrip()) + + # guess if no lexer given + o = self.check_success('-fhtml', stdin=TESTCODE) + o = re.sub('<[^>]*>', '', o) + # rstrip is necessary since HTML inserts a \n after the last </div> + self.assertEqual(o.rstrip(), TESTCODE.rstrip()) + + def test_outfile(self): + # test that output file works with and without encoding + fd, name = tempfile.mkstemp() + os.close(fd) + for opts in [['-fhtml', '-o', name, TESTFILE], + ['-flatex', '-o', name, TESTFILE], + ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]: + try: + self.check_success(*opts) + finally: + os.unlink(name) + + def test_stream_opt(self): + o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE) + o = re.sub(r'\x1b\[.*?m', '', o) + self.assertEqual(o, TESTCODE) + def test_L_opt(self): - c, o, e = run_cmdline("-L") - self.assertEqual(c, 0) + o = self.check_success("-L") self.assertTrue("Lexers" in o and "Formatters" in o and "Filters" in o and "Styles" in o) - c, o, e = run_cmdline("-L", "lexer") - self.assertEqual(c, 0) + o = self.check_success("-L", "lexer") self.assertTrue("Lexers" in o and "Formatters" not in o) - c, o, e = run_cmdline("-L", "lexers") - self.assertEqual(c, 0) + self.check_success("-L", "lexers") def test_O_opt(self): filename = TESTFILE - c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar", - "-fhtml", filename) - self.assertEqual(c, 0) + o = self.check_success("-Ofull=1,linenos=true,foo=bar", + "-fhtml", filename) self.assertTrue("<html" in o) self.assertTrue('class="linenos"' in o) + # "foobar" is invalid for a bool option + e = self.check_failure("-Ostripnl=foobar", TESTFILE) + self.assertTrue('Error: Invalid value' in e) + e = self.check_failure("-Ostripnl=foobar", "-lpy") + self.assertTrue('Error: Invalid value' in e) + def test_P_opt(self): filename = TESTFILE - c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,", - "-fhtml", filename) - self.assertEqual(c, 0) + o = self.check_success("-Pfull", "-Ptitle=foo, bar=baz=,", + "-fhtml", filename) self.assertTrue("<title>foo, bar=baz=,</title>" in o) def test_F_opt(self): filename = TESTFILE - c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb," - "names=TESTFILE filename", - "-fhtml", filename) - self.assertEqual(c, 0) + o = self.check_success("-Fhighlight:tokentype=Name.Blubb," + "names=TESTFILE filename", + "-fhtml", filename) self.assertTrue('<span class="n-Blubb' in o) def test_H_opt(self): - c, o, e = run_cmdline("-H", "formatter", "html") - self.assertEqual(c, 0) + o = self.check_success("-H", "formatter", "html") self.assertTrue('HTML' in o) + o = self.check_success("-H", "lexer", "python") + self.assertTrue('Python' in o) + o = self.check_success("-H", "filter", "raiseonerror") + self.assertTrue('raiseonerror', o) def test_S_opt(self): - c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1") - self.assertEqual(c, 0) + o = self.check_success("-S", "default", "-f", "html", "-O", "linenos=1") + lines = o.splitlines() + for line in lines: + # every line is for a token class + parts = line.split() + self.assertTrue(parts[0].startswith('.')) + self.assertTrue(parts[1] == '{') + if parts[0] != '.hll': + self.assertTrue(parts[-4] == '}') + self.assertTrue(parts[-3] == '/*') + self.assertTrue(parts[-1] == '*/') + + def test_N_opt(self): + o = self.check_success("-N", "test.py") + self.assertEqual('python\n', o) + o = self.check_success("-N", "test.unknown") + self.assertEqual('text\n', o) def test_invalid_opts(self): for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"), ("-a",), ("-Sst", "-lpy"), ("-H",), - ("-H", "formatter")]: - self.assertTrue(run_cmdline(*opts)[0] == 2) - - def test_normal(self): - # test that cmdline gives the same output as library api - from pygments.lexers import PythonLexer - from pygments.formatters import HtmlFormatter - filename = TESTFILE - with open(filename, 'rb') as fp: - code = fp.read() - - output = highlight(code, PythonLexer(), HtmlFormatter()) - - c, o, e = run_cmdline("-lpython", "-fhtml", filename) - - self.assertEqual(o, output) - self.assertEqual(e, "") - self.assertEqual(c, 0) - - def test_outfile(self): - # test that output file works with and without encoding - fd, name = tempfile.mkstemp() - os.close(fd) - for opts in [['-fhtml', '-o', name, TESTFILE], - ['-flatex', '-o', name, TESTFILE], - ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]: - try: - self.assertEqual(run_cmdline(*opts)[0], 0) - finally: - os.unlink(name) - - def check_failure(self, *cmdline): - c, o, e = run_cmdline(*cmdline) - self.assertEqual(c, 1) - self.assertEqual(o, '') - return e + ("-H", "formatter"), ("-H", "foo", "bar"), ("-s",)]: + self.check_failure(*opts, code=2) def test_errors(self): # input file not found @@ -156,10 +207,23 @@ class CmdLineTest(unittest.TestCase): self.assertTrue('Error: filter \'foo\' not found' in e) def test_exception(self): - # unexpected exception while highlighting - cmdline.highlight = None # override callable + cmdline.highlight = None # override callable to provoke TypeError try: + # unexpected exception while highlighting e = self.check_failure('-lpython', TESTFILE) + self.assertTrue('*** Error while highlighting:' in e) + self.assertTrue('TypeError' in e) + + # same with -v: should reraise the exception + try: + self.check_failure('-lpython', '-v', TESTFILE) + except Exception: + pass + else: + self.fail('exception not reraised') finally: cmdline.highlight = highlight - self.assertTrue('*** Error while highlighting:' in e) + + def test_parse_opts(self): + self.assertEqual(cmdline._parse_options([' ', 'keyonly,key = value ']), + {'keyonly': True, 'key': 'value'}) diff --git a/tests/test_util.py b/tests/test_util.py index e480b503..d6ebd51d 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -10,7 +10,7 @@ import re import unittest -from pygments import util +from pygments import util, console class FakeLexer(object): @@ -40,7 +40,6 @@ class UtilTest(unittest.TestCase): equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2']) raises(util.OptionError, util.get_list_opt, {}, 'a', 1) - def test_docstring_headline(self): def f1(): """ @@ -157,3 +156,20 @@ class UtilTest(unittest.TestCase): # keeps first x = util.duplicates_removed(('a', 'b', 'a')) self.assertEqual(['a', 'b'], x) + + +class ConsoleTest(unittest.TestCase): + + def test_ansiformat(self): + f = console.ansiformat + c = console.codes + all_attrs = f('+*_blue_*+', 'text') + self.assertTrue(c['blue'] in all_attrs and c['blink'] in all_attrs + and c['bold'] in all_attrs and c['underline'] in all_attrs + and c['reset'] in all_attrs) + self.assertRaises(KeyError, f, '*mauve*', 'text') + + def test_functions(self): + self.assertEqual(console.reset_color(), console.codes['reset']) + self.assertEqual(console.colorize('blue', 'text'), + console.codes['blue'] + 'text' + console.codes['reset']) |