(null) merge with stable

7 files changed, 194 insertions, 98 deletions

diff --git a/CHANGES b/CHANGES
index 7031852e..92110efd 100644
--- a/CHANGES
+++ b/CHANGES
@@ -2,9 +2,9 @@ Pygments changelog
 ==================
 
 Issue numbers refer to the tracker at
-<http://bitbucket.org/birkenfeld/pygments-main/issues>,
+<https://bitbucket.org/birkenfeld/pygments-main/issues>,
 pull request numbers to the requests at
-<http://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
+<https://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
 
 Version 2.0.1
diff --git a/Makefile b/Makefile
index a5741ffa..65ce0769 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ reindent:
 	@$(PYTHON) scripts/reindent.py -r -B .
 
 test:
-	@$(PYTHON) tests/run.py $(TEST)
+	@$(PYTHON) tests/run.py -d $(TEST)
 
 test-coverage:
-	@$(PYTHON) tests/run.py --with-coverage --cover-package=pygments $(TEST)
+	@$(PYTHON) tests/run.py -d --with-coverage --cover-package=pygments $(TEST)
diff --git a/pygments/__init__.py b/pygments/__init__.py
index dc377369..c1ea2af4 100644
--- a/pygments/__init__.py
+++ b/pygments/__init__.py
@@ -26,7 +26,7 @@
     :license: BSD, see LICENSE for details.
 """
 
-__version__ = '2.0.1'
+__version__ = '2.1a0'
 __docformat__ = 'restructuredtext'
 
 __all__ = ['lex', 'format', 'highlight']
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 0f629ece..40c1a599 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -19,7 +19,7 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    get_lexer_for_filename, find_lexer_class_for_filename, TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
@@ -30,7 +30,7 @@ from pygments.styles import get_all_styles, get_style_by_name
 
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-          [-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
+          [-O <options>] [-P <option=value>] [-s] [-v] [-o <outfile>] [<infile>]
 
        %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
        %s -L [<which> ...]
@@ -90,6 +90,9 @@ waiting to process the entire file.  This only works for stdin, and
 is intended for streaming input such as you get from 'tail -f'.
 Example usage: "tail -f sql.log | pygmentize -s -l sql"
 
+The -v option prints a detailed traceback on unhandled exceptions,
+which is useful for debugging and bug reports.
+
 The -h option prints this help.
 The -V option prints the package version.
 """
@@ -100,7 +103,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -132,7 +135,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -143,7 +146,7 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
 
 
@@ -271,13 +274,9 @@ def main_inner(popts, args, usage):
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -387,6 +386,20 @@ def main_inner(popts, args, usage):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    else:  # -s option needs a lexer with -l
+        if not lexer:
+            print('Error: when using -s a lexer has to be selected with -l',
+                  file=sys.stderr)
+            return 2
+
+    # process filters
+    for fname, fopts in F_opts:
+        try:
+            lexer.add_filter(fname, **fopts)
+        except ClassNotFound as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
     # select formatter
     outfn = opts.pop('-o', None)
     fmter = opts.pop('-f', None)
@@ -429,7 +442,7 @@ def main_inner(popts, args, usage):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
             from pygments.util import UnclosingTextIOWrapper
@@ -452,24 +465,12 @@ def main_inner(popts, args, usage):
         right = escapeinside[1]
         lexer = LatexEmbeddedLexer(left, right, lexer)
 
-    # process filters
-    for fname, fopts in F_opts:
-        try:
-            lexer.add_filter(fname, **fopts)
-        except ClassNotFound as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
-
     # ... and do it!
     if '-s' not in opts:
         # process whole input as per normal...
         highlight(code, lexer, fmter, outfile)
         return 0
     else:
-        if not lexer:
-            print('Error: when using -s a lexer has to be selected with -l',
-                  file=sys.stderr)
-            return 1
         # line by line processing of stdin (eg: for 'tail -f')...
         try:
             while 1:
@@ -485,7 +486,8 @@ def main_inner(popts, args, usage):
                 highlight(line, lexer, fmter, outfile)
                 if hasattr(outfile, 'flush'):
                     outfile.flush()
-        except KeyboardInterrupt:
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
             return 0
 
 
@@ -496,7 +498,7 @@ def main(args=sys.argv):
     usage = USAGE % ((args[0],) * 6)
 
     try:
-        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
+        popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:vhVHgs")
     except getopt.GetoptError:
         print(usage, file=sys.stderr)
         return 2
@@ -504,6 +506,18 @@ def main(args=sys.argv):
     try:
         return main_inner(popts, args, usage)
     except Exception:
+        if '-v' in dict(popts):
+            print(file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print('An unhandled exception occurred while highlighting.',
+                  file=sys.stderr)
+            print('Please report the whole traceback to the issue tracker at',
+                  file=sys.stderr)
+            print('<https://bitbucket.org/birkenfeld/pygments-main/issues>.',
+                  file=sys.stderr)
+            print('*' * 65, file=sys.stderr)
+            print(file=sys.stderr)
+            raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
@@ -513,4 +527,6 @@ def main(args=sys.argv):
         print(file=sys.stderr)
         print('*** Error while highlighting:', file=sys.stderr)
         print(msg, file=sys.stderr)
+        print('*** If this is a bug you want to report, please rerun with -v.',
+              file=sys.stderr)
         return 1
diff --git a/setup.py b/setup.py
index c46d805e..36db0017 100755
--- a/setup.py
+++ b/setup.py
@@ -54,7 +54,7 @@ else:
 
 setup(
     name = 'Pygments',
-    version = '2.0.1',
+    version = '2.1a0',
     url = 'http://pygments.org/',
     license = 'BSD License',
     author = 'Georg Brandl',
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 64c4245e..cf935152 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -11,32 +11,44 @@ from __future__ import print_function
 
 import io
 import os
+import re
 import sys
 import tempfile
 import unittest
 
-from pygments import highlight, cmdline
-from pygments.util import StringIO, BytesIO
-
 import support
+from pygments import cmdline, highlight
+from pygments.util import BytesIO, StringIO
+
 
 TESTFILE, TESTDIR = support.location(__file__)
+TESTCODE = '''\
+def func(args):
+    pass
+'''
 
 
-def run_cmdline(*args):
+def run_cmdline(*args, **kwds):
+    saved_stdin = sys.stdin
     saved_stdout = sys.stdout
     saved_stderr = sys.stderr
     if sys.version_info > (3,):
+        stdin_buffer = BytesIO()
         stdout_buffer = BytesIO()
         stderr_buffer = BytesIO()
+        new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8')
         new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
         new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
     else:
+        stdin_buffer = new_stdin = sys.stdin = StringIO()
         stdout_buffer = new_stdout = sys.stdout = StringIO()
         stderr_buffer = new_stderr = sys.stderr = StringIO()
+    new_stdin.write(kwds.get('stdin', ''))
+    new_stdin.seek(0, 0)
     try:
         ret = cmdline.main(["pygmentize"] + list(args))
     finally:
+        sys.stdin = saved_stdin
         sys.stdout = saved_stdout
         sys.stderr = saved_stderr
     new_stdout.flush()
@@ -48,88 +60,127 @@ def run_cmdline(*args):
 
 class CmdLineTest(unittest.TestCase):
 
+    def check_success(self, *cmdline, **kwds):
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, 0)
+        self.assertEqual(err, '')
+        return out
+
+    def check_failure(self, *cmdline, **kwds):
+        expected_code = kwds.pop('code', 1)
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, expected_code)
+        self.assertEqual(out, '')
+        return err
+
+    def test_normal(self):
+        # test that cmdline gives the same output as library api
+        from pygments.lexers import PythonLexer
+        from pygments.formatters import HtmlFormatter
+        filename = TESTFILE
+        with open(filename, 'rb') as fp:
+            code = fp.read()
+
+        output = highlight(code, PythonLexer(), HtmlFormatter())
+
+        o = self.check_success("-lpython", "-fhtml", filename)
+        self.assertEqual(o, output)
+
+    def test_stdin(self):
+        o = self.check_success('-lpython', '-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # guess if no lexer given
+        o = self.check_success('-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+    def test_outfile(self):
+        # test that output file works with and without encoding
+        fd, name = tempfile.mkstemp()
+        os.close(fd)
+        for opts in [['-fhtml', '-o', name, TESTFILE],
+                     ['-flatex', '-o', name, TESTFILE],
+                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
+            try:
+                self.check_success(*opts)
+            finally:
+                os.unlink(name)
+
+    def test_stream_opt(self):
+        o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE)
+        o = re.sub(r'\x1b\[.*?m', '', o)
+        self.assertEqual(o, TESTCODE)
+
     def test_L_opt(self):
-        c, o, e = run_cmdline("-L")
-        self.assertEqual(c, 0)
+        o = self.check_success("-L")
         self.assertTrue("Lexers" in o and "Formatters" in o and
                         "Filters" in o and "Styles" in o)
-        c, o, e = run_cmdline("-L", "lexer")
-        self.assertEqual(c, 0)
+        o = self.check_success("-L", "lexer")
         self.assertTrue("Lexers" in o and "Formatters" not in o)
-        c, o, e = run_cmdline("-L", "lexers")
-        self.assertEqual(c, 0)
+        self.check_success("-L", "lexers")
 
     def test_O_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Ofull=1,linenos=true,foo=bar",
+                               "-fhtml", filename)
         self.assertTrue("<html" in o)
         self.assertTrue('class="linenos"' in o)
 
+        # "foobar" is invalid for a bool option
+        e = self.check_failure("-Ostripnl=foobar", TESTFILE)
+        self.assertTrue('Error: Invalid value' in e)
+        e = self.check_failure("-Ostripnl=foobar", "-lpy")
+        self.assertTrue('Error: Invalid value' in e)
+
     def test_P_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Pfull", "-Ptitle=foo, bar=baz=,",
+                               "-fhtml", filename)
         self.assertTrue("<title>foo, bar=baz=,</title>" in o)
 
     def test_F_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
-                              "names=TESTFILE filename",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Fhighlight:tokentype=Name.Blubb,"
+                               "names=TESTFILE filename",
+                               "-fhtml", filename)
         self.assertTrue('<span class="n-Blubb' in o)
 
     def test_H_opt(self):
-        c, o, e = run_cmdline("-H", "formatter", "html")
-        self.assertEqual(c, 0)
+        o = self.check_success("-H", "formatter", "html")
         self.assertTrue('HTML' in o)
+        o = self.check_success("-H", "lexer", "python")
+        self.assertTrue('Python' in o)
+        o = self.check_success("-H", "filter", "raiseonerror")
+        self.assertTrue('raiseonerror', o)
 
     def test_S_opt(self):
-        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
-        self.assertEqual(c, 0)
+        o = self.check_success("-S", "default", "-f", "html", "-O", "linenos=1")
+        lines = o.splitlines()
+        for line in lines:
+            # every line is for a token class
+            parts = line.split()
+            self.assertTrue(parts[0].startswith('.'))
+            self.assertTrue(parts[1] == '{')
+            if parts[0] != '.hll':
+                self.assertTrue(parts[-4] == '}')
+                self.assertTrue(parts[-3] == '/*')
+                self.assertTrue(parts[-1] == '*/')
+
+    def test_N_opt(self):
+        o = self.check_success("-N", "test.py")
+        self.assertEqual('python\n', o)
+        o = self.check_success("-N", "test.unknown")
+        self.assertEqual('text\n', o)
 
     def test_invalid_opts(self):
         for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
                      ("-a",), ("-Sst", "-lpy"), ("-H",),
-                     ("-H", "formatter")]:
-            self.assertTrue(run_cmdline(*opts)[0] == 2)
-
-    def test_normal(self):
-        # test that cmdline gives the same output as library api
-        from pygments.lexers import PythonLexer
-        from pygments.formatters import HtmlFormatter
-        filename = TESTFILE
-        with open(filename, 'rb') as fp:
-            code = fp.read()
-
-        output = highlight(code, PythonLexer(), HtmlFormatter())
-
-        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
-
-        self.assertEqual(o, output)
-        self.assertEqual(e, "")
-        self.assertEqual(c, 0)
-
-    def test_outfile(self):
-        # test that output file works with and without encoding
-        fd, name = tempfile.mkstemp()
-        os.close(fd)
-        for opts in [['-fhtml', '-o', name, TESTFILE],
-                     ['-flatex', '-o', name, TESTFILE],
-                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
-            try:
-                self.assertEqual(run_cmdline(*opts)[0], 0)
-            finally:
-                os.unlink(name)
-
-    def check_failure(self, *cmdline):
-        c, o, e = run_cmdline(*cmdline)
-        self.assertEqual(c, 1)
-        self.assertEqual(o, '')
-        return e
+                     ("-H", "formatter"), ("-H", "foo", "bar"), ("-s",)]:
+            self.check_failure(*opts, code=2)
 
     def test_errors(self):
         # input file not found
@@ -156,10 +207,23 @@ class CmdLineTest(unittest.TestCase):
         self.assertTrue('Error: filter \'foo\' not found' in e)
 
     def test_exception(self):
-        # unexpected exception while highlighting
-        cmdline.highlight = None  # override callable
+        cmdline.highlight = None  # override callable to provoke TypeError
         try:
+            # unexpected exception while highlighting
             e = self.check_failure('-lpython', TESTFILE)
+            self.assertTrue('*** Error while highlighting:' in e)
+            self.assertTrue('TypeError' in e)
+
+            # same with -v: should reraise the exception
+            try:
+                self.check_failure('-lpython', '-v', TESTFILE)
+            except Exception:
+                pass
+            else:
+                self.fail('exception not reraised')
         finally:
             cmdline.highlight = highlight
-        self.assertTrue('*** Error while highlighting:' in e)
+
+    def test_parse_opts(self):
+        self.assertEqual(cmdline._parse_options(['  ', 'keyonly,key = value ']),
+                         {'keyonly': True, 'key': 'value'})
diff --git a/tests/test_util.py b/tests/test_util.py
index e480b503..d6ebd51d 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -10,7 +10,7 @@
 import re
 import unittest
 
-from pygments import util
+from pygments import util, console
 
 
 class FakeLexer(object):
@@ -40,7 +40,6 @@ class UtilTest(unittest.TestCase):
         equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
         raises(util.OptionError, util.get_list_opt, {}, 'a', 1)
 
-
     def test_docstring_headline(self):
         def f1():
             """
@@ -157,3 +156,20 @@ class UtilTest(unittest.TestCase):
         # keeps first
         x = util.duplicates_removed(('a', 'b', 'a'))
         self.assertEqual(['a', 'b'], x)
+
+
+class ConsoleTest(unittest.TestCase):
+
+    def test_ansiformat(self):
+        f = console.ansiformat
+        c = console.codes
+        all_attrs = f('+*_blue_*+', 'text')
+        self.assertTrue(c['blue'] in all_attrs and c['blink'] in all_attrs
+                        and c['bold'] in all_attrs and c['underline'] in all_attrs
+                        and c['reset'] in all_attrs)
+        self.assertRaises(KeyError, f, '*mauve*', 'text')
+
+    def test_functions(self):
+        self.assertEqual(console.reset_color(), console.codes['reset'])
+        self.assertEqual(console.colorize('blue', 'text'),
+                         console.codes['blue'] + 'text' + console.codes['reset'])