merge with stable

4 files changed, 238 insertions, 77 deletions

diff --git a/tests/examplefiles/example.stan b/tests/examplefiles/example.stan
index 716b4d12..69c9ac70 100644
--- a/tests/examplefiles/example.stan
+++ b/tests/examplefiles/example.stan
@@ -111,6 +111,9 @@ model {
   print("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_~@#$%^&*`'-+={}[].,;: ");
   print("Hello, world!");
   print("");
+
+  // reject statement
+  reject("I just don't like it");
   
 }
 generated quantities {
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index e1aa123d..c1e83077 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -11,32 +11,44 @@ from __future__ import print_function
 
 import io
 import os
+import re
 import sys
 import tempfile
 import unittest
 
-from pygments import highlight, cmdline
-from pygments.util import StringIO, BytesIO
-
 import support
+from pygments import cmdline, highlight
+from pygments.util import BytesIO, StringIO
+
 
 TESTFILE, TESTDIR = support.location(__file__)
+TESTCODE = '''\
+def func(args):
+    pass
+'''
 
 
-def run_cmdline(*args):
+def run_cmdline(*args, **kwds):
+    saved_stdin = sys.stdin
     saved_stdout = sys.stdout
     saved_stderr = sys.stderr
     if sys.version_info > (3,):
+        stdin_buffer = BytesIO()
         stdout_buffer = BytesIO()
         stderr_buffer = BytesIO()
+        new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8')
         new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
         new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
     else:
+        stdin_buffer = new_stdin = sys.stdin = StringIO()
         stdout_buffer = new_stdout = sys.stdout = StringIO()
         stderr_buffer = new_stderr = sys.stderr = StringIO()
+    new_stdin.write(kwds.get('stdin', ''))
+    new_stdin.seek(0, 0)
     try:
-        ret = cmdline.main(["pygmentize"] + list(args))
+        ret = cmdline.main(['pygmentize'] + list(args))
     finally:
+        sys.stdin = saved_stdin
         sys.stdout = saved_stdout
         sys.stderr = saved_stderr
     new_stdout.flush()
@@ -48,54 +60,18 @@ def run_cmdline(*args):
 
 class CmdLineTest(unittest.TestCase):
 
-    def test_L_opt(self):
-        c, o, e = run_cmdline("-L")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" in o and
-                        "Filters" in o and "Styles" in o)
-        c, o, e = run_cmdline("-L", "lexer")
-        self.assertEqual(c, 0)
-        self.assertTrue("Lexers" in o and "Formatters" not in o)
-        c, o, e = run_cmdline("-L", "lexers")
-        self.assertEqual(c, 0)
-
-    def test_O_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<html" in o)
-        self.assertTrue('class="linenos"' in o)
+    def check_success(self, *cmdline, **kwds):
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, 0)
+        self.assertEqual(err, '')
+        return out
 
-    def test_P_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue("<title>foo, bar=baz=,</title>" in o)
-
-    def test_F_opt(self):
-        filename = TESTFILE
-        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
-                              "names=TESTFILE filename",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
-        self.assertTrue('<span class="n-Blubb' in o)
-
-    def test_H_opt(self):
-        c, o, e = run_cmdline("-H", "formatter", "html")
-        self.assertEqual(c, 0)
-        self.assertTrue('HTML' in o)
-
-    def test_S_opt(self):
-        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
-        self.assertEqual(c, 0)
-
-    def test_invalid_opts(self):
-        for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
-                     ("-a",), ("-Sst", "-lpy"), ("-H",),
-                     ("-H", "formatter")]:
-            self.assertTrue(run_cmdline(*opts)[0] == 2)
+    def check_failure(self, *cmdline, **kwds):
+        expected_code = kwds.pop('code', 1)
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, expected_code)
+        self.assertEqual(out, '')
+        return err
 
     def test_normal(self):
         # test that cmdline gives the same output as library api
@@ -107,11 +83,20 @@ class CmdLineTest(unittest.TestCase):
 
         output = highlight(code, PythonLexer(), HtmlFormatter())
 
-        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
-
+        o = self.check_success('-lpython', '-fhtml', filename)
         self.assertEqual(o, output)
-        self.assertEqual(e, "")
-        self.assertEqual(c, 0)
+
+    def test_stdin(self):
+        o = self.check_success('-lpython', '-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # guess if no lexer given
+        o = self.check_success('-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
 
     def test_outfile(self):
         # test that output file works with and without encoding
@@ -121,15 +106,100 @@ class CmdLineTest(unittest.TestCase):
                      ['-flatex', '-o', name, TESTFILE],
                      ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
             try:
-                self.assertEqual(run_cmdline(*opts)[0], 0)
+                self.check_success(*opts)
             finally:
                 os.unlink(name)
 
-    def check_failure(self, *cmdline):
-        c, o, e = run_cmdline(*cmdline)
-        self.assertEqual(c, 1)
-        self.assertEqual(o, '')
-        return e
+    def test_stream_opt(self):
+        o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE)
+        o = re.sub(r'\x1b\[.*?m', '', o)
+        self.assertEqual(o.replace('\r\n', '\n'), TESTCODE)
+
+    def test_h_opt(self):
+        o = self.check_success('-h')
+        self.assertTrue('Usage:' in o)
+
+    def test_L_opt(self):
+        o = self.check_success('-L')
+        self.assertTrue('Lexers' in o and 'Formatters' in o and
+                        'Filters' in o and 'Styles' in o)
+        o = self.check_success('-L', 'lexer')
+        self.assertTrue('Lexers' in o and 'Formatters' not in o)
+        self.check_success('-L', 'lexers')
+
+    def test_O_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Ofull=1,linenos=true,foo=bar',
+                               '-fhtml', filename)
+        self.assertTrue('<html' in o)
+        self.assertTrue('class="linenos"' in o)
+
+        # "foobar" is invalid for a bool option
+        e = self.check_failure('-Ostripnl=foobar', TESTFILE)
+        self.assertTrue('Error: Invalid value' in e)
+        e = self.check_failure('-Ostripnl=foobar', '-lpy')
+        self.assertTrue('Error: Invalid value' in e)
+
+    def test_P_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Pfull', '-Ptitle=foo, bar=baz=,',
+                               '-fhtml', filename)
+        self.assertTrue('<title>foo, bar=baz=,</title>' in o)
+
+    def test_F_opt(self):
+        filename = TESTFILE
+        o = self.check_success('-Fhighlight:tokentype=Name.Blubb,'
+                               'names=TESTFILE filename',
+                               '-fhtml', filename)
+        self.assertTrue('<span class="n-Blubb' in o)
+
+    def test_H_opt(self):
+        o = self.check_success('-H', 'formatter', 'html')
+        self.assertTrue('HTML' in o)
+        o = self.check_success('-H', 'lexer', 'python')
+        self.assertTrue('Python' in o)
+        o = self.check_success('-H', 'filter', 'raiseonerror')
+        self.assertTrue('raiseonerror', o)
+        e = self.check_failure('-H', 'lexer', 'foobar')
+        self.assertTrue('not found' in e)
+
+    def test_S_opt(self):
+        o = self.check_success('-S', 'default', '-f', 'html', '-O', 'linenos=1')
+        lines = o.splitlines()
+        for line in lines:
+            # every line is for a token class
+            parts = line.split()
+            self.assertTrue(parts[0].startswith('.'))
+            self.assertTrue(parts[1] == '{')
+            if parts[0] != '.hll':
+                self.assertTrue(parts[-4] == '}')
+                self.assertTrue(parts[-3] == '/*')
+                self.assertTrue(parts[-1] == '*/')
+        self.check_failure('-S', 'default', '-f', 'foobar')
+
+    def test_N_opt(self):
+        o = self.check_success('-N', 'test.py')
+        self.assertEqual('python', o.strip())
+        o = self.check_success('-N', 'test.unknown')
+        self.assertEqual('text', o.strip())
+
+    def test_invalid_opts(self):
+        for opts in [
+            ('-X',),
+            ('-L', '-lpy'),
+            ('-L', '-fhtml'),
+            ('-L', '-Ox'),
+            ('-S', 'default', '-l', 'py', '-f', 'html'),
+            ('-S', 'default'),
+            ('-a', 'arg'),
+            ('-H',),
+            (TESTFILE, TESTFILE),
+            ('-H', 'formatter'),
+            ('-H', 'foo', 'bar'),
+            ('-s',),
+            ('-s', TESTFILE),
+        ]:
+            self.check_failure(*opts, code=2)
 
     def test_errors(self):
         # input file not found
@@ -145,6 +215,10 @@ class CmdLineTest(unittest.TestCase):
         e = self.check_failure('-lpython', '-ffoo', TESTFILE)
         self.assertTrue('Error: no formatter found for name' in e)
 
+        # formatter for outfile not found
+        e = self.check_failure('-ofoo.foo', TESTFILE)
+        self.assertTrue('Error: no formatter found for file name' in e)
+
         # output file not writable
         e = self.check_failure('-o', os.path.join('nonexistent', 'dir', 'out.html'),
                                '-lpython', TESTFILE)
@@ -156,10 +230,23 @@ class CmdLineTest(unittest.TestCase):
         self.assertTrue('Error: filter \'foo\' not found' in e)
 
     def test_exception(self):
-        # unexpected exception while highlighting
-        cmdline.highlight = None  # override callable
+        cmdline.highlight = None  # override callable to provoke TypeError
         try:
+            # unexpected exception while highlighting
             e = self.check_failure('-lpython', TESTFILE)
+            self.assertTrue('*** Error while highlighting:' in e)
+            self.assertTrue('TypeError' in e)
+
+            # same with -v: should reraise the exception
+            try:
+                self.check_failure('-lpython', '-v', TESTFILE)
+            except Exception:
+                pass
+            else:
+                self.fail('exception not reraised')
         finally:
             cmdline.highlight = highlight
-        self.assertTrue('*** Error while highlighting:' in e)
+
+    def test_parse_opts(self):
+        self.assertEqual(cmdline._parse_options(['  ', 'keyonly,key = value ']),
+                         {'keyonly': True, 'key': 'value'})
diff --git a/tests/test_html_formatter.py b/tests/test_html_formatter.py
index 79bf5072..a82aaaf7 100644
--- a/tests/test_html_formatter.py
+++ b/tests/test_html_formatter.py
@@ -68,15 +68,32 @@ class HtmlFormatterTest(unittest.TestCase):
             pass
 
     def test_all_options(self):
-        for optdict in [dict(nowrap=True),
-                        dict(linenos=True),
-                        dict(linenos=True, full=True),
-                        dict(linenos=True, full=True, noclasses=True)]:
-
+        def check(optdict):
             outfile = StringIO()
             fmt = HtmlFormatter(**optdict)
             fmt.format(tokensource, outfile)
 
+        for optdict in [
+            dict(nowrap=True),
+            dict(linenos=True, full=True),
+            dict(linenos=True, linespans='L'),
+            dict(hl_lines=[1, 5, 10, 'xxx']),
+            dict(hl_lines=[1, 5, 10], noclasses=True),
+        ]:
+            check(optdict)
+
+        for linenos in [False, 'table', 'inline']:
+            for noclasses in [False, True]:
+                for linenospecial in [0, 5]:
+                    for anchorlinenos in [False, True]:
+                        optdict = dict(
+                            linenos=linenos,
+                            noclasses=noclasses,
+                            linenospecial=linenospecial,
+                            anchorlinenos=anchorlinenos,
+                        )
+                        check(optdict)
+
     def test_linenos(self):
         optdict = dict(linenos=True)
         outfile = StringIO()
diff --git a/tests/test_util.py b/tests/test_util.py
index 092a9474..720b384a 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -10,12 +10,12 @@
 import re
 import unittest
 
-from pygments import util
+from pygments import util, console
 
 
 class FakeLexer(object):
     def analyse(text):
-        return float(text)
+        return text
     analyse = util.make_analysator(analyse)
 
 
@@ -40,6 +40,10 @@ class UtilTest(unittest.TestCase):
         equals(util.get_list_opt({}, 'a', '1 2'), ['1', '2'])
         raises(util.OptionError, util.get_list_opt, {}, 'a', 1)
 
+        equals(util.get_choice_opt({}, 'a', ['foo', 'bar'], 'bar'), 'bar')
+        equals(util.get_choice_opt({}, 'a', ['foo', 'bar'], 'Bar', True), 'bar')
+        raises(util.OptionError, util.get_choice_opt, {}, 'a',
+               ['foo', 'bar'], 'baz')
 
     def test_docstring_headline(self):
         def f1():
@@ -55,9 +59,12 @@ class UtilTest(unittest.TestCase):
 
             other text
             """
+        def f3():
+            pass
 
-        self.assertEqual(util.docstring_headline(f1), "docstring headline")
-        self.assertEqual(util.docstring_headline(f2), "docstring headline")
+        self.assertEqual(util.docstring_headline(f1), 'docstring headline')
+        self.assertEqual(util.docstring_headline(f2), 'docstring headline')
+        self.assertEqual(util.docstring_headline(f3), '')
 
     def test_analysator_returns_float(self):
         # If an analysator wrapped by make_analysator returns a floating point
@@ -88,10 +95,10 @@ class UtilTest(unittest.TestCase):
     def test_analysator_type_error(self):
         # When converting the analysator's return value to a float a
         # TypeError may occur.  If that happens 0.0 is returned instead.
-        self.assertEqual(FakeLexer.analyse(None), 0.0)
+        self.assertEqual(FakeLexer.analyse('xxx'), 0.0)
 
     def test_shebang_matches(self):
-        self.assertTrue(util.shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?'))
+        self.assertTrue(util.shebang_matches('#!/usr/bin/env python\n', r'python(2\.\d)?'))
         self.assertTrue(util.shebang_matches('#!/usr/bin/python2.4', r'python(2\.\d)?'))
         self.assertTrue(util.shebang_matches('#!/usr/bin/startsomethingwith python',
                                              r'python(2\.\d)?'))
@@ -106,7 +113,7 @@ class UtilTest(unittest.TestCase):
 
     def test_doctype_matches(self):
         self.assertTrue(util.doctype_matches(
-            '<!DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
+            '<!DOCTYPE html> <html>', 'html.*'))
         self.assertFalse(util.doctype_matches(
             '<?xml ?> <DOCTYPE html PUBLIC "a"> <html>', 'html.*'))
         self.assertTrue(util.html_doctype_matches(
@@ -157,3 +164,50 @@ class UtilTest(unittest.TestCase):
         # keeps first
         x = util.duplicates_removed(('a', 'b', 'a'))
         self.assertEqual(['a', 'b'], x)
+
+    def test_guess_decode(self):
+        # UTF-8 should be decoded as UTF-8
+        s = util.guess_decode(u'\xff'.encode('utf-8'))
+        self.assertEqual(s, (u'\xff', 'utf-8'))
+
+        # otherwise, it could be latin1 or the locale encoding...
+        import locale
+        s = util.guess_decode(b'\xff')
+        self.assertTrue(s[1] in ('latin1', locale.getpreferredencoding()))
+
+    def test_guess_decode_from_terminal(self):
+        class Term:
+            encoding = 'utf-7'
+
+        s = util.guess_decode_from_terminal(u'\xff'.encode('utf-7'), Term)
+        self.assertEqual(s, (u'\xff', 'utf-7'))
+
+        s = util.guess_decode_from_terminal(u'\xff'.encode('utf-8'), Term)
+        self.assertEqual(s, (u'\xff', 'utf-8'))
+
+    def test_add_metaclass(self):
+        class Meta(type):
+            pass
+
+        @util.add_metaclass(Meta)
+        class Cls:
+            pass
+
+        self.assertEqual(type(Cls), Meta)
+
+
+class ConsoleTest(unittest.TestCase):
+
+    def test_ansiformat(self):
+        f = console.ansiformat
+        c = console.codes
+        all_attrs = f('+*_blue_*+', 'text')
+        self.assertTrue(c['blue'] in all_attrs and c['blink'] in all_attrs
+                        and c['bold'] in all_attrs and c['underline'] in all_attrs
+                        and c['reset'] in all_attrs)
+        self.assertRaises(KeyError, f, '*mauve*', 'text')
+
+    def test_functions(self):
+        self.assertEqual(console.reset_color(), console.codes['reset'])
+        self.assertEqual(console.colorize('blue', 'text'),
+                         console.codes['blue'] + 'text' + console.codes['reset'])