Bring pygments.cmdline coverage to ~100%, fix a few glitches in the process.

2 files changed, 126 insertions, 74 deletions

diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 53e858b5..10b47f89 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -19,7 +19,7 @@ from pygments import __version__, highlight
 from pygments.util import ClassNotFound, OptionError, docstring_headline, \
     guess_decode, guess_decode_from_terminal, terminal_encoding
 from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
-    get_lexer_for_filename, find_lexer_class, TextLexer
+    get_lexer_for_filename, find_lexer_class_for_filename, TextLexer
 from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
     get_formatter_for_filename, find_formatter_class, \
@@ -103,7 +103,7 @@ def _parse_options(o_strs):
     if not o_strs:
         return opts
     for o_str in o_strs:
-        if not o_str:
+        if not o_str.strip():
             continue
         o_args = o_str.split(',')
         for o_arg in o_args:
@@ -135,7 +135,7 @@ def _parse_filters(f_strs):
 def _print_help(what, name):
     try:
         if what == 'lexer':
-            cls = find_lexer_class(name)
+            cls = get_lexer_by_name(name)
             print("Help on the %s lexer:" % cls.name)
             print(dedent(cls.__doc__))
         elif what == 'formatter':
@@ -146,7 +146,7 @@ def _print_help(what, name):
             cls = find_filter_class(name)
             print("Help on the %s filter:" % name)
             print(dedent(cls.__doc__))
-    except AttributeError:
+    except (AttributeError, ValueError):
         print("%s not found!" % what, file=sys.stderr)
 
 
@@ -274,13 +274,9 @@ def main_inner(popts, args, usage):
     # handle ``pygmentize -N``
     infn = opts.pop('-N', None)
     if infn is not None:
-        try:
-            lexer = get_lexer_for_filename(infn, **parsed_opts)
-        except ClassNotFound as err:
-            lexer = TextLexer()
-        except OptionError as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
+        lexer = find_lexer_class_for_filename(infn)
+        if lexer is None:
+            lexer = TextLexer
 
         print(lexer.aliases[0])
         return 0
@@ -432,7 +428,7 @@ def main_inner(popts, args, usage):
 
     # provide coloring under Windows, if possible
     if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
+       fmter.name in ('Terminal', 'Terminal256'):  # pragma: no cover
         # unfortunately colorama doesn't support binary streams on Py3
         if sys.version_info > (3,):
             import io
@@ -472,7 +468,7 @@ def main_inner(popts, args, usage):
         if not lexer:
             print('Error: when using -s a lexer has to be selected with -l',
                   file=sys.stderr)
-            return 1
+            return 2
         # line by line processing of stdin (eg: for 'tail -f')...
         try:
             while 1:
@@ -488,7 +484,8 @@ def main_inner(popts, args, usage):
                 highlight(line, lexer, fmter, outfile)
                 if hasattr(outfile, 'flush'):
                     outfile.flush()
-        except KeyboardInterrupt:
+            return 0
+        except KeyboardInterrupt:  # pragma: no cover
             return 0
 
 
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 8929ce0a..cf935152 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -11,32 +11,44 @@ from __future__ import print_function
 
 import io
 import os
+import re
 import sys
 import tempfile
 import unittest
 
-from pygments import highlight, cmdline
-from pygments.util import StringIO, BytesIO
-
 import support
+from pygments import cmdline, highlight
+from pygments.util import BytesIO, StringIO
+
 
 TESTFILE, TESTDIR = support.location(__file__)
+TESTCODE = '''\
+def func(args):
+    pass
+'''
 
 
-def run_cmdline(*args):
+def run_cmdline(*args, **kwds):
+    saved_stdin = sys.stdin
     saved_stdout = sys.stdout
     saved_stderr = sys.stderr
     if sys.version_info > (3,):
+        stdin_buffer = BytesIO()
         stdout_buffer = BytesIO()
         stderr_buffer = BytesIO()
+        new_stdin = sys.stdin = io.TextIOWrapper(stdin_buffer, 'utf-8')
         new_stdout = sys.stdout = io.TextIOWrapper(stdout_buffer, 'utf-8')
         new_stderr = sys.stderr = io.TextIOWrapper(stderr_buffer, 'utf-8')
     else:
+        stdin_buffer = new_stdin = sys.stdin = StringIO()
         stdout_buffer = new_stdout = sys.stdout = StringIO()
         stderr_buffer = new_stderr = sys.stderr = StringIO()
+    new_stdin.write(kwds.get('stdin', ''))
+    new_stdin.seek(0, 0)
     try:
         ret = cmdline.main(["pygmentize"] + list(args))
     finally:
+        sys.stdin = saved_stdin
         sys.stdout = saved_stdout
         sys.stderr = saved_stderr
     new_stdout.flush()
@@ -48,88 +60,127 @@ def run_cmdline(*args):
 
 class CmdLineTest(unittest.TestCase):
 
+    def check_success(self, *cmdline, **kwds):
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, 0)
+        self.assertEqual(err, '')
+        return out
+
+    def check_failure(self, *cmdline, **kwds):
+        expected_code = kwds.pop('code', 1)
+        code, out, err = run_cmdline(*cmdline, **kwds)
+        self.assertEqual(code, expected_code)
+        self.assertEqual(out, '')
+        return err
+
+    def test_normal(self):
+        # test that cmdline gives the same output as library api
+        from pygments.lexers import PythonLexer
+        from pygments.formatters import HtmlFormatter
+        filename = TESTFILE
+        with open(filename, 'rb') as fp:
+            code = fp.read()
+
+        output = highlight(code, PythonLexer(), HtmlFormatter())
+
+        o = self.check_success("-lpython", "-fhtml", filename)
+        self.assertEqual(o, output)
+
+    def test_stdin(self):
+        o = self.check_success('-lpython', '-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+        # guess if no lexer given
+        o = self.check_success('-fhtml', stdin=TESTCODE)
+        o = re.sub('<[^>]*>', '', o)
+        # rstrip is necessary since HTML inserts a \n after the last </div>
+        self.assertEqual(o.rstrip(), TESTCODE.rstrip())
+
+    def test_outfile(self):
+        # test that output file works with and without encoding
+        fd, name = tempfile.mkstemp()
+        os.close(fd)
+        for opts in [['-fhtml', '-o', name, TESTFILE],
+                     ['-flatex', '-o', name, TESTFILE],
+                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
+            try:
+                self.check_success(*opts)
+            finally:
+                os.unlink(name)
+
+    def test_stream_opt(self):
+        o = self.check_success('-lpython', '-s', '-fterminal', stdin=TESTCODE)
+        o = re.sub(r'\x1b\[.*?m', '', o)
+        self.assertEqual(o, TESTCODE)
+
     def test_L_opt(self):
-        c, o, e = run_cmdline("-L")
-        self.assertEqual(c, 0)
+        o = self.check_success("-L")
         self.assertTrue("Lexers" in o and "Formatters" in o and
                         "Filters" in o and "Styles" in o)
-        c, o, e = run_cmdline("-L", "lexer")
-        self.assertEqual(c, 0)
+        o = self.check_success("-L", "lexer")
         self.assertTrue("Lexers" in o and "Formatters" not in o)
-        c, o, e = run_cmdline("-L", "lexers")
-        self.assertEqual(c, 0)
+        self.check_success("-L", "lexers")
 
     def test_O_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Ofull=1,linenos=true,foo=bar",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Ofull=1,linenos=true,foo=bar",
+                               "-fhtml", filename)
         self.assertTrue("<html" in o)
         self.assertTrue('class="linenos"' in o)
 
+        # "foobar" is invalid for a bool option
+        e = self.check_failure("-Ostripnl=foobar", TESTFILE)
+        self.assertTrue('Error: Invalid value' in e)
+        e = self.check_failure("-Ostripnl=foobar", "-lpy")
+        self.assertTrue('Error: Invalid value' in e)
+
     def test_P_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Pfull", "-Ptitle=foo, bar=baz=,",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Pfull", "-Ptitle=foo, bar=baz=,",
+                               "-fhtml", filename)
         self.assertTrue("<title>foo, bar=baz=,</title>" in o)
 
     def test_F_opt(self):
         filename = TESTFILE
-        c, o, e = run_cmdline("-Fhighlight:tokentype=Name.Blubb,"
-                              "names=TESTFILE filename",
-                              "-fhtml", filename)
-        self.assertEqual(c, 0)
+        o = self.check_success("-Fhighlight:tokentype=Name.Blubb,"
+                               "names=TESTFILE filename",
+                               "-fhtml", filename)
         self.assertTrue('<span class="n-Blubb' in o)
 
     def test_H_opt(self):
-        c, o, e = run_cmdline("-H", "formatter", "html")
-        self.assertEqual(c, 0)
+        o = self.check_success("-H", "formatter", "html")
         self.assertTrue('HTML' in o)
+        o = self.check_success("-H", "lexer", "python")
+        self.assertTrue('Python' in o)
+        o = self.check_success("-H", "filter", "raiseonerror")
+        self.assertTrue('raiseonerror', o)
 
     def test_S_opt(self):
-        c, o, e = run_cmdline("-S", "default", "-f", "html", "-O", "linenos=1")
-        self.assertEqual(c, 0)
+        o = self.check_success("-S", "default", "-f", "html", "-O", "linenos=1")
+        lines = o.splitlines()
+        for line in lines:
+            # every line is for a token class
+            parts = line.split()
+            self.assertTrue(parts[0].startswith('.'))
+            self.assertTrue(parts[1] == '{')
+            if parts[0] != '.hll':
+                self.assertTrue(parts[-4] == '}')
+                self.assertTrue(parts[-3] == '/*')
+                self.assertTrue(parts[-1] == '*/')
+
+    def test_N_opt(self):
+        o = self.check_success("-N", "test.py")
+        self.assertEqual('python\n', o)
+        o = self.check_success("-N", "test.unknown")
+        self.assertEqual('text\n', o)
 
     def test_invalid_opts(self):
         for opts in [("-L", "-lpy"), ("-L", "-fhtml"), ("-L", "-Ox"),
                      ("-a",), ("-Sst", "-lpy"), ("-H",),
-                     ("-H", "formatter")]:
-            self.assertTrue(run_cmdline(*opts)[0] == 2)
-
-    def test_normal(self):
-        # test that cmdline gives the same output as library api
-        from pygments.lexers import PythonLexer
-        from pygments.formatters import HtmlFormatter
-        filename = TESTFILE
-        with open(filename, 'rb') as fp:
-            code = fp.read()
-
-        output = highlight(code, PythonLexer(), HtmlFormatter())
-
-        c, o, e = run_cmdline("-lpython", "-fhtml", filename)
-
-        self.assertEqual(o, output)
-        self.assertEqual(e, "")
-        self.assertEqual(c, 0)
-
-    def test_outfile(self):
-        # test that output file works with and without encoding
-        fd, name = tempfile.mkstemp()
-        os.close(fd)
-        for opts in [['-fhtml', '-o', name, TESTFILE],
-                     ['-flatex', '-o', name, TESTFILE],
-                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
-            try:
-                self.assertEqual(run_cmdline(*opts)[0], 0)
-            finally:
-                os.unlink(name)
-
-    def check_failure(self, *cmdline):
-        c, o, e = run_cmdline(*cmdline)
-        self.assertEqual(c, 1)
-        self.assertEqual(o, '')
-        return e
+                     ("-H", "formatter"), ("-H", "foo", "bar"), ("-s",)]:
+            self.check_failure(*opts, code=2)
 
     def test_errors(self):
         # input file not found
@@ -172,3 +223,7 @@ class CmdLineTest(unittest.TestCase):
                 self.fail('exception not reraised')
         finally:
             cmdline.highlight = highlight
+
+    def test_parse_opts(self):
+        self.assertEqual(cmdline._parse_options(['  ', 'keyonly,key = value ']),
+                         {'keyonly': True, 'key': 'value'})