Closes #1058: Fix an encoding issue when using ``pygmentize`` with the ``-o`` option.

3 files changed, 79 insertions, 56 deletions

diff --git a/CHANGES b/CHANGES
index 1537480f..7031852e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -6,6 +6,14 @@ Issue numbers refer to the tracker at
 pull request numbers to the requests at
 <http://bitbucket.org/birkenfeld/pygments-main/pull-requests/merged>.
 
+
+Version 2.0.1
+-------------
+(released Nov 10, 2014)
+
+- Fix an encoding issue when using ``pygmentize`` with the ``-o`` option.
+
+
 Version 2.0
 -----------
 (released Nov 9, 2014)
diff --git a/pygments/cmdline.py b/pygments/cmdline.py
index 1a1f8faa..b6c319cf 100644
--- a/pygments/cmdline.py
+++ b/pygments/cmdline.py
@@ -330,62 +330,6 @@ def main(args=sys.argv):
     F_opts = _parse_filters(F_opts)
     opts.pop('-F', None)
 
-    # select formatter
-    outfn = opts.pop('-o', None)
-    fmter = opts.pop('-f', None)
-    if fmter:
-        try:
-            fmter = get_formatter_by_name(fmter, **parsed_opts)
-        except (OptionError, ClassNotFound) as err:
-            print('Error:', err, file=sys.stderr)
-            return 1
-
-    if outfn:
-        if not fmter:
-            try:
-                fmter = get_formatter_for_filename(outfn, **parsed_opts)
-            except (OptionError, ClassNotFound) as err:
-                print('Error:', err, file=sys.stderr)
-                return 1
-        try:
-            outfile = open(outfn, 'wb')
-        except Exception as err:
-            print('Error: cannot open outfile:', err, file=sys.stderr)
-            return 1
-    else:
-        if not fmter:
-            fmter = TerminalFormatter(**parsed_opts)
-        if sys.version_info > (3,):
-            # Python 3: we have to use .buffer to get a binary stream
-            outfile = sys.stdout.buffer
-        else:
-            outfile = sys.stdout
-
-    # determine output encoding if not explicitly selected
-    if not outencoding:
-        if outfn:
-            # output file? -> encoding pass-through
-            fmter.encoding = inencoding
-        else:
-            # else use terminal encoding
-            fmter.encoding = terminal_encoding(sys.stdout)
-
-    # provide coloring under Windows, if possible
-    if not outfn and sys.platform in ('win32', 'cygwin') and \
-       fmter.name in ('Terminal', 'Terminal256'):
-        # unfortunately colorama doesn't support binary streams on Py3
-        if sys.version_info > (3,):
-            import io
-            outfile = io.TextIOWrapper(outfile, encoding=fmter.encoding)
-            fmter.encoding = None
-        try:
-            import colorama.initialise
-        except ImportError:
-            pass
-        else:
-            outfile = colorama.initialise.wrap_stream(
-                outfile, convert=None, strip=None, autoreset=False, wrap=True)
-
     # select lexer
     lexer = opts.pop('-l', None)
     if lexer:
@@ -452,6 +396,62 @@ def main(args=sys.argv):
             except ClassNotFound:
                 lexer = TextLexer(**parsed_opts)
 
+    # select formatter
+    outfn = opts.pop('-o', None)
+    fmter = opts.pop('-f', None)
+    if fmter:
+        try:
+            fmter = get_formatter_by_name(fmter, **parsed_opts)
+        except (OptionError, ClassNotFound) as err:
+            print('Error:', err, file=sys.stderr)
+            return 1
+
+    if outfn:
+        if not fmter:
+            try:
+                fmter = get_formatter_for_filename(outfn, **parsed_opts)
+            except (OptionError, ClassNotFound) as err:
+                print('Error:', err, file=sys.stderr)
+                return 1
+        try:
+            outfile = open(outfn, 'wb')
+        except Exception as err:
+            print('Error: cannot open outfile:', err, file=sys.stderr)
+            return 1
+    else:
+        if not fmter:
+            fmter = TerminalFormatter(**parsed_opts)
+        if sys.version_info > (3,):
+            # Python 3: we have to use .buffer to get a binary stream
+            outfile = sys.stdout.buffer
+        else:
+            outfile = sys.stdout
+
+    # determine output encoding if not explicitly selected
+    if not outencoding:
+        if outfn:
+            # output file? use lexer encoding for now (can still be None)
+            fmter.encoding = inencoding
+        else:
+            # else use terminal encoding
+            fmter.encoding = terminal_encoding(sys.stdout)
+
+    # provide coloring under Windows, if possible
+    if not outfn and sys.platform in ('win32', 'cygwin') and \
+       fmter.name in ('Terminal', 'Terminal256'):
+        # unfortunately colorama doesn't support binary streams on Py3
+        if sys.version_info > (3,):
+            import io
+            outfile = io.TextIOWrapper(outfile, encoding=fmter.encoding)
+            fmter.encoding = None
+        try:
+            import colorama.initialise
+        except ImportError:
+            pass
+        else:
+            outfile = colorama.initialise.wrap_stream(
+                outfile, convert=None, strip=None, autoreset=False, wrap=True)
+
     # When using the LaTeX formatter and the option `escapeinside` is
     # specified, we need a special lexer which collects escaped text
     # before running the chosen language lexer.
@@ -494,6 +494,7 @@ def main(args=sys.argv):
                 return 0
 
     except Exception:
+        raise
         import traceback
         info = traceback.format_exception(*sys.exc_info())
         msg = info[-1].strip()
diff --git a/tests/test_cmdline.py b/tests/test_cmdline.py
index 2e02ae52..038b96bf 100644
--- a/tests/test_cmdline.py
+++ b/tests/test_cmdline.py
@@ -10,7 +10,9 @@
 from __future__ import print_function
 
 import io
+import os
 import sys
+import tempfile
 import unittest
 
 from pygments import highlight
@@ -111,3 +113,15 @@ class CmdLineTest(unittest.TestCase):
         self.assertEqual(o, output)
         self.assertEqual(e, "")
         self.assertEqual(c, 0)
+
+    def test_outfile(self):
+        # test that output file works with and without encoding
+        fd, name = tempfile.mkstemp()
+        os.close(fd)
+        for opts in [['-fhtml', '-o', name, TESTFILE],
+                     ['-flatex', '-o', name, TESTFILE],
+                     ['-fhtml', '-o', name, '-O', 'encoding=utf-8', TESTFILE]]:
+            try:
+                self.assertEqual(run_cmdline(*opts)[0], 0)
+            finally:
+                os.unlink(name)