Fix Raw token lexer/formatter with new default encoding.

2 files changed, 3 insertions, 1 deletions

diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py
index 6100f70b..ea8138c7 100644
--- a/pygments/formatters/other.py
+++ b/pygments/formatters/other.py
@@ -64,6 +64,7 @@ class RawTokenFormatter(Formatter):
         Formatter.__init__(self, **options)
         # We ignore self.encoding if it is set, since it gets set for lexer
         # and formatter if given with -Oencoding on the command line.
+        # The RawTokenFormatter outputs only ASCII. Override here.
         self.encoding = 'ascii'  # let pygments.format() do the right thing
         self.compress = get_choice_opt(options, 'compress',
                                        ['', 'none', 'gz', 'bz2'], '')
diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py
index 9ea2e22c..bd869044 100644
--- a/pygments/lexers/special.py
+++ b/pygments/lexers/special.py
@@ -36,6 +36,7 @@ _ttype_cache = {}
 
 line_re = re.compile(b'.*?\n')
 
+
 class RawTokenLexer(Lexer):
     """
     Recreate a token stream formatted with the `RawTokenFormatter`.  This
@@ -82,7 +83,7 @@ class RawTokenLexer(Lexer):
             try:
                 ttypestr, val = match.group().split(b'\t', 1)
             except ValueError:
-                val = match.group().decode(self.encoding)
+                val = match.group().decode('ascii', 'replace')
                 ttype = Error
             else:
                 ttype = _ttype_cache.get(ttypestr)