diff options
Diffstat (limited to 'pygments/lexers/special.py')
| -rw-r--r-- | pygments/lexers/special.py | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 28c382ab..5af9320a 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -60,7 +60,12 @@ class RawTokenLexer(Lexer): elif self.compress == 'bz2': import bz2 text = bz2.decompress(text) - return Lexer.get_tokens(self, text) + + # do not call Lexer.get_tokens() because we do not want Unicode + # decoding to occur, and stripping is not optional. + text = text.strip('\n') + '\n' + for i, t, v in self.get_tokens_unprocessed(text): + yield t, v def get_tokens_unprocessed(self, text): length = 0 @@ -68,7 +73,7 @@ class RawTokenLexer(Lexer): try: ttypestr, val = match.group().split('\t', 1) except ValueError: - val = match.group() + val = match.group().decode(self.encoding) ttype = Error else: ttype = _ttype_cache.get(ttypestr) |