[svn] Fix unittest.

1 files changed, 7 insertions, 2 deletions

diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py
index 28c382ab..5af9320a 100644
--- a/pygments/lexers/special.py
+++ b/pygments/lexers/special.py
@@ -60,7 +60,12 @@ class RawTokenLexer(Lexer):
         elif self.compress == 'bz2':
             import bz2
             text = bz2.decompress(text)
-        return Lexer.get_tokens(self, text)
+
+        # do not call Lexer.get_tokens() because we do not want Unicode
+        # decoding to occur, and stripping is not optional.
+        text = text.strip('\n') + '\n'
+        for i, t, v in self.get_tokens_unprocessed(text):
+            yield t, v
 
     def get_tokens_unprocessed(self, text):
         length = 0
@@ -68,7 +73,7 @@ class RawTokenLexer(Lexer):
             try:
                 ttypestr, val = match.group().split('\t', 1)
             except ValueError:
-                val = match.group()
+                val = match.group().decode(self.encoding)
                 ttype = Error
             else:
                 ttype = _ttype_cache.get(ttypestr)