diff options
Diffstat (limited to 'pygments/lexers/special.py')
| -rw-r--r-- | pygments/lexers/special.py | 25 |
1 files changed, 15 insertions, 10 deletions
diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 621145a2..dcf9899c 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -13,7 +13,7 @@ import re import cStringIO from pygments.lexer import Lexer, RegexLexer -from pygments.token import Token, \ +from pygments.token import Token, Error, \ Text, Comment, Operator, Keyword, Name, String, Number @@ -66,14 +66,19 @@ class RawTokenLexer(Lexer): def get_tokens_unprocessed(self, text): length = 0 for match in line_re.finditer(text): - ttypestr, val = match.group().split('\t', 1) - ttype = _ttype_cache.get(ttypestr) - if not ttype: - ttype = Token - ttypes = ttypestr.split('.')[1:] - for ttype_ in ttypes: - ttype = getattr(ttype, ttype_) - _ttype_cache[ttypestr] = ttype - val = val[1:-2].decode('string-escape') + try: + ttypestr, val = match.group().split('\t', 1) + except ValueError: + val = match.group() + ttype = Error + else: + ttype = _ttype_cache.get(ttypestr) + if not ttype: + ttype = Token + ttypes = ttypestr.split('.')[1:] + for ttype_ in ttypes: + ttype = getattr(ttype, ttype_) + _ttype_cache[ttypestr] = ttype + val = val[1:-2].decode('string-escape') yield length, ttype, val length += len(val) |