diff options
Diffstat (limited to 'pygments')
| -rw-r--r-- | pygments/lexers/__init__.py | 2 | ||||
| -rw-r--r-- | pygments/lexers/special.py | 25 |
2 files changed, 16 insertions, 11 deletions
diff --git a/pygments/lexers/__init__.py b/pygments/lexers/__init__.py index 3b405f88..96e68a28 100644 --- a/pygments/lexers/__init__.py +++ b/pygments/lexers/__init__.py @@ -91,7 +91,7 @@ def _iter_lexerclasses(): """ Returns an iterator over all lexer classes. """ - for module_name, name, _, _ in LEXERS.itervalues(): + for module_name, name, _, _, _ in LEXERS.itervalues(): if name not in _lexer_cache: _load_lexers(module_name) yield _lexer_cache[name] diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 621145a2..dcf9899c 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -13,7 +13,7 @@ import re import cStringIO from pygments.lexer import Lexer, RegexLexer -from pygments.token import Token, \ +from pygments.token import Token, Error, \ Text, Comment, Operator, Keyword, Name, String, Number @@ -66,14 +66,19 @@ class RawTokenLexer(Lexer): def get_tokens_unprocessed(self, text): length = 0 for match in line_re.finditer(text): - ttypestr, val = match.group().split('\t', 1) - ttype = _ttype_cache.get(ttypestr) - if not ttype: - ttype = Token - ttypes = ttypestr.split('.')[1:] - for ttype_ in ttypes: - ttype = getattr(ttype, ttype_) - _ttype_cache[ttypestr] = ttype - val = val[1:-2].decode('string-escape') + try: + ttypestr, val = match.group().split('\t', 1) + except ValueError: + val = match.group() + ttype = Error + else: + ttype = _ttype_cache.get(ttypestr) + if not ttype: + ttype = Token + ttypes = ttypestr.split('.')[1:] + for ttype_ in ttypes: + ttype = getattr(ttype, ttype_) + _ttype_cache[ttypestr] = ttype + val = val[1:-2].decode('string-escape') yield length, ttype, val length += len(val) |