diff options
Diffstat (limited to 'pygments')
| -rw-r--r-- | pygments/lexers/special.py | 4 | ||||
| -rw-r--r-- | pygments/lexers/templates.py | 2 |
2 files changed, 5 insertions, 1 deletions
diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py index 14e21150..18a0e0cf 100644 --- a/pygments/lexers/special.py +++ b/pygments/lexers/special.py @@ -62,6 +62,7 @@ class RawTokenLexer(Lexer): return Lexer.get_tokens(self, text) def get_tokens_unprocessed(self, text): + length = 0 for match in line_re.finditer(text): ttypestr, val = match.group().split('\t', 1) ttype = _ttype_cache.get(ttypestr) @@ -72,4 +73,5 @@ class RawTokenLexer(Lexer): ttype = getattr(ttype, ttype_) _ttype_cache[ttypestr] = ttype val = val[1:-2].decode('string-escape') - yield 0, ttype, val + yield length, ttype, val + length += len(val) diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index 99d5ba48..ed9ea633 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -221,6 +221,8 @@ class DjangoLexer(RegexLexer): return rv +# Genshi lexers courtesy of Matt Good. + class GenshiTextLexer(RegexLexer): name = 'Genshi Text' aliases = ['genshitext'] |