[svn] Correct token indices in the raw token lexer.

2 files changed, 5 insertions, 1 deletions

diff --git a/pygments/lexers/special.py b/pygments/lexers/special.py
index 14e21150..18a0e0cf 100644
--- a/pygments/lexers/special.py
+++ b/pygments/lexers/special.py
@@ -62,6 +62,7 @@ class RawTokenLexer(Lexer):
         return Lexer.get_tokens(self, text)
 
     def get_tokens_unprocessed(self, text):
+        length = 0
         for match in line_re.finditer(text):
             ttypestr, val = match.group().split('\t', 1)
             ttype = _ttype_cache.get(ttypestr)
@@ -72,4 +73,5 @@ class RawTokenLexer(Lexer):
                     ttype = getattr(ttype, ttype_)
                 _ttype_cache[ttypestr] = ttype
             val = val[1:-2].decode('string-escape')
-            yield 0, ttype, val
+            yield length, ttype, val
+            length += len(val)
diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py
index 99d5ba48..ed9ea633 100644
--- a/pygments/lexers/templates.py
+++ b/pygments/lexers/templates.py
@@ -221,6 +221,8 @@ class DjangoLexer(RegexLexer):
         return rv
 
 
+# Genshi lexers courtesy of Matt Good.
+
 class GenshiTextLexer(RegexLexer):
     name = 'Genshi Text'
     aliases = ['genshitext']