diff options
| -rw-r--r-- | pygments/lexer.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/pygments/lexer.py b/pygments/lexer.py index 662c059e..8538fe7c 100644 --- a/pygments/lexer.py +++ b/pygments/lexer.py @@ -192,7 +192,9 @@ class Lexer(object): def get_tokens_unprocessed(self, text): """ - Return an iterable of (tokentype, value) pairs. + Return an iterable of (index, tokentype, value) pairs where "index" + is the starting position of the token within the input text. + In subclasses, implement this method as a generator to maximize effectiveness. """ |