diff options
Diffstat (limited to 'sqlparse/lexer.py')
| -rw-r--r-- | sqlparse/lexer.py | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index e7996b2..1979550 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -50,11 +50,14 @@ class Lexer(object): if not m: continue elif isinstance(action, tokens._TokenType): + consume_pos = m.end() - pos - 1 yield action, m.group() elif callable(action): - yield action(m.group()) + ttype, value = action(m.group(), text[pos:]) + consume_pos = len(value) - 1 + yield ttype, value - consume(iterable, m.end() - pos - 1) + consume(iterable, consume_pos) break else: yield tokens.Error, char |