diff options
Diffstat (limited to 'sqlparse/lexer.py')
| -rw-r--r-- | sqlparse/lexer.py | 14 |
1 files changed, 8 insertions, 6 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 82a6169..a0ce5d2 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -279,6 +279,7 @@ class Lexer(object): m = rexmatch(text, pos) if m: if hasmore and m.end() == len(text): + # Since this is end, token may be truncated continue # print rex.pattern @@ -315,13 +316,14 @@ class Lexer(object): statetokens = tokendefs[statestack[-1]] break else: + if hasmore: + buf = self._decode(stream.read(self.bufsize)) + hasmore = len(buf) == self.bufsize + text = text[pos:] + buf + pos = 0 + continue + try: - if hasmore: - buf = self._decode(stream.read(self.bufsize)) - hasmore = len(buf) == self.bufsize - text = text[pos:] + buf - pos = 0 - continue if text[pos] == '\n': # at EOL, reset state to "root" pos += 1 |