Oops; doesnt handle UTF-8 correctly when reading from stream

1 files changed, 4 insertions, 3 deletions

diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index a0ce5d2..dc794ab 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -271,8 +271,9 @@ class Lexer(object):
         statetokens = tokendefs[statestack[-1]]
         known_names = {}
 
-        text = self._decode(stream.read(self.bufsize))
+        text = stream.read(self.bufsize)
         hasmore = len(text) == self.bufsize
+        text = self._decode(text)
 
         while 1:
             for rexmatch, action, new_state in statetokens:
@@ -317,9 +318,9 @@ class Lexer(object):
                     break
             else:
                 if hasmore:
-                    buf = self._decode(stream.read(self.bufsize))
+                    buf = stream.read(self.bufsize)
                     hasmore = len(buf) == self.bufsize
-                    text = text[pos:] + buf
+                    text = text[pos:] + self._decode(buf)
                     pos = 0
                     continue