diff options
author | Andi Albrecht <albrecht.andi@gmail.com> | 2013-10-23 10:27:33 +0200 |
---|---|---|
committer | Andi Albrecht <albrecht.andi@gmail.com> | 2013-10-23 10:27:33 +0200 |
commit | 2aa7f0c68f5ba77b347e216994fd01be63e8aeb0 (patch) | |
tree | 0f143e2c706f565c9abff233ff162eea0d873060 /sqlparse/lexer.py | |
parent | a1aa080aece81d62b0f971c4577eb4a91391db10 (diff) | |
download | sqlparse-2aa7f0c68f5ba77b347e216994fd01be63e8aeb0.tar.gz |
Remove buffered reading again (fixes #114).
It causes problems with some sources.
Diffstat (limited to 'sqlparse/lexer.py')
-rw-r--r-- | sqlparse/lexer.py | 22 |
1 files changed, 1 insertions, 21 deletions
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py index 45ba4cb..a8e66f9 100644 --- a/sqlparse/lexer.py +++ b/sqlparse/lexer.py @@ -161,9 +161,6 @@ class Lexer(object): stripnl = False tabsize = 0 flags = re.IGNORECASE | re.UNICODE - DEFAULT_BUFSIZE = 4096 - MAX_BUFSIZE = 2 ** 31 - bufsize = DEFAULT_BUFSIZE tokens = { 'root': [ @@ -286,18 +283,13 @@ class Lexer(object): statetokens = tokendefs[statestack[-1]] known_names = {} - text = stream.read(self.bufsize) - hasmore = len(text) == self.bufsize + text = stream.read() text = self._decode(text) while 1: for rexmatch, action, new_state in statetokens: m = rexmatch(text, pos) if m: - if hasmore and m.end() == len(text): - # Since this is end, token may be truncated - continue - # print rex.pattern value = m.group() if value in known_names: @@ -330,20 +322,8 @@ class Lexer(object): else: assert False, "wrong state def: %r" % new_state statetokens = tokendefs[statestack[-1]] - # reset bufsize - self.bufsize = self.DEFAULT_BUFSIZE break else: - if hasmore: - # we have no match, increase bufsize to parse lengthy - # tokens faster (see #86). - self.bufsize = min(self.bufsize * 2, self.MAX_BUFSIZE) - buf = stream.read(self.bufsize) - hasmore = len(buf) == self.bufsize - text = text[pos:] + self._decode(buf) - pos = 0 - continue - try: if text[pos] == '\n': # at EOL, reset state to "root" |