From a8ded6465322b3b6f9b6eb7299172268b4e4bd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 15:03:33 +0200 Subject: Put common code from Python and PHP output filters in OutputFilter --- sqlparse/filters.py | 70 ++++++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 35 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 99ef80c..8f4b3dd 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -503,14 +503,28 @@ def Tokens2Unicode(stream): return result -class OutputPythonFilter(Filter): - +class OutputFilter(Filter): def __init__(self, varname='sql'): - self.varname = varname - self.cnt = 0 + self.varname = self.varname_prefix + varname + self.count = 0 - def _process(self, stream, varname, count, has_nl): - if count > 1: + def process(self, stack, stmt): + self.count += 1 + if self.count > 1: + varname = '%s%d' % (self.varname, self.count) + else: + varname = self.varname + + has_nl = len(unicode(stmt).strip().splitlines()) > 1 + stmt.tokens = self._process(stmt.tokens, varname, has_nl) + return stmt + + +class OutputPythonFilter(OutputFilter): + varname_prefix = '' + + def _process(self, stream, varname, has_nl): + if self.count > 1: yield sql.Token(T.Whitespace, '\n') yield sql.Token(T.Name, varname) yield sql.Token(T.Whitespace, ' ') @@ -519,6 +533,7 @@ class OutputPythonFilter(Filter): if has_nl: yield sql.Token(T.Operator, '(') yield sql.Token(T.Text, "'") + cnt = 0 for token in stream: cnt += 1 @@ -528,74 +543,59 @@ class OutputPythonFilter(Filter): after_lb = token.value.split('\n', 1)[1] yield sql.Token(T.Text, " '") yield sql.Token(T.Whitespace, '\n') - for i in range(len(varname) + 4): - yield sql.Token(T.Whitespace, ' ') + + yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4)) yield sql.Token(T.Text, "'") if after_lb: # it's the indendation yield sql.Token(T.Whitespace, after_lb) continue + elif token.value and "'" in token.value: token.value = token.value.replace("'", "\\'") yield sql.Token(T.Text, token.value or '') + yield sql.Token(T.Text, "'") if has_nl: yield sql.Token(T.Operator, ')') - def process(self, stack, stmt): - self.cnt += 1 - if self.cnt > 1: - varname = '%s%d' % (self.varname, self.cnt) - else: - varname = self.varname - has_nl = len(unicode(stmt).strip().splitlines()) > 1 - stmt.tokens = self._process(stmt.tokens, varname, self.cnt, has_nl) - return stmt - - -class OutputPHPFilter(Filter): - def __init__(self, varname='sql'): - self.varname = '$%s' % varname - self.count = 0 +class OutputPHPFilter(OutputFilter): + varname_prefix = '$' - def _process(self, stream, varname): + def _process(self, stream, varname, has_nl): if self.count > 1: yield sql.Token(T.Whitespace, '\n') yield sql.Token(T.Name, varname) yield sql.Token(T.Whitespace, ' ') + if has_nl: + yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Operator, '=') yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Text, '"') + for token in stream: if token.is_whitespace() and '\n' in token.value: after_lb = token.value.split('\n', 1)[1] yield sql.Token(T.Text, ' "') yield sql.Token(T.Operator, ';') yield sql.Token(T.Whitespace, '\n') + yield sql.Token(T.Name, varname) yield sql.Token(T.Whitespace, ' ') - yield sql.Token(T.Punctuation, '.') - yield sql.Token(T.Operator, '=') + yield sql.Token(T.Operator, '.=') yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Text, '"') if after_lb: yield sql.Token(T.Text, after_lb) continue + elif '"' in token.value: token.value = token.value.replace('"', '\\"') yield sql.Token(T.Text, token.value) + yield sql.Token(T.Text, '"') yield sql.Token(T.Punctuation, ';') - def process(self, stack, stmt): - self.count += 1 - if self.count > 1: - varname = '%s%d' % (self.varname, self.count) - else: - varname = self.varname - stmt.tokens = tuple(self._process(stmt.tokens, varname)) - return stmt - class Limit(Filter): """Get the LIMIT of a query. -- cgit v1.2.1 From 0c438d34bfe7b9b2b4984ceb09ce39416c446ccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 15:05:53 +0200 Subject: Removed Pep8 warnings --- sqlparse/filters.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 8f4b3dd..3906bad 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -503,11 +503,16 @@ def Tokens2Unicode(stream): return result -class OutputFilter(Filter): +class OutputFilter(TokenFilter): + varname_prefix = '' + def __init__(self, varname='sql'): self.varname = self.varname_prefix + varname self.count = 0 + def _process(self, stream, varname, has_nl): + raise NotImplementedError + def process(self, stack, stmt): self.count += 1 if self.count > 1: @@ -521,8 +526,6 @@ class OutputFilter(Filter): class OutputPythonFilter(OutputFilter): - varname_prefix = '' - def _process(self, stream, varname, has_nl): if self.count > 1: yield sql.Token(T.Whitespace, '\n') -- cgit v1.2.1 From 95ae0ac9a308df72a547911a423cc1ddadccc60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 17:07:55 +0200 Subject: Clean-up and optimizations --- sqlparse/filters.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 3906bad..8f913b0 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -527,6 +527,7 @@ class OutputFilter(TokenFilter): class OutputPythonFilter(OutputFilter): def _process(self, stream, varname, has_nl): + # SQL query asignation to varname if self.count > 1: yield sql.Token(T.Whitespace, '\n') yield sql.Token(T.Name, varname) @@ -537,18 +538,16 @@ class OutputPythonFilter(OutputFilter): yield sql.Token(T.Operator, '(') yield sql.Token(T.Text, "'") - cnt = 0 for token in stream: - cnt += 1 if token.is_whitespace() and '\n' in token.value: - if cnt == 1: - continue - after_lb = token.value.split('\n', 1)[1] yield sql.Token(T.Text, " '") yield sql.Token(T.Whitespace, '\n') + # Quote header on secondary lines yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4)) yield sql.Token(T.Text, "'") + + after_lb = token.value.split('\n', 1)[1] if after_lb: # it's the indendation yield sql.Token(T.Whitespace, after_lb) continue @@ -557,6 +556,7 @@ class OutputPythonFilter(OutputFilter): token.value = token.value.replace("'", "\\'") yield sql.Token(T.Text, token.value or '') + # Close quote yield sql.Token(T.Text, "'") if has_nl: yield sql.Token(T.Operator, ')') @@ -566,6 +566,7 @@ class OutputPHPFilter(OutputFilter): varname_prefix = '$' def _process(self, stream, varname, has_nl): + # SQL query asignation to varname (quote header) if self.count > 1: yield sql.Token(T.Whitespace, '\n') yield sql.Token(T.Name, varname) @@ -578,16 +579,18 @@ class OutputPHPFilter(OutputFilter): for token in stream: if token.is_whitespace() and '\n' in token.value: - after_lb = token.value.split('\n', 1)[1] yield sql.Token(T.Text, ' "') yield sql.Token(T.Operator, ';') yield sql.Token(T.Whitespace, '\n') + # Quote header on secondary lines yield sql.Token(T.Name, varname) yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Operator, '.=') yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Text, '"') + + after_lb = token.value.split('\n', 1)[1] if after_lb: yield sql.Token(T.Text, after_lb) continue @@ -596,6 +599,7 @@ class OutputPHPFilter(OutputFilter): token.value = token.value.replace('"', '\\"') yield sql.Token(T.Text, token.value) + # Close quote yield sql.Token(T.Text, '"') yield sql.Token(T.Punctuation, ';') -- cgit v1.2.1 From f64954c1bf90ecb6af314a2e61df753ef7c8fd7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 17:46:51 +0200 Subject: Finished homogeneization --- sqlparse/filters.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 8f913b0..6719d6f 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -538,8 +538,11 @@ class OutputPythonFilter(OutputFilter): yield sql.Token(T.Operator, '(') yield sql.Token(T.Text, "'") + # Print the tokens on the quote for token in stream: + # Token is a new line separator if token.is_whitespace() and '\n' in token.value: + # Close quote and add a new line yield sql.Token(T.Text, " '") yield sql.Token(T.Whitespace, '\n') @@ -547,14 +550,18 @@ class OutputPythonFilter(OutputFilter): yield sql.Token(T.Whitespace, ' ' * (len(varname) + 4)) yield sql.Token(T.Text, "'") + # Indentation after_lb = token.value.split('\n', 1)[1] - if after_lb: # it's the indendation + if after_lb: yield sql.Token(T.Whitespace, after_lb) continue - elif token.value and "'" in token.value: + # Token has escape chars + elif "'" in token.value: token.value = token.value.replace("'", "\\'") - yield sql.Token(T.Text, token.value or '') + + # Put the token + yield sql.Token(T.Text, token.value) # Close quote yield sql.Token(T.Text, "'") @@ -577,10 +584,12 @@ class OutputPHPFilter(OutputFilter): yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Text, '"') + # Print the tokens on the quote for token in stream: + # Token is a new line separator if token.is_whitespace() and '\n' in token.value: - yield sql.Token(T.Text, ' "') - yield sql.Token(T.Operator, ';') + # Close quote and add a new line + yield sql.Token(T.Text, ' ";') yield sql.Token(T.Whitespace, '\n') # Quote header on secondary lines @@ -590,13 +599,17 @@ class OutputPHPFilter(OutputFilter): yield sql.Token(T.Whitespace, ' ') yield sql.Token(T.Text, '"') + # Indentation after_lb = token.value.split('\n', 1)[1] if after_lb: - yield sql.Token(T.Text, after_lb) + yield sql.Token(T.Whitespace, after_lb) continue + # Token has escape chars elif '"' in token.value: token.value = token.value.replace('"', '\\"') + + # Put the token yield sql.Token(T.Text, token.value) # Close quote -- cgit v1.2.1 From 4f29ddb9de8d7d39859c908444bc831d16fba80f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 18:01:41 +0200 Subject: Unified Filter and TokenFilter (they were equal --- sqlparse/filters.py | 28 +++++++++++----------------- 1 file changed, 11 insertions(+), 17 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/filters.py b/sqlparse/filters.py index 6719d6f..fbbdaab 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -11,13 +11,7 @@ from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation, String, Whitespace) -class Filter(object): - - def process(self, *args): - raise NotImplementedError - - -class TokenFilter(Filter): +class TokenFilter: def process(self, stack, stream): raise NotImplementedError @@ -57,7 +51,7 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value -class GetComments(Filter): +class GetComments(TokenFilter): """Get the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -65,7 +59,7 @@ class GetComments(Filter): yield token_type, value -class StripComments(Filter): +class StripComments(TokenFilter): """Strip the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -101,7 +95,7 @@ def StripWhitespace(stream): last_type = token_type -class IncludeStatement(Filter): +class IncludeStatement(TokenFilter): """Filter that enable a INCLUDE statement""" def __init__(self, dirpath=".", maxRecursive=10): @@ -163,7 +157,7 @@ class IncludeStatement(Filter): # ---------------------- # statement process -class StripCommentsFilter(Filter): +class StripCommentsFilter(TokenFilter): def _get_next_comment(self, tlist): # TODO(andi) Comment types should be unified, see related issue38 @@ -194,7 +188,7 @@ class StripCommentsFilter(Filter): self._process(stmt) -class StripWhitespaceFilter(Filter): +class StripWhitespaceFilter(TokenFilter): def _stripws(self, tlist): func_name = '_stripws_%s' % tlist.__class__.__name__.lower() @@ -226,7 +220,7 @@ class StripWhitespaceFilter(Filter): stmt.tokens.pop(-1) -class ReindentFilter(Filter): +class ReindentFilter(TokenFilter): def __init__(self, width=2, char=' ', line_width=None): self.width = width @@ -391,7 +385,7 @@ class ReindentFilter(Filter): # FIXME: Doesn't work ;) -class RightMarginFilter(Filter): +class RightMarginFilter(TokenFilter): keep_together = ( # sql.TypeCast, sql.Identifier, sql.Alias, @@ -429,7 +423,7 @@ class RightMarginFilter(Filter): group.tokens = self._process(stack, group, group.tokens) -class ColumnsSelect(Filter): +class ColumnsSelect(TokenFilter): """Get the columns names of a SELECT query""" def process(self, stack, stream): mode = 0 @@ -483,7 +477,7 @@ class ColumnsSelect(Filter): # --------------------------- # postprocess -class SerializerUnicode(Filter): +class SerializerUnicode(TokenFilter): def process(self, stack, stmt): raw = unicode(stmt) @@ -617,7 +611,7 @@ class OutputPHPFilter(OutputFilter): yield sql.Token(T.Punctuation, ';') -class Limit(Filter): +class Limit(TokenFilter): """Get the LIMIT of a query. If not defined, return -1 (SQL specification for no LIMIT query) -- cgit v1.2.1 From 7d85204ab35d004739042feccdfa36ed2d26ff28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 18:07:31 +0200 Subject: Removed TokenFilter and using duck-typing instead (more pythonic) --- sqlparse/engine/filter.py | 13 +------------ sqlparse/filters.py | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 30 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/engine/filter.py b/sqlparse/engine/filter.py index 9ea9703..9af2f99 100644 --- a/sqlparse/engine/filter.py +++ b/sqlparse/engine/filter.py @@ -4,21 +4,10 @@ from sqlparse.sql import Statement, Token from sqlparse import tokens as T -class TokenFilter(object): - - def __init__(self, **options): - self.options = options - - def process(self, stack, stream): - """Process token stream.""" - raise NotImplementedError - - -class StatementFilter(TokenFilter): +class StatementFilter: "Filter that split stream at individual statements" def __init__(self): - TokenFilter.__init__(self) self._in_declare = False self._in_dbldollar = False self._is_create = False diff --git a/sqlparse/filters.py b/sqlparse/filters.py index fbbdaab..291a613 100644 --- a/sqlparse/filters.py +++ b/sqlparse/filters.py @@ -11,16 +11,10 @@ from sqlparse.tokens import (Comment, Comparison, Keyword, Name, Punctuation, String, Whitespace) -class TokenFilter: - - def process(self, stack, stream): - raise NotImplementedError - - # -------------------------- # token process -class _CaseFilter(TokenFilter): +class _CaseFilter: ttype = None @@ -51,7 +45,7 @@ class IdentifierCaseFilter(_CaseFilter): yield ttype, value -class GetComments(TokenFilter): +class GetComments: """Get the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -59,7 +53,7 @@ class GetComments(TokenFilter): yield token_type, value -class StripComments(TokenFilter): +class StripComments: """Strip the comments from a stack""" def process(self, stack, stream): for token_type, value in stream: @@ -95,7 +89,7 @@ def StripWhitespace(stream): last_type = token_type -class IncludeStatement(TokenFilter): +class IncludeStatement: """Filter that enable a INCLUDE statement""" def __init__(self, dirpath=".", maxRecursive=10): @@ -157,7 +151,7 @@ class IncludeStatement(TokenFilter): # ---------------------- # statement process -class StripCommentsFilter(TokenFilter): +class StripCommentsFilter: def _get_next_comment(self, tlist): # TODO(andi) Comment types should be unified, see related issue38 @@ -188,7 +182,7 @@ class StripCommentsFilter(TokenFilter): self._process(stmt) -class StripWhitespaceFilter(TokenFilter): +class StripWhitespaceFilter: def _stripws(self, tlist): func_name = '_stripws_%s' % tlist.__class__.__name__.lower() @@ -220,7 +214,7 @@ class StripWhitespaceFilter(TokenFilter): stmt.tokens.pop(-1) -class ReindentFilter(TokenFilter): +class ReindentFilter: def __init__(self, width=2, char=' ', line_width=None): self.width = width @@ -385,7 +379,7 @@ class ReindentFilter(TokenFilter): # FIXME: Doesn't work ;) -class RightMarginFilter(TokenFilter): +class RightMarginFilter: keep_together = ( # sql.TypeCast, sql.Identifier, sql.Alias, @@ -423,7 +417,7 @@ class RightMarginFilter(TokenFilter): group.tokens = self._process(stack, group, group.tokens) -class ColumnsSelect(TokenFilter): +class ColumnsSelect: """Get the columns names of a SELECT query""" def process(self, stack, stream): mode = 0 @@ -477,7 +471,7 @@ class ColumnsSelect(TokenFilter): # --------------------------- # postprocess -class SerializerUnicode(TokenFilter): +class SerializerUnicode: def process(self, stack, stmt): raw = unicode(stmt) @@ -497,7 +491,7 @@ def Tokens2Unicode(stream): return result -class OutputFilter(TokenFilter): +class OutputFilter: varname_prefix = '' def __init__(self, varname='sql'): @@ -611,7 +605,7 @@ class OutputPHPFilter(OutputFilter): yield sql.Token(T.Punctuation, ';') -class Limit(TokenFilter): +class Limit: """Get the LIMIT of a query. If not defined, return -1 (SQL specification for no LIMIT query) -- cgit v1.2.1 From 52e02fd771f46a5ad16e9dac92c44880c7be25db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro=20=22Piranna=22?= Date: Sat, 28 Apr 2012 18:20:13 +0200 Subject: Some little optimizations --- sqlparse/sql.py | 55 ++++++++++++++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 27 deletions(-) (limited to 'sqlparse') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 31fa34d..4941e3d 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -38,7 +38,7 @@ class Token(object): def to_unicode(self): """Returns a unicode representation of this object. - + @deprecated: please use __unicode__() """ return unicode(self) @@ -49,10 +49,8 @@ class Token(object): def _get_repr_value(self): raw = unicode(self) if len(raw) > 7: - short = raw[:6] + u'...' - else: - short = raw - return re.sub('\s+', ' ', short) + raw = raw[:6] + u'...' + return re.sub('\s+', ' ', raw) def flatten(self): """Resolve subgroups.""" @@ -73,31 +71,34 @@ class Token(object): type_matched = self.ttype is ttype if not type_matched or values is None: return type_matched + if regex: if isinstance(values, basestring): values = set([values]) + if self.ttype is T.Keyword: - values = set([re.compile(v, re.IGNORECASE) for v in values]) + values = set(re.compile(v, re.IGNORECASE) for v in values) else: - values = set([re.compile(v) for v in values]) + values = set(re.compile(v) for v in values) + for pattern in values: if pattern.search(self.value): return True return False - else: - if isinstance(values, basestring): - if self.is_keyword: - return values.upper() == self.normalized - else: - return values == self.value + + if isinstance(values, basestring): if self.is_keyword: - for v in values: - if v.upper() == self.normalized: - return True - return False - else: - print len(values) - return self.value in values + return values.upper() == self.normalized + return values == self.value + + if self.is_keyword: + for v in values: + if v.upper() == self.normalized: + return True + return False + + print len(values) + return self.value in values def is_group(self): """Returns ``True`` if this object has children.""" @@ -271,7 +272,7 @@ class TokenList(Token): if not isinstance(idx, int): idx = self.token_index(idx) - while idx != 0: + while idx: idx -= 1 if self.tokens[idx].is_whitespace() and skip_ws: continue @@ -379,12 +380,12 @@ class TokenList(Token): dot = self.token_next_match(0, T.Punctuation, '.') if dot is None: return self.token_next_by_type(0, T.Name).value - else: - next_ = self.token_next_by_type(self.token_index(dot), - (T.Name, T.Wildcard)) - if next_ is None: # invalid identifier, e.g. "a." - return None - return next_.value + + next_ = self.token_next_by_type(self.token_index(dot), + (T.Name, T.Wildcard)) + if next_ is None: # invalid identifier, e.g. "a." + return None + return next_.value class Statement(TokenList): -- cgit v1.2.1 From c45e0c62373407544c089ada7240c99da79ff29e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Legan=C3=A9s=20Combarro?= Date: Wed, 16 May 2012 02:06:44 +0300 Subject: Removed debug code --- sqlparse/sql.py | 1 - 1 file changed, 1 deletion(-) (limited to 'sqlparse') diff --git a/sqlparse/sql.py b/sqlparse/sql.py index 4941e3d..05e078d 100644 --- a/sqlparse/sql.py +++ b/sqlparse/sql.py @@ -97,7 +97,6 @@ class Token(object): return True return False - print len(values) return self.value in values def is_group(self): -- cgit v1.2.1