Merge branch 'master' of github.com:andialbrecht/sqlparse

11 files changed, 112 insertions, 68 deletions

diff --git a/.hgignore b/.gitignore
index 7444a39..77fe44c 100644
--- a/.hgignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-syntax: glob
 *.pyc
 docs/build
 dist
diff --git a/.hgtags b/.hgtags
deleted file mode 100644
index 1cdbba6..0000000
--- a/.hgtags
+++ /dev/null
@@ -1,4 +0,0 @@
-49f461d2899eb3175ba7d935337e5938403372d4 0.1.0
-294617d5c43131aacd11ddccc9ccb238a41bf563 0.1.1
-079b282ad3ee0d620128ee64e2bba5791e733b62 0.1.2
-ac9ea581527a4540d46d766c07b23a88dd04704c 0.1.3
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..08cf9ad
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,6 @@
+language: python
+python:
+  - "2.5"
+  - "2.6"
+  - "2.7"
+script: python tests/run_tests.py
\ No newline at end of file
diff --git a/CHANGES b/CHANGES
index be82b6d..06a4949 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,12 +1,18 @@
-Development
------------
+Release 0.1.4 (Apr 20, 2012)
+----------------------------
 
 Bug Fixes
- * Avoid "stair case" effects when identifiers and functions are mixed
-   in identifier lists (issue45) and when asterisks are used as
-   operators (issue58).
+ * Avoid "stair case" effects when identifiers, functions,
+   placeholders or keywords are mixed in identifier lists (issue45,
+   issue49, issue52) and when asterisks are used as operators
+   (issue58).
  * Make keyword detection more restrict (issue47).
  * Improve handling of CASE statements (issue46).
+ * Fix statement splitting when parsing recursive statements (issue57,
+   thanks to piranna).
+ * Fix for negative numbers (issue56, thanks to kevinjqiu).
+ * Pretty format comments in identifier lists (issue59).
+ * Several minor bug fixes and improvements.
 
 
 Release 0.1.3 (Jul 29, 2011)
diff --git a/README b/README
deleted file mode 100644
index 2ec6b02..0000000
--- a/README
+++ /dev/null
@@ -1,37 +0,0 @@
-python-sqlparse - Parse SQL statements
-======================================
-
-sqlparse is a non-validating SQL parser module for Python.
-
-
-Install
--------
-
-Run
-
-  python setup.py install
-
-to install python-sqlparse on your system.
-
-
-Run Tests
----------
-
-  python test/run_tests.py
-
-
-Links
------
-
-Project Page:  http://python-sqlparse.googlecode.com
-Documentation: http://python-sqlparse.googlecode.com/svn/docs/api/index.html
-Discussions:   http://groups.google.com/group/sqlparse
-Issues/Bugs:   http://code.google.com/p/python-sqlparse/issues/list
-Online Demo:   http://sqlformat.appspot.com
-
-
-python-sqlparse is licensed under the BSD license.
-
-Parts of the code are based on pygments written by Georg Brandl and others.
-pygments-Homepage: http://pygments.org/
-
diff --git a/README.rst b/README.rst
new file mode 100644
index 0000000..0b9086a
--- /dev/null
+++ b/README.rst
@@ -0,0 +1,52 @@
+python-sqlparse - Parse SQL statements
+======================================
+
+sqlparse is a non-validating SQL parser module for Python.
+
+|buildstatus|_
+
+
+Install
+-------
+
+Run::
+
+  python setup.py install
+
+to install python-sqlparse on your system.
+
+
+Run Tests
+---------
+
+To run the test suite::
+
+  python test/run_tests.py
+
+
+Links
+-----
+
+Project Page
+  https://github.com/andialbrecht/sqlparse
+
+Documentation
+  http://readthedocs.org/docs/sqlparse/en/latest/
+
+Discussions
+  http://groups.google.com/group/sqlparse
+
+Issues/Bugs
+  https://github.com/andialbrecht/sqlparse/issues
+
+Online Demo
+  http://sqlformat.appspot.com
+
+
+python-sqlparse is licensed under the BSD license.
+
+Parts of the code are based on pygments written by Georg Brandl and others.
+pygments-Homepage: http://pygments.org/
+
+.. |buildstatus| image:: https://secure.travis-ci.org/andialbrecht/sqlparse.png?branch=master
+.. _buildstatus: http://travis-ci.org/#!/andialbrecht/sqlparse
diff --git a/sqlparse/__init__.py b/sqlparse/__init__.py
index 5ccf092..f924c04 100644
--- a/sqlparse/__init__.py
+++ b/sqlparse/__init__.py
@@ -6,7 +6,7 @@
 """Parse SQL statements."""
 
 
-__version__ = '0.1.3'
+__version__ = '0.1.4'
 
 
 class SQLParseError(Exception):
@@ -58,4 +58,4 @@ def split(sql):
 from sqlparse.engine.filter import StatementFilter
 def split2(stream):
     splitter = StatementFilter()
-    return list(splitter.process(None, stream))
\ No newline at end of file
+    return list(splitter.process(None, stream))
diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index b0c6c2c..55ec7e2 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -194,9 +194,12 @@ def group_identifier_list(tlist):
                    lambda t: t.ttype == T.Name,
                    lambda t: t.ttype == T.Wildcard,
                    lambda t: t.match(T.Keyword, 'null'),
+                   lambda t: t.match(T.Keyword, 'role'),
                    lambda t: t.ttype == T.Number.Integer,
                    lambda t: t.ttype == T.String.Single,
+                   lambda t: t.ttype == T.Name.Placeholder,
                    lambda t: isinstance(t, sql.Comparison),
+                   lambda t: isinstance(t, sql.Comment),
                    ]
     tcomma = tlist.token_next_match(idx, T.Punctuation, ',')
     start = None
@@ -314,18 +317,19 @@ def group_functions(tlist):
 
 
 def group(tlist):
-    for func in [group_parenthesis,
-                 group_functions,
-                 group_comments,
-                 group_where,
-                 group_case,
-                 group_identifier,
-                 group_typecasts,
-                 group_as,
-                 group_aliased,
-                 group_assignment,
-                 group_comparison,
-                 group_identifier_list,
-                 group_if,
-                 group_for]:
+    for func in [
+            group_comments,
+            group_parenthesis,
+            group_functions,
+            group_where,
+            group_case,
+            group_identifier,
+            group_typecasts,
+            group_as,
+            group_aliased,
+            group_assignment,
+            group_comparison,
+            group_identifier_list,
+            group_if,
+            group_for]:
         func(tlist)
diff --git a/sqlparse/filters.py b/sqlparse/filters.py
index 6bb3415..bfa757d 100644
--- a/sqlparse/filters.py
+++ b/sqlparse/filters.py
@@ -218,10 +218,11 @@ class StripWhitespaceFilter(Filter):
             tlist.tokens.pop(-2)
         self._stripws_default(tlist)
 
-    def process(self, stack, stmt):
-        [self.process(stack, sgroup) for sgroup in stmt.get_sublists()]
+    def process(self, stack, stmt, depth=0):
+        [self.process(stack, sgroup, depth+1)
+         for sgroup in stmt.get_sublists()]
         self._stripws(stmt)
-        if stmt.tokens[-1].is_whitespace():
+        if depth == 0 and stmt.tokens[-1].is_whitespace():
             stmt.tokens.pop(-1)
 
 
@@ -331,6 +332,9 @@ class ReindentFilter(Filter):
             self.offset += num_offset
             for token in identifiers[1:]:
                 tlist.insert_before(token, self.nl())
+            for token in tlist.tokens:
+                if isinstance(token, sql.Comment):
+                    tlist.insert_after(token, self.nl())
             self.offset -= num_offset
         self._process_default(tlist)
 
@@ -610,4 +614,4 @@ class Limit(Filter):
             if index and token_type in Keyword and value == 'LIMIT':
                 return stream[4 - index][1]
 
-        return -1
\ No newline at end of file
+        return -1
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 72609e2..9c7aeee 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -137,7 +137,7 @@ class TokenList(Token):
         if tokens is None:
             tokens = []
         self.tokens = tokens
-        Token.__init__(self, None, None)
+        Token.__init__(self, None, unicode(self))
 
     def __unicode__(self):
         return ''.join(unicode(x) for x in self.flatten())
@@ -322,6 +322,14 @@ class TokenList(Token):
         """Inserts *token* before *where*."""
         self.tokens.insert(self.token_index(where), token)
 
+    def insert_after(self, where, token):
+        """Inserts *token* after *where*."""
+        next_token = self.token_next(where)
+        if next_token is None:
+            self.tokens.append(token)
+        else:
+            self.tokens.insert(self.token_index(next_token), token)
+
     def has_alias(self):
         """Returns ``True`` if an alias is present."""
         return self.get_alias() is not None
diff --git a/tests/test_format.py b/tests/test_format.py
index 7a2c655..c9c6f46 100644
--- a/tests/test_format.py
+++ b/tests/test_format.py
@@ -68,6 +68,12 @@ class TestFormat(TestCaseBase):
         self.assertRaises(sqlparse.SQLParseError, sqlparse.format, s,
                           strip_whitespace=None)
 
+    def test_preserve_ws(self):
+        # preserve at least one whitespace after subgroups
+        f = lambda sql: sqlparse.format(sql, strip_whitespace=True)
+        s = 'select\n* /* foo */  from bar '
+        self.ndiffAssertEqual(f(s), 'select * /* foo */ from bar')
+
     def test_outputformat(self):
         sql = 'select * from foo;'
         self.assertRaises(sqlparse.SQLParseError, sqlparse.format, sql,