Add encoding parameter to top-level functions (fixes issue20).

1 files changed, 5 insertions, 2 deletions

diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index e769d7b..4d200a6 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -224,7 +224,8 @@ class Lexer(object):
 
     def _decode(self, text):
         if sys.version_info[0] == 3:
-            return text
+            if isinstance(text, str):
+                return text
         if self.encoding == 'guess':
             try:
                 text = text.decode('utf-8')
@@ -355,11 +356,13 @@ class Lexer(object):
                     break
 
 
-def tokenize(sql):
+def tokenize(sql, encoding=None):
     """Tokenize sql.
 
     Tokenize *sql* using the :class:`Lexer` and return a 2-tuple stream
     of ``(token type, value)`` items.
     """
     lexer = Lexer()
+    if encoding is not None:
+        lexer.encoding = encoding
     return lexer.get_tokens(sql)