fix: Fixes unicode problems with request.query and request.forms (fix #342 fix #344 fix #339)

2 files changed, 30 insertions, 7 deletions

diff --git a/bottle.py b/bottle.py
index 74eaf54..42b35ca 100755
--- a/bottle.py
+++ b/bottle.py
@@ -70,10 +70,6 @@ try: from collections import MutableMapping as DictMixin
 except ImportError: # pragma: no cover
     from UserDict import DictMixin
 
-try: from urlparse import parse_qsl
-except ImportError: # pragma: no cover
-    from cgi import parse_qsl
-
 try: import cPickle as pickle
 except ImportError: # pragma: no cover
     import pickle
@@ -98,6 +94,7 @@ if sys.version_info < (2,6,0):
 
 if py3k: # pragma: no cover
     json_loads = lambda s: json_lds(touni(s))
+    urlunquote = functools.partial(urlunquote, encoding='latin1')
     # See Request.POST
     from io import BytesIO
     def touni(x, enc='utf8', err='strict'):
@@ -930,8 +927,8 @@ class BaseRequest(DictMixin):
             values are sometimes called "URL arguments" or "GET parameters", but
             not to be confused with "URL wildcards" as they are provided by the
             :class:`Router`. '''
-        pairs = parse_qsl(self.query_string, keep_blank_values=True)
         get = self.environ['bottle.get'] = FormsDict()
+        pairs = _parse_qsl(self.environ.get('QUERY_STRING', ''))
         for key, value in pairs[:self.MAX_PARAMS]:
             get[key] = value
         return get
@@ -1027,6 +1024,15 @@ class BaseRequest(DictMixin):
             instances of :class:`cgi.FieldStorage` (file uploads).
         """
         post = FormsDict()
+        # We default to application/x-www-form-urlencoded for everything that
+        # is not multipart and take the fast path (also: 3.1 workaround)
+        if not self.content_type.startswith('multipart/'):
+            maxlen = max(0, min(self.content_length, self.MEMFILE_MAX))
+            pairs = _parse_qsl(tonat(self.body.read(maxlen), 'latin1'))
+            for key, value in pairs[:self.MAX_PARAMS]:
+                post[key] = value
+            return post
+
         safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi
         for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'):
             if key in self.environ: safe_env[key] = self.environ[key]
@@ -1111,6 +1117,11 @@ class BaseRequest(DictMixin):
         return int(self.environ.get('CONTENT_LENGTH') or -1)
 
     @property
+    def content_type(self):
+        ''' The Content-Type header as a lowercase-string (default: empty). '''
+        return self.environ.get('CONTENT_TYPE', '').lower()
+
+    @property
     def is_xhr(self):
         ''' True if the request was triggered by a XMLHttpRequest. This only
             works with JavaScript libraries that support the `X-Requested-With`
@@ -1907,6 +1918,18 @@ def parse_auth(header):
         return None
 
 
+def _parse_qsl(qs):
+    r = []
+    for pair in qs.replace(';','&').split('&'):
+        if not pair: continue
+        nv = pair.split('=', 1)
+        if len(nv) != 2: nv.append('')
+        key = urlunquote(nv[0].replace('+', ' '))
+        value = urlunquote(nv[1].replace('+', ' '))
+        r.append((key, value))
+    return r
+
+
 def _lscmp(a, b):
     ''' Compares two strings in a cryptographically save way:
         Runtime is not affected by length of common prefix. '''
diff --git a/test/test_environ.py b/test/test_environ.py
index 2e54b95..56bb7a1 100755
--- a/test/test_environ.py
+++ b/test/test_environ.py
@@ -137,7 +137,7 @@ class TestRequest(unittest.TestCase):
 
     def test_get(self):
         """ Environ: GET data """ 
-        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=瓶'), 'latin1')
+        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=%e7%93%b6'), 'latin1')
         request = BaseRequest({'QUERY_STRING':qs})
         self.assertTrue('a' in request.query)
         self.assertTrue('b' in request.query)
@@ -150,7 +150,7 @@ class TestRequest(unittest.TestCase):
         
     def test_post(self):
         """ Environ: POST data """ 
-        sq = tob('a=a&a=1&b=b&c=&d&cn=瓶')
+        sq = tob('a=a&a=1&b=b&c=&d&cn=%e7%93%b6')
         e = {}
         wsgiref.util.setup_testing_defaults(e)
         e['wsgi.input'].write(sq)