fix: Fixes unicode problems with request.query and request.forms (fix #342 fix #344 fix #339)

2 files changed, 30 insertions, 6 deletions

diff --git a/bottle.py b/bottle.py
index 652da08..fd18220 100644
--- a/bottle.py
+++ b/bottle.py
@@ -77,8 +77,9 @@ except IOError:
 if py3k:
     import http.client as httplib
     import _thread as thread
-    from urllib.parse import urljoin, parse_qsl, SplitResult as UrlSplitResult
+    from urllib.parse import urljoin, SplitResult as UrlSplitResult
     from urllib.parse import urlencode, quote as urlquote, unquote as urlunquote
+    urlunquote = functools.partial(urlunquote, encoding='latin1')
     from http.cookies import SimpleCookie
     from collections import MutableMapping as DictMixin
     import pickle
@@ -100,12 +101,10 @@ else: # 2.x
     if py25:
         msg = "Python 2.5 support may be dropped in future versions of Bottle."
         warnings.warn(msg, DeprecationWarning)
-        from cgi import parse_qsl
         from UserDict import DictMixin
         def next(it): return it.next()
         bytes = str
     else: # 2.6, 2.7
-        from urlparse import parse_qsl
         from collections import MutableMapping as DictMixin
     json_loads = json_lds
 
@@ -978,8 +977,8 @@ class BaseRequest(object):
             values are sometimes called "URL arguments" or "GET parameters", but
             not to be confused with "URL wildcards" as they are provided by the
             :class:`Router`. '''
-        pairs = parse_qsl(self.query_string, keep_blank_values=True)
         get = self.environ['bottle.get'] = FormsDict()
+        pairs = _parse_qsl(self.environ.get('QUERY_STRING', ''))
         for key, value in pairs[:self.MAX_PARAMS]:
             get[key] = value
         return get
@@ -1075,6 +1074,15 @@ class BaseRequest(object):
             instances of :class:`cgi.FieldStorage` (file uploads).
         """
         post = FormsDict()
+        # We default to application/x-www-form-urlencoded for everything that
+        # is not multipart and take the fast path (also: 3.1 workaround)
+        if not self.content_type.startswith('multipart/'):
+            maxlen = max(0, min(self.content_length, self.MEMFILE_MAX))
+            pairs = _parse_qsl(tonat(self.body.read(maxlen), 'latin1'))
+            for key, value in pairs[:self.MAX_PARAMS]:
+                post[key] = value
+            return post
+
         safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi
         for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'):
             if key in self.environ: safe_env[key] = self.environ[key]
@@ -1159,6 +1167,11 @@ class BaseRequest(object):
         return int(self.environ.get('CONTENT_LENGTH') or -1)
 
     @property
+    def content_type(self):
+        ''' The Content-Type header as a lowercase-string (default: empty). '''
+        return self.environ.get('CONTENT_TYPE', '').lower()
+
+    @property
     def is_xhr(self):
         ''' True if the request was triggered by a XMLHttpRequest. This only
             works with JavaScript libraries that support the `X-Requested-With`
@@ -2151,6 +2164,17 @@ def parse_range_header(header, maxlen=0):
         except ValueError:
             pass
 
+def _parse_qsl(qs):
+    r = []
+    for pair in qs.replace(';','&').split('&'):
+        if not pair: continue
+        nv = pair.split('=', 1)
+        if len(nv) != 2: nv.append('')
+        key = urlunquote(nv[0].replace('+', ' '))
+        value = urlunquote(nv[1].replace('+', ' '))
+        r.append((key, value))
+    return r
+
 def _lscmp(a, b):
     ''' Compares two strings in a cryptographically safe way:
         Runtime is not affected by length of common prefix. '''
diff --git a/test/test_environ.py b/test/test_environ.py
index 0195b34..6637fa4 100755
--- a/test/test_environ.py
+++ b/test/test_environ.py
@@ -145,7 +145,7 @@ class TestRequest(unittest.TestCase):
 
     def test_get(self):
         """ Environ: GET data """ 
-        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=瓶'), 'latin1')
+        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=%e7%93%b6'), 'latin1')
         request = BaseRequest({'QUERY_STRING':qs})
         self.assertTrue('a' in request.query)
         self.assertTrue('b' in request.query)
@@ -158,7 +158,7 @@ class TestRequest(unittest.TestCase):
         
     def test_post(self):
         """ Environ: POST data """ 
-        sq = tob('a=a&a=1&b=b&c=&d&cn=瓶')
+        sq = tob('a=a&a=1&b=b&c=&d&cn=%e7%93%b6')
         e = {}
         wsgiref.util.setup_testing_defaults(e)
         e['wsgi.input'].write(sq)