fix: Unicode errors with Python 3.2+ POST parameters. (fix #344, fix #349)

3 files changed, 31 insertions, 16 deletions

diff --git a/bottle.py b/bottle.py
index 098f434..8c8c7b3 100755
--- a/bottle.py
+++ b/bottle.py
@@ -88,7 +88,8 @@ except ImportError: # pragma: no cover
                 raise ImportError("JSON support requires Python 2.6 or simplejson.")
             json_lds = json_dumps
 
-py3k = sys.version_info >= (3,0,0)
+py = sys.version_info
+py3k = py >= (3,0,0)
 NCTextIOWrapper = None
 
 if sys.version_info < (2,6,0):
@@ -128,6 +129,15 @@ def try_update_wrapper(wrapper, wrapped, *a, **ka):
         functools.update_wrapper(wrapper, wrapped, *a, **ka)
     except AttributeError: pass
 
+# 3.2 fixes cgi.FieldStorage to accept bytes (which makes a lot of sense).
+#     but defaults to utf-8 (which is not always true)
+# 3.1 needs a workaround.
+NCTextIOWrapper = None
+if (3,0,0) < py < (3,2,0):
+    from io import TextIOWrapper
+    class NCTextIOWrapper(TextIOWrapper):
+        def close(self): pass # Keep wrapped buffer open.
+
 # Backward compatibility
 def depr(message):
     warnings.warn(message, DeprecationWarning, stacklevel=3)
@@ -1020,11 +1030,13 @@ class BaseRequest(DictMixin):
         safe_env = {'QUERY_STRING':''} # Build a safe environment for cgi
         for key in ('REQUEST_METHOD', 'CONTENT_TYPE', 'CONTENT_LENGTH'):
             if key in self.environ: safe_env[key] = self.environ[key]
+        args = dict(fp=self.body, environ=safe_env, keep_blank_values=True)
+        if py >= (3,2,0):
+            args['encoding'] = 'ISO-8859-1'
         if NCTextIOWrapper:
-            fb = NCTextIOWrapper(self.body, encoding='ISO-8859-1', newline='\n')
-        else:
-            fb = self.body
-        data = cgi.FieldStorage(fp=fb, environ=safe_env, keep_blank_values=True)
+            args['fp'] = NCTextIOWrapper(args['fp'], encoding='ISO-8859-1',
+                                         newline='\n')
+        data = cgi.FieldStorage(**args)
         for item in (data.list or [])[:self.MAX_PARAMS]:
             post[item.name] = item if item.filename else item.value
         return post
diff --git a/test/test_environ.py b/test/test_environ.py
index f54db62..2e54b95 100755
--- a/test/test_environ.py
+++ b/test/test_environ.py
@@ -137,17 +137,20 @@ class TestRequest(unittest.TestCase):
 
     def test_get(self):
         """ Environ: GET data """ 
-        request = BaseRequest({'QUERY_STRING':'a=a&a=1&b=b&c=c'})
+        qs = tonat(tob('a=a&a=1&b=b&c=c&cn=瓶'), 'latin1')
+        request = BaseRequest({'QUERY_STRING':qs})
         self.assertTrue('a' in request.query)
         self.assertTrue('b' in request.query)
         self.assertEqual(['a','1'], request.query.getall('a'))
         self.assertEqual(['b'], request.query.getall('b'))
         self.assertEqual('1', request.query['a'])
         self.assertEqual('b', request.query['b'])
+        self.assertEqual(tonat(tob('瓶'), 'latin1'), request.query['cn'])
+        self.assertEqual(touni('瓶'), request.query.cn)
         
     def test_post(self):
         """ Environ: POST data """ 
-        sq = u'a=a&a=1&b=b&c=&d'.encode('utf8')
+        sq = tob('a=a&a=1&b=b&c=&d&cn=瓶')
         e = {}
         wsgiref.util.setup_testing_defaults(e)
         e['wsgi.input'].write(sq)
@@ -163,6 +166,8 @@ class TestRequest(unittest.TestCase):
         self.assertEqual('b', request.POST['b'])
         self.assertEqual('', request.POST['c'])
         self.assertEqual('', request.POST['d'])
+        self.assertEqual(tonat(tob('瓶'), 'latin1'), request.POST['cn'])
+        self.assertEqual(touni('瓶'), request.POST.cn)
 
     def test_bodypost(self):
         sq = u'foobar'.encode('utf8')
diff --git a/test/test_formsdict.py b/test/test_formsdict.py
index fd862b8..c95fa82 100644
--- a/test/test_formsdict.py
+++ b/test/test_formsdict.py
@@ -1,4 +1,5 @@
 # -*- coding: utf-8 -*-
+# '瓶' means "Bottle"
 
 import unittest
 from bottle import FormsDict, touni, tob
@@ -6,10 +7,9 @@ from bottle import FormsDict, touni, tob
 class TestFormsDict(unittest.TestCase):
     def test_attr_access(self):
         """ FomsDict.attribute returs string values as unicode. """
-        data = u'äöü'.encode('utf8')
-        d = FormsDict(py2=data, py3=data.decode('latin1'))
-        self.assertEqual(u'äöü', d.py2)
-        self.assertEqual(u'äöü', d.py3)
+        d = FormsDict(py2=tob('瓶'), py3=tob('瓶').decode('latin1'))
+        self.assertEqual(touni('瓶'), d.py2)
+        self.assertEqual(touni('瓶'), d.py3)
 
     def test_attr_missing(self):
         """ FomsDict.attribute returs u'' on missing keys. """
@@ -18,12 +18,10 @@ class TestFormsDict(unittest.TestCase):
 
     def test_attr_unicode_error(self):
         """ FomsDict.attribute returs u'' on UnicodeError. """
-        d = FormsDict(latin=u'äöü'.encode('latin1'))
-        self.assertEqual(u'', d.latin)
+        d = FormsDict(latin=touni('öäüß').encode('latin1'))
+        self.assertEqual(touni(''), d.latin)
         d.input_encoding = 'latin1'
-        self.assertEqual(u'äöü', d.latin)
-
-
+        self.assertEqual(touni('öäüß'), d.latin)
 
    
 if __name__ == '__main__': #pragma: no cover