- Added "unicode_errors" parameter to String, Unicode, etc.

Behaves like the 'errors' keyword argument to
the standard library's string.decode() functions.   This flag
requires that `convert_unicode` is set to `"force"` - otherwise,
SQLAlchemy is not guaranteed to handle the task of unicode
conversion.   Note that this flag adds significant performance
overhead to row-fetching operations for backends that already
return unicode objects natively (which most DBAPIs do).  This
flag should only be used as an absolute last resort for reading
strings from a column with varied or corrupted encodings,
which only applies to databases that accept invalid encodings
in the first place (i.e. MySQL. *not* PG, Sqlite, etc.)

1 files changed, 8 insertions, 4 deletions

diff --git a/lib/sqlalchemy/processors.py b/lib/sqlalchemy/processors.py
index cb4b72545..2bf9221b7 100644
--- a/lib/sqlalchemy/processors.py
+++ b/lib/sqlalchemy/processors.py
@@ -32,14 +32,18 @@ try:
                                        str_to_datetime, str_to_time, \
                                        str_to_date
 
-    def to_unicode_processor_factory(encoding):
-        return UnicodeResultProcessor(encoding).process
+    def to_unicode_processor_factory(encoding, error=None):
+        # TODO: this is cumbersome
+        if error is not None:
+            return UnicodeResultProcessor(encoding, error).process
+        else:
+            return UnicodeResultProcessor(encoding).process
 
     def to_decimal_processor_factory(target_class):
         return DecimalResultProcessor(target_class).process
 
 except ImportError:
-    def to_unicode_processor_factory(encoding):
+    def to_unicode_processor_factory(encoding, error=None):
         decoder = codecs.getdecoder(encoding)
 
         def process(value):
@@ -50,7 +54,7 @@ except ImportError:
                 # len part is safe: it is done that way in the normal
                 # 'xx'.decode(encoding) code path.
                 # cfr python-source/Python/codecs.c:PyCodec_Decode
-                return decoder(value)[0]
+                return decoder(value, error)[0]
         return process
 
     def to_decimal_processor_factory(target_class):