buildlogger: make everything unicode

1 files changed, 19 insertions, 0 deletions

diff --git a/buildscripts/utils.py b/buildscripts/utils.py
index 413f22681af..be16d0b9f82 100644
--- a/buildscripts/utils.py
+++ b/buildscripts/utils.py
@@ -1,4 +1,5 @@
 
+import codecs
 import re
 import socket
 import time
@@ -191,3 +192,21 @@ def run_smoke_command(*args):
     # otherwise SCons treats it as a list of dependencies.
     return [smoke_command(*args)]
 
+# unicode is a pain. some strings cannot be unicode()'d
+# but we want to just preserve the bytes in a human-readable
+# fashion. this codec error handler will substitute the
+# repr() of the offending bytes into the decoded string
+# at the position they occurred
+def replace_with_repr(unicode_error):
+    offender = unicode_error.object[unicode_error.start:unicode_error.end]
+    return (unicode(repr(offender).strip("'").strip('"')), unicode_error.end)
+
+codecs.register_error('repr', replace_with_repr)
+
+def unicode_dammit(string, encoding='utf8'):
+    # convert a string to a unicode, using the Python
+    # representation of non-ascii bytes when necessary
+    #
+    # name inpsired by BeautifulSoup's "UnicodeDammit"
+    return string.decode(encoding, 'repr')
+