Allow ffi.embedding_init_code("unicode-with-non-ascii-chars")

2 files changed, 22 insertions, 2 deletions

diff --git a/cffi/recompiler.py b/cffi/recompiler.py
index d66ff7f..1309572 100644
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -1296,14 +1296,28 @@ class Recompiler:
     def _print_string_literal_in_array(self, s):
         prnt = self._prnt
         prnt('// # NB. this is not a string because of a size limit in MSVC')
+        if not isinstance(s, bytes):    # unicode
+            s = s.encode('utf-8')       # -> bytes
+        else:
+            s.decode('utf-8')           # got bytes, check for valid utf-8
+        try:
+            s.decode('ascii')
+        except UnicodeDecodeError:
+            s = b'# -*- encoding: utf8 -*-\n' + s
         for line in s.splitlines(True):
-            prnt(('// ' + line).rstrip())
+            comment = line
+            if type('//') is bytes:     # python2
+                line = map(ord, line)   #     make a list of integers
+            else:                       # python3
+                # type(line) is bytes, which enumerates like a list of integers
+                comment = ascii(comment)[1:-1]
+            prnt(('// ' + comment).rstrip())
             printed_line = ''
             for c in line:
                 if len(printed_line) >= 76:
                     prnt(printed_line)
                     printed_line = ''
-                printed_line += '%d,' % (ord(c),)
+                printed_line += '%d,' % (c,)
             prnt(printed_line)
 
     # ----------
diff --git a/testing/embedding/test_basic.py b/testing/embedding/test_basic.py
index 894ace5..8d2e776 100644
--- a/testing/embedding/test_basic.py
+++ b/testing/embedding/test_basic.py
@@ -206,3 +206,9 @@ class TestBasic(EmbeddingTests):
         self.compile('add1-test', [initerror_cffi])
         output = self.execute('add1-test')
         assert output == "got: 0 0\n"    # plus lots of info to stderr
+
+    def test_embedding_with_unicode(self):
+        withunicode_cffi = self.prepare_module('withunicode')
+        self.compile('add1-test', [withunicode_cffi])
+        output = self.execute('add1-test')
+        assert output == "255\n4660\n65244\ngot: 0 0\n"