giscanner: Use unicode literals in all Python files

Add unicode_literals future import which turns any string literal
into a unicode string. Return unicode strings from the Python C extension
module. Force writing of annotations (g-ir-annotation-tool) to output utf8
encoded data to stdout.
This is an initial pass at following the "unicode sandwich"
model of programming (http://nedbatchelder.com/text/unipain.html)
needed for supporting Python 3.

https://bugzilla.gnome.org/show_bug.cgi?id=679438

26 files changed, 108 insertions, 35 deletions

diff --git a/giscanner/annotationmain.py b/giscanner/annotationmain.py
index 190269e2..b82ff818 100644
--- a/giscanner/annotationmain.py
+++ b/giscanner/annotationmain.py
@@ -21,8 +21,12 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
+import sys
 import optparse
+import codecs
+from contextlib import contextmanager
 
 from giscanner import message
 from giscanner.annotationparser import GtkDocCommentBlockParser, GtkDocCommentBlockWriter
@@ -31,6 +35,24 @@ from giscanner.scannermain import (get_preprocessor_option_group,
                                    process_packages)
 
 
+@contextmanager
+def encode_stdout(encoding):
+    """Force stdout into a specific encoding."""
+    # Python 2 does not encode stdout writes so wrap it with 'encoding' encoded writer.
+    # Python 3 uses a io.TextIOBase wrapped stdout with the system default encoding.
+    # Re-wrap the underlying buffer with a new writer with the given 'encoding'.
+    # See: https://docs.python.org/3/library/sys.html#sys.stdout
+    old_stdout = sys.stdout
+    if sys.version_info.major < 3:
+        binary_stdout = sys.stdout
+    else:
+        binary_stdout = sys.stdout.buffer
+
+    sys.stdout = codecs.getwriter(encoding)(binary_stdout)
+    yield
+    sys.stdout = old_stdout
+
+
 def annotation_main(args):
     parser = optparse.OptionParser('%prog [options] sources')
 
@@ -65,16 +87,18 @@ def annotation_main(args):
         parser = GtkDocCommentBlockParser()
         writer = GtkDocCommentBlockWriter(indent=False)
         blocks = parser.parse_comment_blocks(ss.get_comments())
-        print('/' + ('*' * 60) + '/')
-        print('/* THIS FILE IS GENERATED DO NOT EDIT */')
-        print('/' + ('*' * 60) + '/')
-        print('')
-        for block in sorted(blocks.values()):
-            print(writer.write(block))
+
+        with encode_stdout('utf-8'):
+            print('/' + ('*' * 60) + '/')
+            print('/* THIS FILE IS GENERATED DO NOT EDIT */')
+            print('/' + ('*' * 60) + '/')
+            print('')
+            for block in sorted(blocks.values()):
+                print(writer.write(block))
+                print('')
             print('')
-        print('')
-        print('/' + ('*' * 60) + '/')
-        print('/* THIS FILE IS GENERATED DO NOT EDIT */')
-        print('/' + ('*' * 60) + '/')
+            print('/' + ('*' * 60) + '/')
+            print('/* THIS FILE IS GENERATED DO NOT EDIT */')
+            print('/' + ('*' * 60) + '/')
 
     return 0
diff --git a/giscanner/annotationparser.py b/giscanner/annotationparser.py
index 09026a4c..cd6fa4fa 100644
--- a/giscanner/annotationparser.py
+++ b/giscanner/annotationparser.py
@@ -110,6 +110,7 @@ Refer to the `GTK-Doc manual`_ for more detailed usage information.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import re
@@ -1125,9 +1126,10 @@ class GtkDocCommentBlockParser(object):
         for (comment, filename, lineno) in comments:
             try:
                 comment_block = self.parse_comment_block(comment, filename, lineno)
-            except Exception:
+            except Exception as e:
                 error('unrecoverable parse error, please file a GObject-Introspection bug'
-                      'report including the complete comment block at the indicated location.',
+                      'report including the complete comment block at the indicated location. %s' %
+                      str(e),
                       Position(filename, lineno))
                 continue
 
diff --git a/giscanner/ast.py b/giscanner/ast.py
index db1f07be..74a70811 100644
--- a/giscanner/ast.py
+++ b/giscanner/ast.py
@@ -22,6 +22,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import copy
 from itertools import chain
diff --git a/giscanner/cachestore.py b/giscanner/cachestore.py
index 4f5e6f2b..f2352591 100644
--- a/giscanner/cachestore.py
+++ b/giscanner/cachestore.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import errno
 import cPickle
diff --git a/giscanner/ccompiler.py b/giscanner/ccompiler.py
index a401cfd7..481b1e4c 100644
--- a/giscanner/ccompiler.py
+++ b/giscanner/ccompiler.py
@@ -236,7 +236,7 @@ class CCompiler(object):
                                      include_dirs=includes,
                                      extra_postargs=extra_postargs,
                                      output_dir=source_str[tmpdir_idx + 1:
-                                                           source_str.rfind(os.sep)])
+                                                           source_str.rfind(os.sep)].encode('UTF-8'))
 
     def link(self, output, objects, lib_args):
         # Note: This is used for non-libtool builds only!
diff --git a/giscanner/codegen.py b/giscanner/codegen.py
index 3138ac2a..ff1a435a 100644
--- a/giscanner/codegen.py
+++ b/giscanner/codegen.py
@@ -22,6 +22,7 @@ from __future__ import with_statement
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 from contextlib import contextmanager
 
diff --git a/giscanner/collections/__init__.py b/giscanner/collections/__init__.py
index 5ab935f3..6cfcc511 100644
--- a/giscanner/collections/__init__.py
+++ b/giscanner/collections/__init__.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 from .counter import Counter
 from .ordereddict import OrderedDict
diff --git a/giscanner/docmain.py b/giscanner/docmain.py
index a9f6904c..3bea5421 100644
--- a/giscanner/docmain.py
+++ b/giscanner/docmain.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import argparse
diff --git a/giscanner/docwriter.py b/giscanner/docwriter.py
index 70f83efe..acbf279a 100644
--- a/giscanner/docwriter.py
+++ b/giscanner/docwriter.py
@@ -24,6 +24,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import re
diff --git a/giscanner/dumper.py b/giscanner/dumper.py
index 40877f8f..0a59e773 100644
--- a/giscanner/dumper.py
+++ b/giscanner/dumper.py
@@ -22,6 +22,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys
diff --git a/giscanner/gdumpparser.py b/giscanner/gdumpparser.py
index 6fcf5896..9bdc2bc1 100644
--- a/giscanner/gdumpparser.py
+++ b/giscanner/gdumpparser.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys
diff --git a/giscanner/girparser.py b/giscanner/girparser.py
index 2e198ce1..463ff9a9 100644
--- a/giscanner/girparser.py
+++ b/giscanner/girparser.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 
diff --git a/giscanner/girwriter.py b/giscanner/girwriter.py
index aafedc64..0df87ed4 100644
--- a/giscanner/girwriter.py
+++ b/giscanner/girwriter.py
@@ -23,6 +23,7 @@ from __future__ import with_statement
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 from . import ast
 from .xmlwriter import XMLWriter
diff --git a/giscanner/giscannermodule.c b/giscanner/giscannermodule.c
index b9512275..f6945da3 100644
--- a/giscanner/giscannermodule.c
+++ b/giscanner/giscannermodule.c
@@ -141,7 +141,7 @@ symbol_get_ident (PyGISourceSymbol *self,
       return Py_None;
     }
 
-  return PyString_FromString (self->symbol->ident);
+  return PyUnicode_FromString (self->symbol->ident);
 }
 
 static PyObject *
@@ -189,7 +189,7 @@ symbol_get_const_string (PyGISourceSymbol *self,
       return Py_None;
     }
 
-  return PyString_FromString (self->symbol->const_string);
+  return PyUnicode_FromString (self->symbol->const_string);
 }
 
 static PyObject *
@@ -215,7 +215,7 @@ symbol_get_source_filename (PyGISourceSymbol *self,
       return Py_None;
     }
 
-  return PyString_FromString (self->symbol->source_filename);
+  return PyUnicode_FromString (self->symbol->source_filename);
 }
 
 static const PyGetSetDef _PyGISourceSymbol_getsets[] = {
@@ -296,7 +296,7 @@ type_get_name (PyGISourceType *self,
       return Py_None;
     }
 
-  return PyString_FromString (self->type->name);
+  return PyUnicode_FromString (self->type->name);
 }
 
 static PyObject *
@@ -593,10 +593,35 @@ pygi_source_scanner_get_comments (PyGISourceScanner *self)
   for (l = comments; l; l = l->next)
     {
       GISourceComment *comment = l->data;
-      PyObject *item = Py_BuildValue ("(ssi)", comment->comment,
-                                      comment->filename,
-                                      comment->line);
+      PyObject *comment_obj;
+      PyObject *filename_obj;
+      PyObject *item;
+
+      if (comment->comment)
+        {
+          comment_obj = PyUnicode_FromString (comment->comment);
+        }
+      else
+        {
+          Py_INCREF (Py_None);
+          comment_obj = Py_None;
+        }
+
+      if (comment->filename)
+        {
+          filename_obj = PyUnicode_FromString (comment->filename);
+        }
+      else
+        {
+          Py_INCREF (Py_None);
+          filename_obj = Py_None;
+        }
+
+      item = Py_BuildValue ("(OOi)", comment_obj, filename_obj, comment->line);
       PyList_SetItem (list, i++, item);
+
+      Py_DECREF (comment_obj);
+      Py_DECREF (filename_obj);
     }
 
   g_slist_free (comments);
diff --git a/giscanner/introspectablepass.py b/giscanner/introspectablepass.py
index 859118e3..19d1388f 100644
--- a/giscanner/introspectablepass.py
+++ b/giscanner/introspectablepass.py
@@ -19,6 +19,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 from . import ast
 from . import message
diff --git a/giscanner/libtoolimporter.py b/giscanner/libtoolimporter.py
index 6507696a..0135bb82 100644
--- a/giscanner/libtoolimporter.py
+++ b/giscanner/libtoolimporter.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import imp
 import os
diff --git a/giscanner/maintransformer.py b/giscanner/maintransformer.py
index 716f6c28..fc59fa80 100644
--- a/giscanner/maintransformer.py
+++ b/giscanner/maintransformer.py
@@ -20,6 +20,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import re
 
diff --git a/giscanner/message.py b/giscanner/message.py
index 96cf455a..d474ae9d 100644
--- a/giscanner/message.py
+++ b/giscanner/message.py
@@ -23,6 +23,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys
diff --git a/giscanner/scannermain.py b/giscanner/scannermain.py
index dd3f2b76..4d2fcbf7 100755
--- a/giscanner/scannermain.py
+++ b/giscanner/scannermain.py
@@ -23,6 +23,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import errno
 import optparse
diff --git a/giscanner/sectionparser.py b/giscanner/sectionparser.py
index 69a685e0..e8e584d3 100644
--- a/giscanner/sectionparser.py
+++ b/giscanner/sectionparser.py
@@ -20,6 +20,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import re
 from . import ast
diff --git a/giscanner/shlibs.py b/giscanner/shlibs.py
index 1115f5cf..c1f89bed 100644
--- a/giscanner/shlibs.py
+++ b/giscanner/shlibs.py
@@ -22,6 +22,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import platform
diff --git a/giscanner/sourcescanner.py b/giscanner/sourcescanner.py
index 5de24ff7..88531a7b 100644
--- a/giscanner/sourcescanner.py
+++ b/giscanner/sourcescanner.py
@@ -22,6 +22,7 @@ from __future__ import with_statement
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import subprocess
diff --git a/giscanner/testcodegen.py b/giscanner/testcodegen.py
index ab46e923..35b6d453 100644
--- a/giscanner/testcodegen.py
+++ b/giscanner/testcodegen.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 from StringIO import StringIO
 from . import ast
diff --git a/giscanner/transformer.py b/giscanner/transformer.py
index 2182182f..968db755 100644
--- a/giscanner/transformer.py
+++ b/giscanner/transformer.py
@@ -21,6 +21,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 import sys
@@ -730,7 +731,7 @@ raise ValueError."""
         name = self._strip_symbol(symbol)
         if symbol.const_string is not None:
             typeval = ast.TYPE_STRING
-            value = unicode(symbol.const_string, 'utf-8')
+            value = symbol.const_string
         elif symbol.const_int is not None:
             if symbol.base_type is not None:
                 typeval = self._create_type_from_base(symbol.base_type)
diff --git a/giscanner/utils.py b/giscanner/utils.py
index c40a6ffd..aff5393a 100644
--- a/giscanner/utils.py
+++ b/giscanner/utils.py
@@ -20,6 +20,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import errno
 import re
@@ -170,7 +171,7 @@ def files_are_identical(path1, path2):
     with open(path1, 'rb') as f1, open(path2, 'rb') as f2:
         buf1 = f1.read(8192)
         buf2 = f2.read(8192)
-        while buf1 == buf2 and buf1 != '':
+        while buf1 == buf2 and buf1 != b'':
             buf1 = f1.read(8192)
             buf2 = f2.read(8192)
         return buf1 == buf2
diff --git a/giscanner/xmlwriter.py b/giscanner/xmlwriter.py
index 0c734a33..851c4c07 100755
--- a/giscanner/xmlwriter.py
+++ b/giscanner/xmlwriter.py
@@ -22,6 +22,7 @@ from __future__ import with_statement
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import os
 
@@ -43,13 +44,13 @@ def build_xml_tag(tag_name, attributes=None, data=None, self_indent=0,
                   self_indent_char=' '):
     if attributes is None:
         attributes = []
-    prefix = u'<%s' % (tag_name, )
+    prefix = '<%s' % (tag_name, )
     if data is not None:
-        if isinstance(data, str):
+        if isinstance(data, bytes):
             data = data.decode('UTF-8')
-        suffix = u'>%s</%s>' % (escape(data), tag_name)
+        suffix = '>%s</%s>' % (escape(data), tag_name)
     else:
-        suffix = u'/>'
+        suffix = '/>'
     attrs = collect_attributes(
         tag_name, attributes,
         self_indent,
@@ -62,7 +63,7 @@ class XMLWriter(object):
 
     def __init__(self):
         self._data = StringIO()
-        self._data.write('<?xml version="1.0"?>\n')
+        self._data.write(b'<?xml version="1.0"?>\n')
         self._tag_stack = []
         self._indent = 0
         self._indent_unit = 2
@@ -75,10 +76,10 @@ class XMLWriter(object):
             attributes = []
         attrs = collect_attributes(tag_name, attributes,
                                    self._indent, self._indent_char, len(tag_name) + 2)
-        self.write_line(u'<%s%s>' % (tag_name, attrs))
+        self.write_line('<%s%s>' % (tag_name, attrs))
 
     def _close_tag(self, tag_name):
-        self.write_line(u'</%s>' % (tag_name, ))
+        self.write_line('</%s>' % (tag_name, ))
 
     # Public API
 
@@ -93,18 +94,18 @@ class XMLWriter(object):
     def get_xml(self):
         return self._data.getvalue()
 
-    def write_line(self, line=u'', indent=True, do_escape=False):
-        if isinstance(line, str):
+    def write_line(self, line='', indent=True, do_escape=False):
+        if isinstance(line, bytes):
             line = line.decode('utf-8')
         assert isinstance(line, unicode)
         if do_escape:
             line = escape(line)
         if indent:
-            self._data.write('%s%s%s' % (self._indent_char * self._indent,
-                                         line.encode('utf-8'),
-                                         self._newline_char))
+            self._data.write(('%s%s%s' % (self._indent_char * self._indent,
+                                          line,
+                                          self._newline_char)).encode('UTF-8'))
         else:
-            self._data.write('%s%s' % (line.encode('utf-8'), self._newline_char))
+            self._data.write(('%s%s' % (line, self._newline_char)).encode('UTF-8'))
 
     def write_comment(self, text):
         self.write_line('<!-- %s -->' % (text, ))