giscanner: Use unicode literals in all Python files

Add unicode_literals future import which turns any string literal
into a unicode string. Return unicode strings from the Python C extension
module. Force writing of annotations (g-ir-annotation-tool) to output utf8
encoded data to stdout.
This is an initial pass at following the "unicode sandwich"
model of programming (http://nedbatchelder.com/text/unipain.html)
needed for supporting Python 3.

https://bugzilla.gnome.org/show_bug.cgi?id=679438

4 files changed, 22 insertions, 5 deletions

diff --git a/tests/scanner/annotationparser/test_parser.py b/tests/scanner/annotationparser/test_parser.py
index 2cec9f11..b676a508 100644
--- a/tests/scanner/annotationparser/test_parser.py
+++ b/tests/scanner/annotationparser/test_parser.py
@@ -28,9 +28,11 @@ Tests ensuring annotationparser.py continues to function correctly.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import difflib
 import os
+import sys
 import subprocess
 import unittest
 import xml.etree.ElementTree as etree
@@ -39,6 +41,11 @@ from giscanner.annotationparser import GtkDocCommentBlockParser, GtkDocCommentBl
 from giscanner.ast import Namespace
 from giscanner.message import MessageLogger, WARNING, ERROR, FATAL
 
+if sys.version_info.major < 3:
+    encode_name = lambda s: s.encode('ascii')
+else:
+    encode_name = lambda s: s
+
 
 XML_NS = 'http://schemas.gnome.org/gobject-introspection/2013/test'
 XML_SCHEMA = os.path.abspath(os.path.join(os.path.dirname(__file__), 'tests.xsd'))
@@ -399,7 +406,7 @@ def create_test_case(logger, tests_dir, tests_file):
     for counter, test in enumerate(tests_tree.findall(ns('{}test'))):
         test_name = 'test_%03d' % (counter + 1)
         test_method = TestCommentBlock.__create_test__(logger, test)
-        test_method.__name__ = test_name
+        test_method.__name__ = encode_name(test_name)
         test_methods[test_name] = test_method
 
     # Dynamically generate a new subclass of TestCommentBlock in TitleCase
@@ -407,7 +414,7 @@ def create_test_case(logger, tests_dir, tests_file):
     test_class_name = os.path.relpath(tests_file[:-4], tests_dir)
     test_class_name = test_class_name.replace('/', ' ').replace('\\', ' ').replace('.', ' ')
     test_class_name = 'Test' + test_class_name.title().replace(' ', '')
-    return type(test_class_name, (TestCommentBlock,), test_methods)
+    return type(encode_name(test_class_name), (TestCommentBlock,), test_methods)
 
 
 def create_test_cases():
diff --git a/tests/scanner/annotationparser/test_patterns.py b/tests/scanner/annotationparser/test_patterns.py
index 7d430940..0a0e3175 100644
--- a/tests/scanner/annotationparser/test_patterns.py
+++ b/tests/scanner/annotationparser/test_patterns.py
@@ -32,13 +32,21 @@ against the expected output.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
+
+import sys
+import unittest
 
 from giscanner.annotationparser import (COMMENT_BLOCK_START_RE, COMMENT_BLOCK_END_RE,
                                         COMMENT_ASTERISK_RE, INDENTATION_RE, EMPTY_LINE_RE,
                                         SECTION_RE, SYMBOL_RE, PROPERTY_RE,
                                         SIGNAL_RE, PARAMETER_RE, TAG_RE,
                                         TAG_VALUE_VERSION_RE, TAG_VALUE_STABILITY_RE)
-import unittest
+
+if sys.version_info.major < 3:
+    encode_name = lambda s: s.encode('ascii')
+else:
+    encode_name = lambda s: s
 
 
 comment_start_tests = [
@@ -894,10 +902,10 @@ def create_test_case(tests_class_name, testcases):
     for counter, test in enumerate(testcases):
         test_name = 'test_%03d' % (counter + 1)
         test_method = create_test_method(test)
-        test_method.__name__ = test_name
+        test_method.__name__ = encode_name(test_name)
         test_methods[test_name] = test_method
 
-    return type(tests_class_name, (unittest.TestCase,), test_methods)
+    return type(encode_name(tests_class_name), (unittest.TestCase,), test_methods)
 
 
 def create_test_cases():
diff --git a/tests/scanner/test_sourcescanner.py b/tests/scanner/test_sourcescanner.py
index a47485e2..831af486 100644
--- a/tests/scanner/test_sourcescanner.py
+++ b/tests/scanner/test_sourcescanner.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import unittest
 import tempfile
diff --git a/tests/scanner/test_transformer.py b/tests/scanner/test_transformer.py
index 1e75d688..bd85c8c3 100644
--- a/tests/scanner/test_transformer.py
+++ b/tests/scanner/test_transformer.py
@@ -1,6 +1,7 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
+from __future__ import unicode_literals
 
 import unittest
 import tempfile