Use Cython if available; added Python 3 support to _yaml.pyx.

git-svn-id: http://svn.pyyaml.org/pyyaml/trunk@331 18f92427-320e-0410-9341-c67f048884a3

7 files changed, 187 insertions, 82 deletions

diff --git a/ext/_yaml.h b/ext/_yaml.h
index d8070e5..b39292d 100644
--- a/ext/_yaml.h
+++ b/ext/_yaml.h
@@ -1,3 +1,11 @@
 
 #include <yaml.h>
 
+#if PY_MAJOR_VERSION >= 3
+
+#define PyString_CheckExact PyBytes_CheckExact
+#define PyString_AS_STRING  PyBytes_AS_STRING
+#define PyString_GET_SIZE   PyBytes_GET_SIZE
+#define PyString_FromStringAndSize  PyBytes_FromStringAndSize
+
+#endif
diff --git a/ext/_yaml.pyx b/ext/_yaml.pyx
index b2c4e52..8b11f16 100644
--- a/ext/_yaml.pyx
+++ b/ext/_yaml.pyx
@@ -251,10 +251,16 @@ cdef class CParser:
     cdef object anchors
 
     def __init__(self, stream):
+        cdef is_readable
         if yaml_parser_initialize(&self.parser) == 0:
             raise MemoryError
         self.parsed_event.type = YAML_NO_EVENT
-        if hasattr(stream, 'read'):
+        is_readable = 1
+        try:
+            stream.read
+        except AttributeError:
+            is_readable = 0
+        if is_readable:
             self.stream = stream
             try:
                 self.stream_name = stream.name
@@ -357,23 +363,25 @@ cdef class CParser:
         elif token.type == YAML_STREAM_START_TOKEN:
             encoding = None
             if token.data.stream_start.encoding == YAML_UTF8_ENCODING:
-                encoding = "utf-8"
+                encoding = u"utf-8"
             elif token.data.stream_start.encoding == YAML_UTF16LE_ENCODING:
-                encoding = "utf-16-le"
+                encoding = u"utf-16-le"
             elif token.data.stream_start.encoding == YAML_UTF16BE_ENCODING:
-                encoding = "utf-16-be"
+                encoding = u"utf-16-be"
             return StreamStartToken(start_mark, end_mark, encoding)
         elif token.type == YAML_STREAM_END_TOKEN:
             return StreamEndToken(start_mark, end_mark)
         elif token.type == YAML_VERSION_DIRECTIVE_TOKEN:
-            return DirectiveToken("YAML",
+            return DirectiveToken(u"YAML",
                     (token.data.version_directive.major,
                         token.data.version_directive.minor),
                     start_mark, end_mark)
         elif token.type == YAML_TAG_DIRECTIVE_TOKEN:
-            return DirectiveToken("TAG",
-                    (token.data.tag_directive.handle,
-                        token.data.tag_directive.prefix),
+            handle = PyUnicode_DecodeUTF8(token.data.tag_directive.handle,
+                    strlen(token.data.tag_directive.handle), 'strict')
+            prefix = PyUnicode_DecodeUTF8(token.data.tag_directive.prefix,
+                    strlen(token.data.tag_directive.prefix), 'strict')
+            return DirectiveToken(u"TAG", (handle, prefix),
                     start_mark, end_mark)
         elif token.type == YAML_DOCUMENT_START_TOKEN:
             return DocumentStartToken(start_mark, end_mark)
@@ -870,6 +878,8 @@ cdef int input_handler(void *data, char *buffer, int size, int *read) except 0:
     cdef CParser parser
     parser = <CParser>data
     value = parser.stream.read(size)
+    if PyUnicode_CheckExact(value) != 0:
+        value = PyUnicode_AsUTF8String(value)
     if PyString_CheckExact(value) == 0:
         raise TypeError("a string value is expected")
     if PyString_GET_SIZE(value) > size:
@@ -894,6 +904,7 @@ cdef class CEmitter:
     cdef object anchors
     cdef int last_alias_id
     cdef int closed
+    cdef int decode_output
 
     def __init__(self, stream, canonical=None, indent=None, width=None,
             allow_unicode=None, line_break=None, encoding=None,
@@ -901,6 +912,11 @@ cdef class CEmitter:
         if yaml_emitter_initialize(&self.emitter) == 0:
             raise MemoryError
         self.stream = stream
+        self.decode_output = 1
+        try:
+            stream.encoding
+        except AttributeError:
+            self.decode_output = 0
         yaml_emitter_set_output(&self.emitter, output_handler, <void *>self)    
         if canonical is not None:
             yaml_emitter_set_canonical(&self.emitter, 1)
@@ -1216,7 +1232,7 @@ cdef class CEmitter:
         if node in self.anchors:
             if self.anchors[node] is None:
                 self.last_alias_id = self.last_alias_id+1
-                self.anchors[node] = "id%03d" % self.last_alias_id
+                self.anchors[node] = u"id%03d" % self.last_alias_id
         else:
             self.anchors[node] = None
             node_class = node.__class__
@@ -1245,7 +1261,7 @@ cdef class CEmitter:
         anchor_object = self.anchors[node]
         anchor = NULL
         if anchor_object is not None:
-            anchor = PyString_AS_STRING(anchor_object)
+            anchor = PyString_AS_STRING(PyUnicode_AsUTF8String(anchor_object))
         if node in self.serialized_nodes:
             if yaml_alias_event_initialize(&event, anchor) == 0:
                 raise MemoryError
@@ -1357,7 +1373,10 @@ cdef class CEmitter:
 cdef int output_handler(void *data, char *buffer, int size) except 0:
     cdef CEmitter emitter
     emitter = <CEmitter>data
-    value = PyString_FromStringAndSize(buffer, size)
+    if emitter.decode_output == 0:
+        value = PyString_FromStringAndSize(buffer, size)
+    else:
+        value = PyUnicode_DecodeUTF8(buffer, size, 'strict')
     emitter.stream.write(value)
     return 1
 
diff --git a/lib3/yaml/__init__.py b/lib3/yaml/__init__.py
index b180319..55592e0 100644
--- a/lib3/yaml/__init__.py
+++ b/lib3/yaml/__init__.py
@@ -1,7 +1,4 @@
 
-__version__ = '3.08'
-__with_libyaml__ = False
-
 from .error import *
 
 from .tokens import *
@@ -11,6 +8,13 @@ from .nodes import *
 from .loader import *
 from .dumper import *
 
+__version__ = '3.08'
+try:
+    from .cyaml import *
+    __with_libyaml__ = True
+except ImportError:
+    __with_libyaml__ = False
+
 import io
 
 def scan(stream, Loader=Loader):
diff --git a/lib3/yaml/cyaml.py b/lib3/yaml/cyaml.py
new file mode 100644
index 0000000..d5cb87e
--- /dev/null
+++ b/lib3/yaml/cyaml.py
@@ -0,0 +1,85 @@
+
+__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader',
+        'CBaseDumper', 'CSafeDumper', 'CDumper']
+
+from _yaml import CParser, CEmitter
+
+from .constructor import *
+
+from .serializer import *
+from .representer import *
+
+from .resolver import *
+
+class CBaseLoader(CParser, BaseConstructor, BaseResolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        BaseConstructor.__init__(self)
+        BaseResolver.__init__(self)
+
+class CSafeLoader(CParser, SafeConstructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        SafeConstructor.__init__(self)
+        Resolver.__init__(self)
+
+class CLoader(CParser, Constructor, Resolver):
+
+    def __init__(self, stream):
+        CParser.__init__(self, stream)
+        Constructor.__init__(self)
+        Resolver.__init__(self)
+
+class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+class CSafeDumper(CEmitter, SafeRepresenter, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        SafeRepresenter.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
+class CDumper(CEmitter, Serializer, Representer, Resolver):
+
+    def __init__(self, stream,
+            default_style=None, default_flow_style=None,
+            canonical=None, indent=None, width=None,
+            allow_unicode=None, line_break=None,
+            encoding=None, explicit_start=None, explicit_end=None,
+            version=None, tags=None):
+        CEmitter.__init__(self, stream, canonical=canonical,
+                indent=indent, width=width, encoding=encoding,
+                allow_unicode=allow_unicode, line_break=line_break,
+                explicit_start=explicit_start, explicit_end=explicit_end,
+                version=version, tags=tags)
+        Representer.__init__(self, default_style=default_style,
+                default_flow_style=default_flow_style)
+        Resolver.__init__(self)
+
diff --git a/setup.py b/setup.py
index 9d73944..3f396a9 100644
--- a/setup.py
+++ b/setup.py
@@ -73,12 +73,19 @@ if 'setuptools.extension' in sys.modules:
     sys.modules['distutils.extension'].Extension = _Extension
     sys.modules['distutils.command.build_ext'].Extension = _Extension
 
-try:
-    from Pyrex.Distutils import Extension as _Extension
-    from Pyrex.Distutils import build_ext as _build_ext
-    with_pyrex = True
-except ImportError:
-    with_pyrex = False
+with_pyrex = None
+if sys.version_info[0] < 3:
+    try:
+        from Cython.Distutils.extension import Extension as _Extension
+        from Cython.Distutils import build_ext as _build_ext
+        with_pyrex = 'cython'
+    except ImportError:
+        try:
+            from Pyrex.Distutils import Extension as _Extension
+            from Pyrex.Distutils import build_ext as _build_ext
+            with_pyrex = 'pyrex'
+        except ImportError:
+            pass
 
 
 class Distribution(_Distribution):
@@ -167,8 +174,10 @@ class build_ext(_build_ext):
         self.check_extensions_list(self.extensions)
         filenames = []
         for ext in self.extensions:
-            if with_pyrex:
+            if with_pyrex == 'pyrex':
                 self.pyrex_sources(ext.sources, ext)
+            elif with_pyrex == 'cython':
+                self.cython_sources(ext.sources, ext)
             for filename in ext.sources:
                 filenames.append(filename)
                 base = os.path.splitext(filename)[0]
@@ -197,8 +206,10 @@ class build_ext(_build_ext):
                 with_ext = self.check_extension_availability(ext)
             if not with_ext:
                 continue
-            if with_pyrex:
+            if with_pyrex == 'pyrex':
                 ext.sources = self.pyrex_sources(ext.sources, ext)
+            elif with_pyrex == 'cython':
+                ext.sources = self.cython_sources(ext.sources, ext)
             self.build_extension(ext)
 
     def check_extension_availability(self, ext):
@@ -297,57 +308,37 @@ class test(Command):
 
 if __name__ == '__main__':
 
-    if sys.version_info[0] < 3:
-
-        setup(
-            name=NAME,
-            version=VERSION,
-            description=DESCRIPTION,
-            long_description=LONG_DESCRIPTION,
-            author=AUTHOR,
-            author_email=AUTHOR_EMAIL,
-            license=LICENSE,
-            platforms=PLATFORMS,
-            url=URL,
-            download_url=DOWNLOAD_URL,
-            classifiers=CLASSIFIERS,
-
-            package_dir={'': 'lib'},
-            packages=['yaml'],
-            ext_modules=[
-                Extension('_yaml', ['ext/_yaml.pyx'],
-                    'libyaml', "LibYAML bindings", LIBYAML_CHECK,
-                    libraries=['yaml']),
-            ],
-
-            distclass=Distribution,
-            cmdclass={
-                'build_ext': build_ext,
-                'bdist_rpm': bdist_rpm,
-                'test': test,
-            },
-        )
-
-    else:
-
-        setup(
-            name=NAME,
-            version=VERSION,
-            description=DESCRIPTION,
-            long_description=LONG_DESCRIPTION,
-            author=AUTHOR,
-            author_email=AUTHOR_EMAIL,
-            license=LICENSE,
-            platforms=PLATFORMS,
-            url=URL,
-            download_url=DOWNLOAD_URL,
-            classifiers=CLASSIFIERS,
-
-            package_dir={'': 'lib3'},
-            packages=['yaml'],
-
-            cmdclass={
-                'test': test,
-            },
-        )
+    package_dir = {
+            '2': 'lib',
+    }
+
+    setup(
+        name=NAME,
+        version=VERSION,
+        description=DESCRIPTION,
+        long_description=LONG_DESCRIPTION,
+        author=AUTHOR,
+        author_email=AUTHOR_EMAIL,
+        license=LICENSE,
+        platforms=PLATFORMS,
+        url=URL,
+        download_url=DOWNLOAD_URL,
+        classifiers=CLASSIFIERS,
+
+        package_dir={'': {2: 'lib', 3: 'lib3'}[sys.version_info[0]]},
+        packages=['yaml'],
+        ext_modules=[
+            Extension('_yaml', ['ext/_yaml.pyx'],
+                'libyaml', "LibYAML bindings", LIBYAML_CHECK,
+                libraries=['yaml']),
+        ],
+
+        distclass=Distribution,
+
+        cmdclass={
+            'build_ext': build_ext,
+            'bdist_rpm': bdist_rpm,
+            'test': test,
+        },
+    )
 
diff --git a/tests/lib3/test_structure.py b/tests/lib3/test_structure.py
index 4d9c4ea..6d6f59d 100644
--- a/tests/lib3/test_structure.py
+++ b/tests/lib3/test_structure.py
@@ -139,7 +139,7 @@ def _make_loader():
             return tuple(yaml.Loader.construct_sequence(self, node))
         def construct_mapping(self, node):
             pairs = self.construct_pairs(node)
-            pairs.sort()
+            pairs.sort(key=(lambda i: str(i)))
             return pairs
         def construct_undefined(self, node):
             return self.construct_scalar(node)
@@ -155,7 +155,7 @@ def _make_canonical_loader():
             return tuple(yaml.CanonicalLoader.construct_sequence(self, node))
         def construct_mapping(self, node):
             pairs = self.construct_pairs(node)
-            pairs.sort()
+            pairs.sort(key=(lambda i: str(i)))
             return pairs
         def construct_undefined(self, node):
             return self.construct_scalar(node)
diff --git a/tests/lib3/test_yaml_ext.py b/tests/lib3/test_yaml_ext.py
index 57c02c6..8e36e5d 100644
--- a/tests/lib3/test_yaml_ext.py
+++ b/tests/lib3/test_yaml_ext.py
@@ -252,15 +252,13 @@ def wrap_ext(collections):
     for collection in collections:
         if not isinstance(collection, dict):
             collection = vars(collection)
-        keys = collection.keys()
-        keys.sort()
-        for key in keys:
+        for key in sorted(collection):
             value = collection[key]
             if isinstance(value, types.FunctionType) and hasattr(value, 'unittest'):
                 functions.append(wrap_ext_function(value))
     for function in functions:
-        assert function.unittest_name not in globals()
-        globals()[function.unittest_name] = function
+        assert function.__name__ not in globals()
+        globals()[function.__name__] = function
 
 import test_tokens, test_structure, test_errors, test_resolver, test_constructor,   \
         test_emitter, test_representer, test_recursive