merge

10 files changed, 160 insertions, 141 deletions

diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py
index b9a74c393..f07d70e7b 100644
--- a/Cython/Compiler/Buffer.py
+++ b/Cython/Compiler/Buffer.py
@@ -525,84 +525,109 @@ def buf_lookup_fortran_code(proto, defin, name, nd):
 
 
 def use_py2_buffer_functions(env):
+    env.use_utility_code(GetAndReleaseBufferUtilityCode())
+
+class GetAndReleaseBufferUtilityCode(object):
     # Emulation of PyObject_GetBuffer and PyBuffer_Release for Python 2.
     # For >= 2.6 we do double mode -- use the new buffer interface on objects
     # which has the right tp_flags set, but emulation otherwise.
 
-    # Search all types for __getbuffer__ overloads
-    types = []
-    visited_scopes = set()
-    def find_buffer_types(scope):
-        if scope in visited_scopes:
-            return
-        visited_scopes.add(scope)
-        for m in scope.cimported_modules:
-            find_buffer_types(m)
-        for e in scope.type_entries:
-            t = e.type
-            if t.is_extension_type:
-                if e.name == 'array' and not e.used:
-                    continue
-                release = get = None
-                for x in t.scope.pyfunc_entries:
-                    if x.name == u"__getbuffer__": get = x.func_cname
-                    elif x.name == u"__releasebuffer__": release = x.func_cname
-                if get:
-                    types.append((t.typeptr_cname, get, release))
-
-    find_buffer_types(env)
+    requires = None
 
-    code = dedent("""
-        #if PY_MAJOR_VERSION < 3
-        static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
-          #if PY_VERSION_HEX >= 0x02060000
-          if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_NEWBUFFER)
-              return PyObject_GetBuffer(obj, view, flags);
-          #endif
-    """)
-    if len(types) > 0:
-        clause = "if"
-        for t, get, release in types:
-            code += "  %s (PyObject_TypeCheck(obj, %s)) return %s(obj, view, flags);\n" % (clause, t, get)
-            clause = "else if"
-        code += "  else {\n"
-    code += dedent("""\
-        PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);
-        return -1;
-    """, 2)
-    if len(types) > 0: code += "  }"
-    code += dedent("""
-        }
+    def __init__(self):
+        pass
+
+    def __eq__(self, other):
+        return isinstance(other, GetAndReleaseBufferUtilityCode)
+
+    def __hash__(self):
+        return 24342342
+
+    def get_tree(self): pass
 
-        static void __Pyx_ReleaseBuffer(Py_buffer *view) {
-          PyObject* obj = view->obj;
-          if (obj) {
-    """)
-    if len(types) > 0:
-        clause = "if"
-        for t, get, release in types:
-            if release:
-                code += "%s (PyObject_TypeCheck(obj, %s)) %s(obj, view);" % (clause, t, release)
+    def put_code(self, output):
+        code = output['utility_code_def']
+        proto = output['utility_code_proto']
+        env = output.module_node.scope
+        cython_scope = env.context.cython_scope
+
+        proto.put(dedent("""\
+            #if PY_MAJOR_VERSION < 3
+            static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
+            static void __Pyx_ReleaseBuffer(Py_buffer *view);
+            #else
+            #define __Pyx_GetBuffer PyObject_GetBuffer
+            #define __Pyx_ReleaseBuffer PyBuffer_Release
+            #endif
+        """))
+        
+        # Search all types for __getbuffer__ overloads
+        types = []
+        visited_scopes = set()
+        def find_buffer_types(scope):
+            if scope in visited_scopes:
+                return
+            visited_scopes.add(scope)
+            for m in scope.cimported_modules:
+                find_buffer_types(m)
+            for e in scope.type_entries:
+                t = e.type
+                if t.is_extension_type:
+                    if scope is cython_scope and not e.used:
+                        continue
+                    release = get = None
+                    for x in t.scope.pyfunc_entries:
+                        if x.name == u"__getbuffer__": get = x.func_cname
+                        elif x.name == u"__releasebuffer__": release = x.func_cname
+                    if get:
+                        types.append((t.typeptr_cname, get, release))
+
+        find_buffer_types(env)
+
+        code.put(dedent("""
+            #if PY_MAJOR_VERSION < 3
+            static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags) {
+              #if PY_VERSION_HEX >= 0x02060000
+              if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_HAVE_NEWBUFFER)
+                  return PyObject_GetBuffer(obj, view, flags);
+              #endif
+            """))
+        
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                code.putln("  %s (PyObject_TypeCheck(obj, %s)) return %s(obj, view, flags);" % (clause, t, get))
                 clause = "else if"
-    code += dedent("""
-            Py_DECREF(obj);
-            view->obj = NULL;
-          }
-        }
+            code.putln("  else {")
+        code.put(dedent("""\
+            PyErr_Format(PyExc_TypeError, "'%100s' does not have the buffer interface", Py_TYPE(obj)->tp_name);
+            return -1;
+            """, 2))
+        if len(types) > 0:
+            code.putln("  }")
+        code.put(dedent("""\
+             }
+
+            static void __Pyx_ReleaseBuffer(Py_buffer *view) {
+              PyObject* obj = view->obj;
+              if (obj) {
+        """))
+                 
+        if len(types) > 0:
+            clause = "if"
+            for t, get, release in types:
+                if release:
+                    code.putln("%s (PyObject_TypeCheck(obj, %s)) %s(obj, view);" % (clause, t, release))
+                    clause = "else if"
+        code.put(dedent("""
+                Py_DECREF(obj);
+                view->obj = NULL;
+              }
+            }
+
+            #endif
+        """))
 
-        #endif
-    """)
-                   
-    env.use_utility_code(UtilityCode(
-            proto = dedent("""\
-        #if PY_MAJOR_VERSION < 3
-        static int __Pyx_GetBuffer(PyObject *obj, Py_buffer *view, int flags);
-        static void __Pyx_ReleaseBuffer(Py_buffer *view);
-        #else
-        #define __Pyx_GetBuffer PyObject_GetBuffer
-        #define __Pyx_ReleaseBuffer PyBuffer_Release
-        #endif
-    """), impl = code))
 
 
 def mangle_dtype_name(dtype):
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
index 92eb161ba..af422e44d 100644
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -404,7 +404,7 @@ class GlobalState(object):
     ]
     
 
-    def __init__(self, writer, emit_linenums=False):
+    def __init__(self, writer, module_node, emit_linenums=False):
         self.filename_table = {}
         self.filename_list = []
         self.input_file_contents = {}
@@ -413,6 +413,8 @@ class GlobalState(object):
         self.in_utility_code_generation = False
         self.emit_linenums = emit_linenums
         self.parts = {}
+        self.module_node = module_node # because some utility code generation needs it
+                                       # (generating backwards-compatible Get/ReleaseBuffer
 
         self.const_cname_counter = 1
         self.string_const_index = {}
diff --git a/Cython/Compiler/CythonScope.py b/Cython/Compiler/CythonScope.py
index a44182cc6..69af08a56 100644
--- a/Cython/Compiler/CythonScope.py
+++ b/Cython/Compiler/CythonScope.py
@@ -4,6 +4,7 @@ from UtilityCode import CythonUtilityCode
 from Errors import error
 from Scanning import StringSourceDescriptor
 import Options
+import Buffer
 
 class CythonScope(ModuleScope):
     def __init__(self):
@@ -74,6 +75,8 @@ class CythonScope(ModuleScope):
             defining=1,
             cname='__pyx_cython_view__testscope'
         )
+        entry.utility_code_definition = cythonview_testscope_utility_code
+        
 
         for x in ('strided', 'contig', 'follow', 'direct', 'ptr', 'full'):
             entry = viewscope.declare_var(x, py_object_type, None,
@@ -156,7 +159,8 @@ memview_name = u'memoryview'
 memview_typeptr_cname = Naming.typeptr_prefix+memview_name
 memview_typeobj_cname = '__pyx_tobj_'+memview_name
 memview_objstruct_cname = '__pyx_obj_'+memview_name
-view_utility_code = CythonUtilityCode(u"""
+view_utility_code = CythonUtilityCode(
+u"""
 cdef class Enum(object):
     cdef object name
     def __init__(self, name):
@@ -191,7 +195,9 @@ cdef class memoryview(object):
 
 cdef memoryview memoryview_cwrapper(object o, int flags):
     return memoryview(o, flags)
-""", name="view_code", prefix="__pyx_viewaxis_")
+""", name="view_code",
+    prefix="__pyx_viewaxis_",
+    requires=(Buffer.GetAndReleaseBufferUtilityCode(),))
 
 cyarray_prefix = u'__pyx_cythonarray_'
 cython_array_utility_code = CythonUtilityCode(u'''
diff --git a/Cython/Compiler/MemoryView.py b/Cython/Compiler/MemoryView.py
index daae1c4f4..501d044d2 100644
--- a/Cython/Compiler/MemoryView.py
+++ b/Cython/Compiler/MemoryView.py
@@ -117,31 +117,18 @@ def get_buf_flag(specs):
     else:
         return memview_strided_access
 
-def use_cython_view_util_code(env, lu_name):
-    import CythonScope
-    cythonscope = env.global_scope().context.cython_scope
-    viewscope = cythonscope.viewscope
-    entry = viewscope.lookup_here(lu_name)
-    entry.used = 1
-    return entry
-
-def use_cython_util_code(env, lu_name):
-    import CythonScope
-    cythonscope = env.global_scope().context.cython_scope
-    entry = cythonscope.lookup_here(lu_name)
-    entry.used = 1
-    return entry
-
 def use_memview_util_code(env):
     import CythonScope
-    return use_cython_view_util_code(env, CythonScope.memview_name)
+    env.use_utility_code(CythonScope.view_utility_code)
+    env.use_utility_code(memviewslice_declare_code)
 
 def use_memview_cwrap(env):
     import CythonScope
-    return use_cython_view_util_code(env, CythonScope.memview_cwrap_name)
+    env.use_utility_code(CythonScope.view_utility_code)
 
 def use_cython_array(env):
-    return use_cython_util_code(env, 'array')
+    import CythonScope
+    env.use_utility_code(CythonScope.cython_array_utility_code)
 
 def src_conforms_to_dst(src, dst):
     '''
@@ -326,7 +313,7 @@ def memoryviewslice_get_copy_func(from_memview, to_memview, mode, scope):
 
     copy_contents_name = get_copy_contents_name(from_memview, to_memview)
 
-    scope.declare_cfunction(cython_name,
+    entry = scope.declare_cfunction(cython_name,
                 CFuncType(from_memview,
                     [CFuncTypeArg("memviewslice", from_memview, None)]),
                 pos = None,
@@ -343,7 +330,7 @@ def memoryviewslice_get_copy_func(from_memview, to_memview, mode, scope):
     copy_decl = ("static __Pyx_memviewslice "
                 "%s(const __Pyx_memviewslice); /* proto */\n" % (copy_name,))
 
-    return (copy_decl, copy_impl)
+    return (copy_decl, copy_impl, entry)
 
 def get_copy_contents_func(from_mvs, to_mvs, cfunc_name):
     assert from_mvs.dtype == to_mvs.dtype
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
index 36ee016cb..74dc895dd 100644
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -120,7 +120,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if h_types or h_vars or h_funcs or h_extension_types:
             result.h_file = replace_suffix(result.c_file, ".h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code)
+            Code.GlobalState(h_code, self)
             if options.generate_pxi:
                 result.i_file = replace_suffix(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
@@ -182,7 +182,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if api_funcs or has_api_extension_types:
             result.api_file = replace_suffix(result.c_file, "_api.h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code)
+            Code.GlobalState(h_code, self)
             name = self.api_name(env)
             guard = Naming.api_guard_prefix + name
             h_code.put_h_guard(guard)
@@ -266,7 +266,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         else:
             emit_linenums = options.emit_linenums
             rootwriter = Code.CCodeWriter(emit_linenums=emit_linenums)
-        globalstate = Code.GlobalState(rootwriter, emit_linenums)
+        globalstate = Code.GlobalState(rootwriter, self, emit_linenums)
         globalstate.initialize_main_c_code()
         h_code = globalstate['h_code']
         
diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py
index 4eb7ba59b..5ddec7585 100644
--- a/Cython/Compiler/Nodes.py
+++ b/Cython/Compiler/Nodes.py
@@ -727,8 +727,11 @@ class MemoryViewSliceTypeNode(CBaseTypeNode):
             self.type = PyrexTypes.ErrorType()
             return self.type
 
-        self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs, env)
+        self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
         MemoryView.use_memview_util_code(env)
+        MemoryView.use_cython_array(env)
+        MemoryView.use_memview_util_code(env)
+        env.use_utility_code(MemoryView.memviewslice_declare_code)
         return self.type
 
 class CBufferAccessTypeNode(CBaseTypeNode):
diff --git a/Cython/Compiler/Pipeline.py b/Cython/Compiler/Pipeline.py
index 7a199625f..c88756b49 100644
--- a/Cython/Compiler/Pipeline.py
+++ b/Cython/Compiler/Pipeline.py
@@ -1,6 +1,7 @@
 from Errors import PyrexError, CompileError, InternalError, error
 import Errors
 import DebugFlags
+from Visitor import CythonTransform
 
 #
 # Really small pipeline stages.
@@ -48,31 +49,43 @@ def inject_pxd_code_stage_factory(context):
         return module_node
     return inject_pxd_code_stage
 
-def use_utility_code_definitions(scope, target):
-    for entry in scope.entries.itervalues():
-        if entry.used and entry.utility_code_definition:
-            target.use_utility_code(entry.utility_code_definition)
-        elif entry.as_module:
-            use_utility_code_definitions(entry.as_module, target)
-
 def inject_utility_code_stage_factory(context):
     def inject_utility_code_stage(module_node):
-        # First, make sure any utility code pulled in by using symbols in the cython
-        # scope is included
-        use_utility_code_definitions(context.cython_scope, module_node.scope)
-        
         added = []
         # Note: the list might be extended inside the loop (if some utility code
-        # pulls in other utility code)
+        # pulls in other utility code, explicitly or implicitly)
         for utilcode in module_node.scope.utility_code_list:
             if utilcode in added: continue
             added.append(utilcode)
+            if utilcode.requires:
+                for dep in utilcode.requires:
+                    if not dep in added and not dep in module_node.scope.utility_code_list:
+                        module_node.scope.utility_code_list.append(dep)
             tree = utilcode.get_tree()
             if tree:
                 module_node.merge_in(tree.body, tree.scope, merge_scope=True)
         return module_node
     return inject_utility_code_stage
 
+class UseUtilityCodeDefinitions(CythonTransform):
+    # Temporary hack to use any utility code in nodes' "utility_code_definitions".
+    # This should be moved to the code generation phase of the relevant nodes once
+    # it is safe to generate CythonUtilityCode at code generation time.
+    def __call__(self, node):
+        self.scope = node.scope
+        return super(UseUtilityCodeDefinitions, self).__call__(node)
+
+    def visit_AttributeNode(self, node):
+        if node.entry and node.entry.utility_code_definition:
+            self.scope.use_utility_code(node.entry.utility_code_definition)
+        return node
+
+    def visit_NameNode(self, node):
+        for e in (node.entry, node.type_entry):
+            if e and e.utility_code_definition:
+                self.scope.use_utility_code(e.utility_code_definition)
+        return node
+                     
 #
 # Pipeline factories
 #
@@ -139,6 +152,7 @@ def create_pipeline(context, mode):
         SwitchTransform(),
         FinalOptimizePhase(context),
         GilCheck(),
+        UseUtilityCodeDefinitions(context),
     #            ClearResultCodes(context),
     #            SpecialFunctions(context),
         #        CreateClosureClasses(context),
diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
index 3f66b3b09..1d51f3620 100644
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -234,7 +234,7 @@ class MemoryViewSliceType(PyrexType):
     has_attributes = 1
     scope = None
 
-    def __init__(self, base_dtype, axes, env):
+    def __init__(self, base_dtype, axes):
         '''
         MemoryViewSliceType(base, axes)
 
@@ -271,7 +271,6 @@ class MemoryViewSliceType(PyrexType):
 
         self.dtype = base_dtype
         self.axes = axes
-        self.env = env
 
         import MemoryView
         self.is_c_contig, self.is_f_contig = MemoryView.is_cf_contig(self.axes)
@@ -290,8 +289,6 @@ class MemoryViewSliceType(PyrexType):
         assert not pyrex
         assert not dll_linkage
         import MemoryView
-        if not for_display:
-            self.env.use_utility_code(MemoryView.memviewslice_declare_code)
         return self.base_declaration_code(
                 MemoryView.memviewslice_cname,
                 entity_code)
@@ -301,11 +298,10 @@ class MemoryViewSliceType(PyrexType):
 
             import Symtab, MemoryView
             from MemoryView import axes_to_str
-
             self.scope = scope = Symtab.CClassScope(
                     'mvs_class_'+self.specialization_suffix(),
-                    self.env.global_scope(),
-                    visibility='private')
+                    None,
+                    visibility='extern')
 
             scope.parent_type = self
 
@@ -320,17 +316,17 @@ class MemoryViewSliceType(PyrexType):
                 to_axes_c = [('direct', 'follow')]*(ndim-1) + to_axes_c
                 to_axes_f = to_axes_f + [('direct', 'follow')]*(ndim-1)
 
-            to_memview_c = MemoryViewSliceType(self.dtype, to_axes_c, self.env)
-            to_memview_f = MemoryViewSliceType(self.dtype, to_axes_f, self.env)
+            to_memview_c = MemoryViewSliceType(self.dtype, to_axes_c)
+            to_memview_f = MemoryViewSliceType(self.dtype, to_axes_f)
 
             copy_contents_name_c =\
                     MemoryView.get_copy_contents_name(self, to_memview_c)
             copy_contents_name_f =\
                     MemoryView.get_copy_contents_name(self, to_memview_f)
 
-            c_copy_decl, c_copy_impl = \
+            c_copy_decl, c_copy_impl, c_entry = \
                     MemoryView.memoryviewslice_get_copy_func(self, to_memview_c, 'c', self.scope)
-            f_copy_decl, f_copy_impl = \
+            f_copy_decl, f_copy_impl, f_entry = \
                     MemoryView.memoryviewslice_get_copy_func(self, to_memview_f, 'fortran', self.scope)
 
             c_copy_contents_decl, c_copy_contents_impl = \
@@ -347,19 +343,13 @@ class MemoryViewSliceType(PyrexType):
                     proto = f_copy_decl,
                     impl = f_copy_impl)
 
+            c_entry.utility_code_definition = c_util_code
+            f_entry.utility_code_definition = f_util_code           
+
             if copy_contents_name_c != copy_contents_name_f:
                 f_util_code.proto += f_copy_contents_decl
                 f_util_code.impl  += f_copy_contents_impl
 
-            c_copy_used = [1 for util_code in self.env.global_scope().utility_code_list if c_util_code.proto == util_code.proto]
-            f_copy_used = [1 for util_code in self.env.global_scope().utility_code_list if f_util_code.proto == util_code.proto]
-
-            if not c_copy_used:
-                self.env.use_utility_code(c_util_code)
-
-            if not f_copy_used:
-                self.env.use_utility_code(f_util_code)
-
             # is_c_contiguous and is_f_contiguous functions
 
             for c_or_f, cython_name in (('c', 'is_c_contig'), ('fortran', 'is_f_contig')):
@@ -377,17 +367,6 @@ class MemoryViewSliceType(PyrexType):
                 contig_util_code = UtilityCode(
                         proto = contig_decl, impl = contig_impl)
 
-                contig_used = [1 for util_code in \
-                        self.env.global_scope().utility_code_list \
-                        if contig_decl == util_code.proto]
-
-                if not contig_used:
-                    self.env.use_utility_code(contig_util_code)
-
-            # use the supporting util code
-            MemoryView.use_cython_array(self.env)
-            MemoryView.use_memview_util_code(self.env)
-
         return True
 
     def specialization_suffix(self):
diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py
index 00d8d2231..941b31255 100644
--- a/Cython/Compiler/Symtab.py
+++ b/Cython/Compiler/Symtab.py
@@ -119,6 +119,8 @@ class Entry(object):
     #                             which contains the definition of the entry.
     #                             Currently only supported for CythonScope entries.
 
+    # TODO: utility_code and utility_code_definition serves the same purpose...
+
     inline_func_in_pxd = False
     borrowed = 0
     init = ""
diff --git a/Cython/Compiler/UtilityCode.py b/Cython/Compiler/UtilityCode.py
index 8388b379a..e51d16df9 100644
--- a/Cython/Compiler/UtilityCode.py
+++ b/Cython/Compiler/UtilityCode.py
@@ -35,7 +35,7 @@ class CythonUtilityCode:
     Utility code written in the Cython language itself.
     """
 
-    def __init__(self, pyx, name="__pyxutil", prefix=""):
+    def __init__(self, pyx, name="__pyxutil", prefix="", requires=None):
         # 1) We need to delay the parsing/processing, so that all modules can be
         #    imported without import loops
         # 2) The same utility code object can be used for multiple source files;
@@ -45,6 +45,7 @@ class CythonUtilityCode:
         self.pyx = pyx
         self.name = name
         self.prefix = prefix
+        self.requires = requires
 
     def get_tree(self):
         import Pipeline