A gentle start on #469gsoc-kurt-regraft

6 files changed, 130 insertions, 87 deletions

diff --git a/Cython/Compiler/Annotate.py b/Cython/Compiler/Annotate.py
index d252fda61..c4642d49a 100644
--- a/Cython/Compiler/Annotate.py
+++ b/Cython/Compiler/Annotate.py
@@ -19,8 +19,8 @@ line_pos_comment = re.compile(r'/\*.*?<<<<<<<<<<<<<<.*?\*/\n*', re.DOTALL)
 
 class AnnotationCCodeWriter(CCodeWriter):
 
-    def __init__(self, create_from=None, buffer=None, copy_formatting=True):
-        CCodeWriter.__init__(self, create_from, buffer, copy_formatting=True)
+    def __init__(self, create_from=None, buffer=None, copy_formatting=True, description=None):
+        CCodeWriter.__init__(self, create_from, buffer, copy_formatting=True, description=description)
         if create_from is None:
             self.annotation_buffer = StringIO()
             self.annotations = []
@@ -33,8 +33,8 @@ class AnnotationCCodeWriter(CCodeWriter):
             self.code = create_from.code
             self.last_pos = create_from.last_pos
     
-    def create_new(self, create_from, buffer, copy_formatting):
-        return AnnotationCCodeWriter(create_from, buffer, copy_formatting)
+    def create_new(self, create_from, buffer, copy_formatting, description=None):
+        return AnnotationCCodeWriter(create_from, buffer, copy_formatting, description=description)
 
     def write(self, s):
         CCodeWriter.write(self, s)
diff --git a/Cython/Compiler/Buffer.py b/Cython/Compiler/Buffer.py
index e7199ce1c..c9ebaf43b 100644
--- a/Cython/Compiler/Buffer.py
+++ b/Cython/Compiler/Buffer.py
@@ -445,8 +445,8 @@ def put_buffer_lookup_code(entry, index_signeds, index_cnames, directives, pos,
             params.append(s)
         
     # Make sure the utility code is available
-    if funcname not in code.globalstate.utility_codes:
-        code.globalstate.utility_codes.add(funcname)
+    if funcname not in code.globalstate.processed_objects:
+        code.globalstate.processed_objects.add(funcname)
         protocode = code.globalstate['utility_code_proto']
         defcode = code.globalstate['utility_code_def']
         funcgen(protocode, defcode, name=funcname, nd=nd)
@@ -665,8 +665,8 @@ def get_type_information_cname(code, dtype, maxdepth=None):
     if maxdepth <= 0:
         assert False
 
-    if name not in code.globalstate.utility_codes:
-        code.globalstate.utility_codes.add(name)
+    if name not in code.globalstate.processed_objects:
+        code.globalstate.processed_objects.add(name)
         typecode = code.globalstate['typeinfo']
         
         complex_possible = dtype.is_struct_or_union and dtype.can_be_complex()
diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py
index 845dbe548..935990ca8 100644
--- a/Cython/Compiler/Code.py
+++ b/Cython/Compiler/Code.py
@@ -21,8 +21,6 @@ class UtilityCode(object):
     # Stores utility code to add during code generation.
     #
     # See GlobalState.put_utility_code.
-    #
-    # hashes/equals by instance
 
     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
                  proto_block='utility_code_proto'):
@@ -62,9 +60,6 @@ class UtilityCode(object):
             return s
 
     def put_code(self, output):
-        if self.requires:
-            for dependency in self.requires:
-                output.use_utility_code(dependency)
         if self.proto:
             output[self.proto_block].put(self.proto)
         if self.impl:
@@ -384,8 +379,6 @@ class GlobalState(object):
     #                                  to create this output C code.  This is
     #                                  used to annotate the comments.
     #
-    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
-    #
     # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
     #                                  constants etc. into the pyx-declared ones (i.e,
     #                                  check if constants are already added).
@@ -394,6 +387,8 @@ class GlobalState(object):
     #
     # const_cname_counter int          global counter for constant identifiers
     #
+    # processed_objects  set()         objects which has had their C code generated
+    # process_object_stack [object]    (internal use)
 
     # parts            {string:CCodeWriter}
 
@@ -434,11 +429,13 @@ class GlobalState(object):
     ]
     
 
-    def __init__(self, writer, module_node, emit_linenums=False):
+    def __init__(self, writer, module_node, emit_linenums=False,
+                 header_mode=False):
         self.filename_table = {}
         self.filename_list = []
         self.input_file_contents = {}
-        self.utility_codes = set()
+        self.process_objects_stack = []
+        self.processed_objects = set()
         self.declared_cnames = {}
         self.in_utility_code_generation = False
         self.emit_linenums = emit_linenums
@@ -454,11 +451,14 @@ class GlobalState(object):
         assert writer.globalstate is None
         writer.globalstate = self
         self.rootwriter = writer
+        self.header_mode = header_mode
+        if header_mode:
+            self.parts = {'header' : writer }
 
     def initialize_main_c_code(self):
         rootwriter = self.rootwriter
         for part in self.code_layout:
-            self.parts[part] = rootwriter.insertion_point()
+            self.parts[part] = rootwriter.insertion_point(part)
 
         if not Options.cache_builtins:
             del self.parts['cached_builtins']
@@ -777,21 +777,38 @@ class GlobalState(object):
             return F
 
     #
-    # Utility code state
+    # Stuff about asking objects to output themselves to C
+    # Utility code handled here.
     #
-    
-    def use_utility_code(self, utility_code):
+    def put_object(self, obj):
         """
-        Adds code to the C file. utility_code should
-        a) implement __eq__/__hash__ for the purpose of knowing whether the same
-           code has already been included
-        b) implement put_code, which takes a globalstate instance
-
-        See UtilityCode.
+        Write the C code associated with the given object (such as a type definition,
+        some utility code, etc.). The object:
+         - Must have a "requires" attribute (list of other objects which
+           must be output first)
+         - Must have an __eq__/__hash__, used to determine whether the same
+           object has already been output
+         - Must have a method put_code(self, output), which is called to output
+           the object to C. Output should be indexed with the required part, e.g.
+           writer = output['utility_code_proto']
+
+        There's no guarantee on when put_code will be called, but currently it
+        happens right away (but only once per object).
         """
-        if utility_code not in self.utility_codes:
-            self.utility_codes.add(utility_code)
-            utility_code.put_code(self)
+        if obj in self.processed_objects:
+            return
+        if obj in self.process_objects_stack:
+            raise AssertionError("Object dependency graph is not a DAG, found a loop")
+        self.process_objects_stack.append(obj)
+        if obj.requires is not None and obj.requires:
+            for req in obj.requires:
+                self.put_object(req)
+        self.process_objects_stack.pop()
+        self.processed_objects.add(obj)
+        obj.put_code(self)
+    
+    def use_utility_code(self, utility_code):
+        self.put_object(utility_code)
 
 
 def funccontext_property(name):
@@ -843,7 +860,11 @@ class CCodeWriter(object):
 
     globalstate = None
     
-    def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None):
+    def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None,
+                 description=None):
+        if description is None:
+            description = '<not specified>'
+        self.description = description
         if buffer is None: buffer = StringIOTree()
         self.buffer = buffer
         self.marker = None
@@ -869,10 +890,13 @@ class CCodeWriter(object):
         else:
             self.emit_linenums = emit_linenums
 
-    def create_new(self, create_from, buffer, copy_formatting):
+    def __repr__(self):
+        return '%s@%s' % (object.__repr__(self), self.description)
+
+    def create_new(self, create_from, buffer, copy_formatting, description='(some create_new)'):
         # polymorphic constructor -- very slightly more versatile
         # than using __class__
-        result = CCodeWriter(create_from, buffer, copy_formatting)
+        result = CCodeWriter(create_from, buffer, copy_formatting, description=description)
         return result
 
     def copyto(self, f):
@@ -884,8 +908,9 @@ class CCodeWriter(object):
     def write(self, s):
         self.buffer.write(s)
 
-    def insertion_point(self):
-        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
+    def insertion_point(self, description='(some insertion point)'):
+        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(),
+                                copy_formatting=True, description=description)
         return other
 
     def new_writer(self):
@@ -1331,3 +1356,15 @@ class PyrexCodeWriter(object):
     def dedent(self):
         self.level -= 1
 
+
+#
+# C writing utility functions
+#
+def sue_header_footer(type, kind, name):
+    if type.typedef_flag:
+        header = "typedef %s {" % kind
+        footer = "} %s;" % name
+    else:
+        header = "%s %s {" % (kind, name)
+        footer = "};"
+    return header, footer        
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
index f5d8e6737..8bf214821 100644
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -124,7 +124,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         if h_types or h_vars or h_funcs or h_extension_types:
             result.h_file = replace_suffix(result.c_file, ".h")
             h_code = Code.CCodeWriter()
-            Code.GlobalState(h_code, self)
+            Code.GlobalState(h_code, self, header_mode=True)
             if options.generate_pxi:
                 result.i_file = replace_suffix(result.c_file, ".pxi")
                 i_code = Code.PyrexCodeWriter(result.i_file)
@@ -408,10 +408,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                         self.generate_struct_union_definition(entry, code)
                     elif type.is_enum:
                         self.generate_enum_definition(entry, code)
-                    elif type.is_extension_type and entry not in vtabslot_entries:
-                        self.generate_obj_struct_definition(type, code)
-        for entry in vtabslot_list:
-            self.generate_obj_struct_definition(entry.type, code)
+                    elif type.is_extension_type:# and entry not in vtabslot_entries:
+                        code.globalstate.put_object(type)
         for entry in vtab_list:
             self.generate_typeobject_predeclaration(entry, code)
             self.generate_exttype_vtable_struct(entry, code)
@@ -660,7 +658,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         #for entry in env.type_entries:
         for entry in type_entries:
             if not entry.in_cinclude:
-                #print "generate_type_header_code:", entry.name, repr(entry.type) ###
                 type = entry.type
                 if type.is_typedef: # Must test this first!
                     self.generate_typedef(entry, code)
@@ -669,7 +666,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 elif type.is_enum:
                     self.generate_enum_definition(entry, code)
                 elif type.is_extension_type:
-                    self.generate_obj_struct_definition(type, code)
+                    code.globalstate.put_object(type)
         
     def generate_gcc33_hack(self, env, code):
         # Workaround for spurious warning generation in gcc 3.3
@@ -694,15 +691,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         writer.putln("")
         writer.putln("typedef %s;" % base_type.declaration_code(entry.cname))
 
-    def sue_header_footer(self, type, kind, name):
-        if type.typedef_flag:
-            header = "typedef %s {" % kind
-            footer = "} %s;" % name
-        else:
-            header = "%s %s {" % (kind, name)
-            footer = "};"
-        return header, footer
-    
     def generate_struct_union_definition(self, entry, code):
         code.mark_pos(entry.pos)
         type = entry.type
@@ -714,7 +702,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                 kind = "%s %s" % (type.kind, "__Pyx_PACKED")
                 code.globalstate.use_utility_code(packed_struct_utility_code)
             header, footer = \
-                self.sue_header_footer(type, kind, type.cname)
+                Code.sue_header_footer(type, kind, type.cname)
             code.putln("")
             if packed:
                 code.putln("#if !defined(__GNUC__)")
@@ -741,7 +729,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         type = entry.type
         name = entry.cname or entry.name or ""
         header, footer = \
-            self.sue_header_footer(type, "enum", name)
+            Code.sue_header_footer(type, "enum", name)
         code.putln("")
         code.putln(header)
         enum_values = entry.enum_values
@@ -814,40 +802,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             code.putln("static struct %s *%s;" % (
                 type.vtabstruct_cname,
                 type.vtabptr_cname))
-    
-    def generate_obj_struct_definition(self, type, code):
-        code.mark_pos(type.pos)
-        # Generate object struct definition for an
-        # extension type.
-        if not type.scope:
-            return # Forward declared but never defined
-        header, footer = \
-            self.sue_header_footer(type, "struct", type.objstruct_cname)
-        code.putln("")
-        code.putln(header)
-        base_type = type.base_type
-        if base_type:
-            code.putln(
-                "%s%s %s;" % (
-                    ("struct ", "")[base_type.typedef_flag],
-                    base_type.objstruct_cname,
-                    Naming.obj_base_cname))
-        else:
-            code.putln(
-                "PyObject_HEAD")
-        if type.vtabslot_cname and not (type.base_type and type.base_type.vtabslot_cname):
-            code.putln(
-                "struct %s *%s;" % (
-                    type.vtabstruct_cname,
-                    type.vtabslot_cname))
-        for attr in type.scope.var_entries:
-            code.putln(
-                "%s;" %
-                    attr.type.declaration_code(attr.cname))
-        code.putln(footer)
-        if type.objtypedef_cname is not None:
-            # Only for exposing public typedef name.
-            code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname))
 
     def generate_global_declarations(self, env, code, definition):
         code.putln("")
diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
index 66c114f2f..1409a2d9e 100644
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -106,6 +106,7 @@ class PyrexType(BaseType):
     has_attributes = 0
     default_value = ""
     pymemberdef_typecode = None
+    requires = None
     
     def resolve(self):
         # If a typedef, returns the base type.
@@ -645,6 +646,7 @@ class PyExtensionType(PyObjectType):
     #  vtabstruct_cname string           Name of C method table struct
     #  vtabptr_cname    string           Name of pointer to C method table
     #  vtable_cname     string           Name of C method table definition
+    #  in_cinclude      boolean          Corresponds to the in_cinclude of the corresponding entry
     
     is_extension_type = 1
     has_attributes = 1
@@ -670,6 +672,10 @@ class PyExtensionType(PyObjectType):
         self.vtabptr_cname = None
         self.vtable_cname = None
         self.is_external = is_external
+        if base_type:
+            self.requires = [base_type]
+        else:
+            self.requires = ()
     
     def set_scope(self, scope):
         self.scope = scope
@@ -727,6 +733,50 @@ class PyExtensionType(PyObjectType):
         return "<PyExtensionType %s%s>" % (self.scope.class_name,
             ("", " typedef")[self.typedef_flag])
     
+    
+    def generate_obj_struct_definition(self, code):
+        import Code
+        code.mark_pos(self.pos)
+        # Generate object struct definition for an
+        # extension type.
+        if not self.scope:
+            return # Forward declared but never defined
+        header, footer = \
+            Code.sue_header_footer(self, "struct", self.objstruct_cname)
+        code.putln("")
+        code.putln(header)
+        base_type = self.base_type
+        if base_type:
+            code.putln(
+                "%s%s %s;" % (
+                    ("struct ", "")[base_type.typedef_flag],
+                    base_type.objstruct_cname,
+                    Naming.obj_base_cname))
+        else:
+            code.putln(
+                "PyObject_HEAD")
+        if self.vtabslot_cname and not (self.base_type and self.base_type.vtabslot_cname):
+            code.putln(
+                "struct %s *%s;" % (
+                    self.vtabstruct_cname,
+                    self.vtabslot_cname))
+        for attr in self.scope.var_entries:
+            code.putln(
+                "%s;" %
+                    attr.type.declaration_code(attr.cname))
+        code.putln(footer)
+        if self.objtypedef_cname is not None:
+            # Only for exposing public typedef name.
+            code.putln("typedef struct %s %s;" % (self.objstruct_cname, self.objtypedef_cname))
+
+    def put_code(self, output):
+        if not self.in_cinclude:
+            if output.header_mode:
+                code = output['header']
+            else:
+                code = output['type_declarations']
+            self.generate_obj_struct_definition(code)
+
 
 class CType(PyrexType):
     #
diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py
index 129c38378..fcaaa41af 100644
--- a/Cython/Compiler/Symtab.py
+++ b/Cython/Compiler/Symtab.py
@@ -905,6 +905,7 @@ class ModuleScope(Scope):
         #
         if not entry:
             type = PyrexTypes.PyExtensionType(name, typedef_flag, base_type, visibility == 'extern')
+            # type.in_cinclude is set further down
             type.pos = pos
             type.buffer_defaults = buffer_defaults
             if objtypedef_cname is not None:
@@ -920,6 +921,7 @@ class ModuleScope(Scope):
             entry = self.declare_type(name, type, pos, visibility = visibility,
                 defining = 0)
             entry.is_cclass = True
+            type.in_cinclude = entry.in_cinclude
             if objstruct_cname:
                 type.objstruct_cname = objstruct_cname
             elif not entry.in_cinclude: