Generate "PyType_Spec" structs instead of "PyTypeObject" structs for heap types.gh2343_heap_types

6 files changed, 110 insertions, 31 deletions

diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py
index 35d5e4f6e..6a8b03dd6 100644
--- a/Cython/Compiler/ModuleNode.py
+++ b/Cython/Compiler/ModuleNode.py
@@ -1277,7 +1277,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
                     self.generate_property_accessors(scope, code)
                     self.generate_method_table(scope, code)
                     self.generate_getset_table(scope, code)
+                    code.putln("#if CYTHON_USE_HEAPTYPES")
+                    self.generate_heaptypeobj_definition(entry, code)
+                    code.putln("#else")
                     self.generate_typeobj_definition(full_module_name, entry, code)
+                    code.putln("#endif")
 
     def generate_exttype_vtable(self, scope, code):
         # Generate the definition of an extension type's vtable.
@@ -2166,8 +2170,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             "}")
 
     def generate_typeobj_definition(self, modname, entry, code):
-        type = entry.type
-        scope = type.scope
+        ext_type = entry.type
+        scope = ext_type.scope
         for suite in TypeSlots.substructures:
             suite.generate_substructure(scope, code)
         code.putln("")
@@ -2176,7 +2180,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         else:
             header = "static PyTypeObject %s = {"
         #code.putln(header % scope.parent_type.typeobj_cname)
-        code.putln(header % type.typeobj_cname)
+        code.putln(header % ext_type.typeobj_cname)
         code.putln(
             "PyVarObject_HEAD_INIT(0, 0)")
         classname = scope.class_name.as_c_string_literal()
@@ -2184,10 +2188,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
             '"%s."%s, /*tp_name*/' % (
                 self.full_module_name,
                 classname))
-        if type.typedef_flag:
-            objstruct = type.objstruct_cname
+        if ext_type.typedef_flag:
+            objstruct = ext_type.objstruct_cname
         else:
-            objstruct = "struct %s" % type.objstruct_cname
+            objstruct = "struct %s" % ext_type.objstruct_cname
         code.putln(
             "sizeof(%s), /*tp_basicsize*/" % objstruct)
         code.putln(
@@ -2197,6 +2201,43 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode):
         code.putln(
             "};")
 
+    def generate_heaptypeobj_definition(self, entry, code):
+        """
+        Map the 'PyTypeObject' slots to a 'PyType_Spec' struct.
+        """
+        ext_type = entry.type
+        scope = ext_type.scope
+        slot_array_cname = scope.mangle(Naming.typeslots_prefix, ext_type.name)
+        code.putln("")
+        code.putln("static PyType_Slot %s[] = {" % slot_array_cname)
+        for slot in TypeSlots.slot_table:
+            slot.generate(scope, code, as_heap_type=True)
+        code.putln(
+            "};")
+        code.putln("")
+        code.putln("static PyType_Spec %s = {" % ext_type.typespec_cname)
+        classname = scope.class_name.as_c_string_literal()
+        code.putln('"%s."%s, /*tp_name*/' % (
+            self.full_module_name,
+            classname,
+        ))
+        flags_slot = [
+            slot for slot in TypeSlots.slot_table
+            if slot.slot_name == 'tp_flags'
+        ][0]
+        if ext_type.typedef_flag:
+            objstruct = ext_type.objstruct_cname
+        else:
+            objstruct = "struct %s" % ext_type.objstruct_cname
+        code.putln(
+            "sizeof(%s), /*tp_basicsize*/" % objstruct)
+        code.putln(
+            "0, /*tp_itemsize*/")
+        code.putln('%s, /*tp_flags*/' % flags_slot.slot_code(scope))
+        code.putln('%s /*slots*/' % slot_array_cname)
+        code.putln(
+            "};")
+
     def generate_method_table(self, env, code):
         if env.is_c_class_scope and not env.pyfunc_entries:
             return
diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py
index 304ef9188..1bb678fa6 100644
--- a/Cython/Compiler/Naming.py
+++ b/Cython/Compiler/Naming.py
@@ -38,6 +38,8 @@ typeptr_prefix    = pyrex_prefix + "ptype_"
 prop_set_prefix   = pyrex_prefix + "setprop_"
 type_prefix       = pyrex_prefix + "t_"
 typeobj_prefix    = pyrex_prefix + "type_"
+typeslots_prefix  = pyrex_prefix + "typeslots_"
+typespec_prefix   = pyrex_prefix + "typespec_"
 var_prefix        = pyrex_prefix + "v_"
 varptr_prefix     = pyrex_prefix + "vp_"
 varptr_prefix_api = pyrex_prefix + "api_vp_"
diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py
index f54e7a6eb..9329019d6 100644
--- a/Cython/Compiler/PyrexTypes.py
+++ b/Cython/Compiler/PyrexTypes.py
@@ -1351,6 +1351,7 @@ class PyExtensionType(PyObjectType):
     #  objstruct_cname  string           Name of PyObject struct
     #  objtypedef_cname string           Name of PyObject struct typedef
     #  typeobj_cname    string or None   C code fragment referring to type object
+    #  typespec_cname   string or None   C code fragment referring to type spec
     #  typeptr_cname    string or None   Name of pointer to external type object
     #  vtabslot_cname   string           Name of C method table member
     #  vtabstruct_cname string           Name of C method table struct
@@ -1364,7 +1365,13 @@ class PyExtensionType(PyObjectType):
     has_attributes = 1
     early_init = 1
 
+    # Some object attributes that are not always set, e.g. for external types.
     objtypedef_cname = None
+    typespec_cname = None
+    vtabslot_cname = None
+    vtabstruct_cname = None
+    vtabptr_cname = None
+    vtable_cname = None
 
     def __init__(self, name, typedef_flag, base_type, is_external=0, check_size=None):
         self.name = name
@@ -1377,10 +1384,6 @@ class PyExtensionType(PyObjectType):
         self.objstruct_cname = None
         self.typeobj_cname = None
         self.typeptr_cname = None
-        self.vtabslot_cname = None
-        self.vtabstruct_cname = None
-        self.vtabptr_cname = None
-        self.vtable_cname = None
         self.is_external = is_external
         self.check_size = check_size or 'warn'
         self.defered_declarations = []
diff --git a/Cython/Compiler/Symtab.py b/Cython/Compiler/Symtab.py
index ce35a75e9..fa716f4ad 100644
--- a/Cython/Compiler/Symtab.py
+++ b/Cython/Compiler/Symtab.py
@@ -1664,27 +1664,30 @@ class ModuleScope(Scope):
                 error(entry.pos, "C class '%s' is declared but not defined" % entry.name)
 
     def check_c_class(self, entry):
-        type = entry.type
+        ext_type = entry.type
         name = entry.name
         visibility = entry.visibility
         # Check defined
-        if not type.scope:
+        if not ext_type.scope:
             error(entry.pos, "C class '%s' is declared but not defined" % name)
         # Generate typeobj_cname
-        if visibility != 'extern' and not type.typeobj_cname:
-            type.typeobj_cname = self.mangle(Naming.typeobj_prefix, name)
+        if visibility != 'extern':
+            if not ext_type.typeobj_cname:
+                ext_type.typeobj_cname = self.mangle(Naming.typeobj_prefix, name)
+            if not ext_type.typespec_cname:
+                ext_type.typespec_cname = self.mangle(Naming.typespec_prefix, name)
         ## Generate typeptr_cname
         #type.typeptr_cname = self.mangle(Naming.typeptr_prefix, name)
         # Check C methods defined
-        if type.scope:
-            for method_entry in type.scope.cfunc_entries:
+        if ext_type.scope:
+            for method_entry in ext_type.scope.cfunc_entries:
                 if not method_entry.is_inherited and not method_entry.func_cname:
                     error(method_entry.pos, "C method '%s' is declared but not defined" %
                         method_entry.name)
         # Allocate vtable name if necessary
-        if type.vtabslot_cname:
+        if ext_type.vtabslot_cname:
             #print "ModuleScope.check_c_classes: allocating vtable cname for", self ###
-            type.vtable_cname = self.mangle(Naming.vtable_prefix, entry.name)
+            ext_type.vtable_cname = self.mangle(Naming.vtable_prefix, entry.name)
 
     def check_c_classes(self):
         # Performs post-analysis checking and finishing up of extension types
diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py
index 5a84bc89e..363c5e6c5 100644
--- a/Cython/Compiler/TypeSlots.py
+++ b/Cython/Compiler/TypeSlots.py
@@ -220,6 +220,8 @@ class SlotDescriptor(object):
     #  py2                           Indicates presence of slot in Python 2
     #  ifdef                         Full #ifdef string that slot is wrapped in. Using this causes py3, py2 and flags to be ignored.)
 
+    not_in_heap_type = False  # Not a slot in heap type declarations.
+
     def __init__(self, slot_name, dynamic=False, inherited=False,
                  py3=True, py2=True, ifdef=None):
         self.slot_name = slot_name
@@ -242,7 +244,14 @@ class SlotDescriptor(object):
             guard = ("#if PY_MAJOR_VERSION >= 3")
         return guard
 
-    def generate(self, scope, code):
+    def _generate(self, code, value, as_heap_type, slot_name=None):
+        if as_heap_type:
+            if value != "0" and not self.not_in_heap_type:
+                code.putln("{Py_%s, (void*)%s}," % (slot_name or self.slot_name, value))
+        else:
+            code.putln("%s, /*%s*/" % (value, slot_name or self.slot_name))
+
+    def generate(self, scope, code, as_heap_type=False):
         preprocessor_guard = self.preprocessor_guard_code()
         if preprocessor_guard:
             code.putln(preprocessor_guard)
@@ -267,18 +276,18 @@ class SlotDescriptor(object):
                     inherited_value = self.slot_code(current_scope)
                 if inherited_value != "0":
                     code.putln("#if CYTHON_COMPILING_IN_PYPY")
-                    code.putln("%s, /*%s*/" % (inherited_value, self.slot_name))
+                    self._generate(code, inherited_value, as_heap_type)
                     code.putln("#else")
                     end_pypy_guard = True
 
-        code.putln("%s, /*%s*/" % (value, self.slot_name))
+        self._generate(code, value, as_heap_type)
 
         if end_pypy_guard:
             code.putln("#endif")
 
         if self.py3 == '<RESERVED>':
             code.putln("#else")
-            code.putln("0, /*reserved*/")
+            self._generate(code, "0", as_heap_type, slot_name="reserved")
         if preprocessor_guard:
             code.putln("#endif")
 
@@ -487,6 +496,8 @@ class RichcmpSlot(MethodSlot):
 class TypeFlagsSlot(SlotDescriptor):
     #  Descriptor for the type flags slot.
 
+    not_in_heap_type = True
+
     def slot_code(self, scope):
         value = "Py_TPFLAGS_DEFAULT"
         if scope.directives['type_version_tag']:
@@ -506,6 +517,8 @@ class TypeFlagsSlot(SlotDescriptor):
 class DocStringSlot(SlotDescriptor):
     #  Descriptor for the docstring slot.
 
+    not_in_heap_type = True
+
     def slot_code(self, scope):
         doc = scope.doc
         if doc is None:
@@ -540,20 +553,29 @@ class SuiteSlot(SlotDescriptor):
             return "&%s" % self.substructure_cname(scope)
         return "0"
 
-    def generate_substructure(self, scope, code):
-        if not self.is_empty(scope):
-            code.putln("")
-            if self.ifdef:
-                code.putln("#if %s" % self.ifdef)
+    def generate(self, scope, code, as_heap_type=False):
+        if as_heap_type:
+            self.generate_substructure(scope, code, as_heap_type)
+        else:
+            super(SuiteSlot, self).generate(scope, code, as_heap_type)
+
+    def generate_substructure(self, scope, code, as_heap_type=False):
+        if self.is_empty(scope):
+            return
+        code.putln("")
+        if self.ifdef:
+            code.putln("#if %s" % self.ifdef)
+        if not as_heap_type:
             code.putln(
                 "static %s %s = {" % (
                     self.slot_type,
                     self.substructure_cname(scope)))
-            for slot in self.sub_slots:
-                slot.generate(scope, code)
+        for slot in self.sub_slots:
+            slot.generate(scope, code, as_heap_type)
+        if not as_heap_type:
             code.putln("};")
-            if self.ifdef:
-                code.putln("#endif")
+        if self.ifdef:
+            code.putln("#endif")
 
 substructures = []   # List of all SuiteSlot instances
 
diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c
index dc12bac3d..604f68c13 100644
--- a/Cython/Utility/ModuleSetupCode.c
+++ b/Cython/Utility/ModuleSetupCode.c
@@ -85,6 +85,8 @@
   #define CYTHON_USE_DICT_VERSIONS 0
   #undef CYTHON_USE_EXC_INFO_STACK
   #define CYTHON_USE_EXC_INFO_STACK 0
+  #undef CYTHON_USE_HEAPTYPES
+  #define CYTHON_USE_HEAPTYPES 0
 
 #elif defined(PYSTON_VERSION)
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -132,6 +134,8 @@
   #define CYTHON_USE_DICT_VERSIONS 0
   #undef CYTHON_USE_EXC_INFO_STACK
   #define CYTHON_USE_EXC_INFO_STACK 0
+  #undef CYTHON_USE_HEAPTYPES
+  #define CYTHON_USE_HEAPTYPES 0
 
 #else
   #define CYTHON_COMPILING_IN_PYPY 0
@@ -201,6 +205,10 @@
   #ifndef CYTHON_USE_EXC_INFO_STACK
     #define CYTHON_USE_EXC_INFO_STACK (PY_VERSION_HEX >= 0x030700A3)
   #endif
+  #ifndef CYTHON_USE_HEAPTYPES
+    // FIXME: min CPython version?
+    #define CYTHON_USE_HEAPTYPES (0 && CYTHON_PEP489_MULTI_PHASE_INIT && PY_VERSION_HEX >= 0x03050000)
+  #endif
 #endif
 
 #if !defined(CYTHON_FAST_PYCCALL)