diff options
Diffstat (limited to 'Cython/Compiler/ModuleNode.py')
| -rw-r--r-- | Cython/Compiler/ModuleNode.py | 1390 |
1 files changed, 1078 insertions, 312 deletions
diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index a2400bbe2..34ef35880 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -14,6 +14,7 @@ import json import operator import os import re +import sys from .PyrexTypes import CPtrType from . import Future @@ -26,13 +27,25 @@ from . import TypeSlots from . import PyrexTypes from . import Pythran -from .Errors import error, warning +from .Errors import error, warning, CompileError from .PyrexTypes import py_object_type -from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version -from .Code import UtilityCode, IncludeCode -from .StringEncoding import EncodedString +from ..Utils import open_new_file, replace_suffix, decode_filename, build_hex_version, is_cython_generated_file +from .Code import UtilityCode, IncludeCode, TempitaUtilityCode +from .StringEncoding import EncodedString, encoded_string_or_bytes_literal from .Pythran import has_np_pythran + +def replace_suffix_encoded(path, newsuf): + # calls replace suffix and returns a EncodedString or BytesLiteral with the encoding set + newpath = replace_suffix(path, newsuf) + return as_encoded_filename(newpath) + +def as_encoded_filename(path): + # wraps the path with either EncodedString or BytesLiteral (depending on its input type) + # and sets the encoding to the file system encoding + return encoded_string_or_bytes_literal(path, sys.getfilesystemencoding()) + + def check_c_declarations_pxd(module_node): module_node.scope.check_c_classes_pxd() return module_node @@ -50,11 +63,50 @@ def generate_c_code_config(env, options): else: emit_linenums = options.emit_linenums + if hasattr(options, "emit_code_comments"): + print('Warning: option emit_code_comments is deprecated. ' + 'Instead, use compiler directive emit_code_comments.') + return Code.CCodeConfig( emit_linenums=emit_linenums, emit_code_comments=env.directives['emit_code_comments'], c_line_in_traceback=options.c_line_in_traceback) +# The code required to generate one comparison from another. +# The keys are (from, to). +# The comparison operator always goes first, with equality possibly second. +# The first value specifies if the comparison is inverted. The second is the +# logic op to use, and the third is if the equality is inverted or not. +TOTAL_ORDERING = { + # a > b from (not a < b) and (a != b) + ('__lt__', '__gt__'): (True, '&&', True), + # a <= b from (a < b) or (a == b) + ('__lt__', '__le__'): (False, '||', False), + # a >= b from (not a < b). + ('__lt__', '__ge__'): (True, '', None), + + # a >= b from (not a <= b) or (a == b) + ('__le__', '__ge__'): (True, '||', False), + # a < b, from (a <= b) and (a != b) + ('__le__', '__lt__'): (False, '&&', True), + # a > b from (not a <= b) + ('__le__', '__gt__'): (True, '', None), + + # a < b from (not a > b) and (a != b) + ('__gt__', '__lt__'): (True, '&&', True), + # a >= b from (a > b) or (a == b) + ('__gt__', '__ge__'): (False, '||', False), + # a <= b from (not a > b) + ('__gt__', '__le__'): (True, '', None), + + # Return a <= b from (not a >= b) or (a == b) + ('__ge__', '__le__'): (True, '||', False), + # a > b from (a >= b) and (a != b) + ('__ge__', '__gt__'): (False, '&&', True), + # a < b from (not a >= b) + ('__ge__', '__lt__'): (True, '', None), +} + class ModuleNode(Nodes.Node, Nodes.BlockNode): # doc string or None @@ -69,21 +121,41 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): child_attrs = ["body"] directives = None + # internal - used in merging + pxd_stats = None + utility_code_stats = None - def merge_in(self, tree, scope, merge_scope=False): + + def merge_in(self, tree, scope, stage, merge_scope=False): # Merges in the contents of another tree, and possibly scope. With the # current implementation below, this must be done right prior # to code generation. + # Stage is one of "pxd" or "utility" to indicate pxd file or utility + # code. This helps define the order. # # Note: This way of doing it seems strange -- I believe the # right concept is to split ModuleNode into a ModuleNode and a # CodeGenerator, and tell that CodeGenerator to generate code # from multiple sources. assert isinstance(self.body, Nodes.StatListNode) + assert stage in ('pxd', 'utility') + + if self.pxd_stats is None: + self.pxd_stats = Nodes.StatListNode(self.body.pos, stats=[]) + self.utility_code_stats = Nodes.StatListNode(self.body.pos, stats=[]) + self.body.stats.insert(0, self.pxd_stats) + self.body.stats.insert(0, self.utility_code_stats) + + if scope.directives != self.scope.directives: + # merged in nodes should keep their original compiler directives + # (for example inline cdef functions) + tree = Nodes.CompilerDirectivesNode(tree.pos, body=tree, directives=scope.directives) + + target_stats = self.pxd_stats if stage == "pxd" else self.utility_code_stats if isinstance(tree, Nodes.StatListNode): - self.body.stats.extend(tree.stats) + target_stats.stats.extend(tree.stats) else: - self.body.stats.append(tree) + target_stats.stats.append(tree) self.scope.utility_code_list.extend(scope.utility_code_list) @@ -105,6 +177,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.scope.merge_in(scope) + def with_compiler_directives(self): + # When merging a utility code module into the user code we need to preserve + # the original compiler directives. This returns the body of the module node, + # wrapped in its set of directives. + body = Nodes.CompilerDirectivesNode(self.pos, directives=self.directives, body=self.body) + return body + def analyse_declarations(self, env): if has_np_pythran(env): Pythran.include_pythran_generic(env) @@ -131,8 +210,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.create_import_star_conversion_utility_code(env) for name, entry in sorted(env.entries.items()): if (entry.create_wrapper and entry.scope is env - and entry.is_type and entry.type.is_enum): - entry.type.create_type_wrapper(env) + and entry.is_type and (entry.type.is_enum or entry.type.is_cpp_enum)): + entry.type.create_type_wrapper(env) def process_implementation(self, options, result): env = self.scope @@ -151,6 +230,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): return 1 return 0 + def assure_safe_target(self, path, allow_failed=False): + # Check for a common gotcha for new users: naming your .pyx file after the .c file you want to wrap + if not is_cython_generated_file(path, allow_failed=allow_failed, if_not_found=True): + # Raising a fatal CompileError instead of calling error() to prevent castrating an existing file. + raise CompileError( + self.pos, 'The output file already exists and does not look like it was generated by Cython: "%s"' % + os.path.basename(path)) + def generate_h_code(self, env, options, result): def h_entries(entries, api=0, pxd=0): return [entry for entry in entries @@ -161,65 +248,99 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_vars = h_entries(env.var_entries) h_funcs = h_entries(env.cfunc_entries) h_extension_types = h_entries(env.c_class_entries) - if h_types or h_vars or h_funcs or h_extension_types: - result.h_file = replace_suffix(result.c_file, ".h") - h_code = Code.CCodeWriter() + + if h_types or h_vars or h_funcs or h_extension_types: + result.h_file = replace_suffix_encoded(result.c_file, ".h") + self.assure_safe_target(result.h_file) + + h_code_writer = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) - Code.GlobalState(h_code, self, c_code_config) + globalstate = Code.GlobalState(h_code_writer, self, c_code_config) + globalstate.initialize_main_h_code() # in-case utility code is used in the header + h_code_start = globalstate.parts['h_code'] + h_code_main = globalstate.parts['type_declarations'] + h_code_end = globalstate.parts['end'] if options.generate_pxi: - result.i_file = replace_suffix(result.c_file, ".pxi") + result.i_file = replace_suffix_encoded(result.c_file, ".pxi") i_code = Code.PyrexCodeWriter(result.i_file) else: i_code = None - h_code.put_generated_by() - h_guard = Naming.h_guard_prefix + self.api_name(env) - h_code.put_h_guard(h_guard) - h_code.putln("") - h_code.putln('#include "Python.h"') - self.generate_type_header_code(h_types, h_code) + h_code_start.put_generated_by() + h_guard = self.api_name(Naming.h_guard_prefix, env) + h_code_start.put_h_guard(h_guard) + h_code_start.putln("") + h_code_start.putln('#include "Python.h"') + self.generate_type_header_code(h_types, h_code_start) if options.capi_reexport_cincludes: - self.generate_includes(env, [], h_code) - h_code.putln("") - api_guard = Naming.api_guard_prefix + self.api_name(env) - h_code.putln("#ifndef %s" % api_guard) - h_code.putln("") - self.generate_extern_c_macro_definition(h_code) - h_code.putln("") - self.generate_dl_import_macro(h_code) + self.generate_includes(env, [], h_code_start) + h_code_start.putln("") + api_guard = self.api_name(Naming.api_guard_prefix, env) + h_code_start.putln("#ifndef %s" % api_guard) + h_code_start.putln("") + self.generate_extern_c_macro_definition(h_code_start, env.is_cpp()) + h_code_start.putln("") + self.generate_dl_import_macro(h_code_start) if h_extension_types: - h_code.putln("") + h_code_main.putln("") for entry in h_extension_types: - self.generate_cclass_header_code(entry.type, h_code) + self.generate_cclass_header_code(entry.type, h_code_main) if i_code: self.generate_cclass_include_code(entry.type, i_code) if h_funcs: - h_code.putln("") + h_code_main.putln("") for entry in h_funcs: - self.generate_public_declaration(entry, h_code, i_code) + self.generate_public_declaration(entry, h_code_main, i_code) if h_vars: - h_code.putln("") + h_code_main.putln("") for entry in h_vars: - self.generate_public_declaration(entry, h_code, i_code) - h_code.putln("") - h_code.putln("#endif /* !%s */" % api_guard) - h_code.putln("") - h_code.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") - h_code.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") - h_code.putln("") - h_code.putln("#if PY_MAJOR_VERSION < 3") - h_code.putln("PyMODINIT_FUNC init%s(void);" % env.module_name) - h_code.putln("#else") - h_code.putln("PyMODINIT_FUNC %s(void);" % self.mod_init_func_cname('PyInit', env)) - h_code.putln("#endif") - h_code.putln("") - h_code.putln("#endif /* !%s */" % h_guard) - - f = open_new_file(result.h_file) - try: - h_code.copyto(f) - finally: - f.close() + self.generate_public_declaration(entry, h_code_main, i_code) + h_code_main.putln("") + h_code_main.putln("#endif /* !%s */" % api_guard) + h_code_main.putln("") + h_code_main.putln("/* WARNING: the interface of the module init function changed in CPython 3.5. */") + h_code_main.putln("/* It now returns a PyModuleDef instance instead of a PyModule instance. */") + h_code_main.putln("") + h_code_main.putln("#if PY_MAJOR_VERSION < 3") + if env.module_name.isascii(): + py2_mod_name = env.module_name + else: + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf-8") + h_code_main.putln('#error "Unicode module names are not supported in Python 2";') + h_code_main.putln("PyMODINIT_FUNC init%s(void);" % py2_mod_name) + h_code_main.putln("#else") + py3_mod_func_name = self.mod_init_func_cname('PyInit', env) + warning_string = EncodedString('Use PyImport_AppendInittab("%s", %s) instead of calling %s directly.' % ( + py2_mod_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('/* WARNING: %s from Python 3.5 */' % warning_string.rstrip('.')) + h_code_main.putln("PyMODINIT_FUNC %s(void);" % py3_mod_func_name) + h_code_main.putln("") + h_code_main.putln("#if PY_VERSION_HEX >= 0x03050000 " + "&& (defined(__GNUC__) || defined(__clang__) || defined(_MSC_VER) " + "|| (defined(__cplusplus) && __cplusplus >= 201402L))") + h_code_main.putln("#if defined(__cplusplus) && __cplusplus >= 201402L") + h_code_main.putln("[[deprecated(%s)]] inline" % warning_string.as_c_string_literal()) + h_code_main.putln("#elif defined(__GNUC__) || defined(__clang__)") + h_code_main.putln('__attribute__ ((__deprecated__(%s), __unused__)) __inline__' % ( + warning_string.as_c_string_literal())) + h_code_main.putln("#elif defined(_MSC_VER)") + h_code_main.putln('__declspec(deprecated(%s)) __inline' % ( + warning_string.as_c_string_literal())) + h_code_main.putln('#endif') + h_code_main.putln("static PyObject* __PYX_WARN_IF_%s_INIT_CALLED(PyObject* res) {" % py3_mod_func_name) + h_code_main.putln("return res;") + h_code_main.putln("}") + # Function call is converted to warning macro; uncalled (pointer) is not + h_code_main.putln('#define %s() __PYX_WARN_IF_%s_INIT_CALLED(%s())' % ( + py3_mod_func_name, py3_mod_func_name, py3_mod_func_name)) + h_code_main.putln('#endif') + h_code_main.putln('#endif') + + h_code_end.putln("") + h_code_end.putln("#endif /* !%s */" % h_guard) + + with open_new_file(result.h_file) as f: + h_code_writer.copyto(f) def generate_public_declaration(self, entry, h_code, i_code): h_code.putln("%s %s;" % ( @@ -229,8 +350,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.putln("cdef extern %s" % ( entry.type.declaration_code(entry.cname, pyrex=1))) - def api_name(self, env): - return env.qualified_name.replace(".", "__") + def api_name(self, prefix, env): + api_name = self.punycode_module_name(prefix, env.qualified_name) + return api_name.replace(".", "__") def generate_api_code(self, env, options, result): def api_entries(entries, pxd=0): @@ -239,13 +361,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): api_vars = api_entries(env.var_entries) api_funcs = api_entries(env.cfunc_entries) api_extension_types = api_entries(env.c_class_entries) + if api_vars or api_funcs or api_extension_types: - result.api_file = replace_suffix(result.c_file, "_api.h") + result.api_file = replace_suffix_encoded(result.c_file, "_api.h") + self.assure_safe_target(result.api_file) + h_code = Code.CCodeWriter() c_code_config = generate_c_code_config(env, options) Code.GlobalState(h_code, self, c_code_config) h_code.put_generated_by() - api_guard = Naming.api_guard_prefix + self.api_name(env) + api_guard = self.api_name(Naming.api_guard_prefix, env) h_code.put_h_guard(api_guard) # Work around https://bugs.python.org/issue4709 h_code.putln('#ifdef __MINGW64__') @@ -254,7 +379,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.putln('#include "Python.h"') if result.h_file: - h_code.putln('#include "%s"' % os.path.basename(result.h_file)) + h_filename = os.path.basename(result.h_file) + h_filename = as_encoded_filename(h_filename) + h_code.putln('#include %s' % h_filename.as_c_string_literal()) if api_extension_types: h_code.putln("") for entry in api_extension_types: @@ -274,9 +401,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for entry in api_vars: type = CPtrType(entry.type) cname = env.mangle(Naming.varptr_prefix_api, entry.name) - h_code.putln("static %s = 0;" % type.declaration_code(cname)) + h_code.putln("static %s = 0;" % type.declaration_code(cname)) h_code.putln("#define %s (*%s)" % (entry.name, cname)) - h_code.put(UtilityCode.load_as_string("PyIdentifierFromString", "ImportExport.c")[0]) if api_vars: h_code.put(UtilityCode.load_as_string("VoidPtrImport", "ImportExport.c")[1]) if api_funcs: @@ -285,22 +411,22 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[0]) h_code.put(UtilityCode.load_as_string("TypeImport", "ImportExport.c")[1]) h_code.putln("") - h_code.putln("static int import_%s(void) {" % self.api_name(env)) + h_code.putln("static int %s(void) {" % self.api_name("import", env)) h_code.putln("PyObject *module = 0;") - h_code.putln('module = PyImport_ImportModule("%s");' % env.qualified_name) + h_code.putln('module = PyImport_ImportModule(%s);' % env.qualified_name.as_c_string_literal()) h_code.putln("if (!module) goto bad;") for entry in api_funcs: cname = env.mangle(Naming.func_prefix_api, entry.name) sig = entry.type.signature_string() h_code.putln( - 'if (__Pyx_ImportFunction_%s(module, "%s", (void (**)(void))&%s, "%s") < 0) goto bad;' - % (Naming.cyversion, entry.name, cname, sig)) + 'if (__Pyx_ImportFunction_%s(module, %s, (void (**)(void))&%s, "%s") < 0) goto bad;' + % (Naming.cyversion, entry.name.as_c_string_literal(), cname, sig)) for entry in api_vars: cname = env.mangle(Naming.varptr_prefix_api, entry.name) sig = entry.type.empty_declaration_code() h_code.putln( - 'if (__Pyx_ImportVoidPtr_%s(module, "%s", (void **)&%s, "%s") < 0) goto bad;' - % (Naming.cyversion, entry.name, cname, sig)) + 'if (__Pyx_ImportVoidPtr_%s(module, %s, (void **)&%s, "%s") < 0) goto bad;' + % (Naming.cyversion, entry.name.as_c_string_literal(), cname, sig)) with ModuleImportGenerator(h_code, imported_modules={env.qualified_name: 'module'}) as import_generator: for entry in api_extension_types: self.generate_type_import_call(entry.type, h_code, import_generator, error_code="goto bad;") @@ -339,10 +465,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): i_code.dedent() def generate_c_code(self, env, options, result): + self.assure_safe_target(result.c_file, allow_failed=True) modules = self.referenced_modules if Options.annotate or options.annotate: - rootwriter = Annotate.AnnotationCCodeWriter() + show_entire_c_code = Options.annotate == "fullc" or options.annotate == "fullc" + rootwriter = Annotate.AnnotationCCodeWriter( + show_entire_c_code=show_entire_c_code, + source_desc=self.compilation_source.source_desc, + ) else: rootwriter = Code.CCodeWriter() @@ -364,24 +495,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(refnanny_utility_code) code = globalstate['before_global_var'] - code.putln('#define __Pyx_MODULE_NAME "%s"' % self.full_module_name) - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + code.putln('#define __Pyx_MODULE_NAME %s' % + self.full_module_name.as_c_string_literal()) + module_is_main = self.is_main_module_flag_cname() code.putln("extern int %s;" % module_is_main) code.putln("int %s = 0;" % module_is_main) code.putln("") - code.putln("/* Implementation of '%s' */" % env.qualified_name) + code.putln("/* Implementation of %s */" % env.qualified_name.as_c_string_literal()) code = globalstate['late_includes'] - code.putln("/* Late includes */") self.generate_includes(env, modules, code, early=False) - code = globalstate['all_the_rest'] + code = globalstate['module_code'] self.generate_cached_builtins_decls(env, code) - self.generate_lambda_definitions(env, code) + # generate normal variable and function definitions + self.generate_lambda_definitions(env, code) self.generate_variable_definitions(env, code) - self.body.generate_function_definitions(env, code) code.mark_pos(None) @@ -389,11 +520,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_method_table(env, code) if env.has_import_star: self.generate_import_star(env, code) - self.generate_pymoduledef_struct(env, code) # initialise the macro to reduce the code size of one-time functionality code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip()) + self.generate_module_state_start(env, globalstate['module_state']) + self.generate_module_state_defines(env, globalstate['module_state_defines']) + self.generate_module_state_clear(env, globalstate['module_state_clear']) + self.generate_module_state_traverse(env, globalstate['module_state_traverse']) + # init_globals is inserted before this self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) @@ -408,6 +543,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() + self.generate_module_state_end(env, modules, globalstate) + f = open_new_file(result.c_file) try: rootwriter.copyto(f) @@ -429,11 +566,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): except ImportError: import xml.etree.ElementTree as ET coverage_xml = ET.parse(coverage_xml_filename).getroot() - if hasattr(coverage_xml, 'iter'): - iterator = coverage_xml.iter() # Python 2.7 & 3.2+ - else: - iterator = coverage_xml.getiterator() - for el in iterator: + for el in coverage_xml.iter(): el.tail = None # save some memory else: coverage_xml = None @@ -452,7 +585,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not target_file_dir.startswith(target_dir): # any other directories may not be writable => avoid trying continue - source_file = search_include_file(included_file, "", self.pos, include=True) + source_file = search_include_file(included_file, source_pos=self.pos, include=True) if not source_file: continue if target_file_dir != target_dir and not os.path.exists(target_file_dir): @@ -469,16 +602,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): markers = ccodewriter.buffer.allmarkers() d = defaultdict(list) - for c_lineno, cython_lineno in enumerate(markers): - if cython_lineno > 0: - d[cython_lineno].append(c_lineno + 1) + for c_lineno, (src_desc, src_lineno) in enumerate(markers): + if src_lineno > 0 and src_desc.filename is not None: + d[src_desc, src_lineno].append(c_lineno + 1) tb.start('LineNumberMapping') - for cython_lineno, c_linenos in sorted(d.items()): + for (src_desc, src_lineno), c_linenos in sorted(d.items()): + assert src_desc.filename is not None tb.add_entry( 'LineNumber', c_linenos=' '.join(map(str, c_linenos)), - cython_lineno=str(cython_lineno), + src_path=src_desc.filename, + src_lineno=str(src_lineno), ) tb.end('LineNumberMapping') tb.serialize() @@ -491,33 +626,36 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module_list.append(env) def sort_types_by_inheritance(self, type_dict, type_order, getkey): - # copy the types into a list moving each parent type before - # its first child - type_list = [] - for i, key in enumerate(type_order): + subclasses = defaultdict(list) # maps type key to list of subclass keys + for key in type_order: new_entry = type_dict[key] - # collect all base classes to check for children - hierarchy = set() - base = new_entry + base = new_entry.type.base_type while base: - base_type = base.type.base_type - if not base_type: - break - base_key = getkey(base_type) - hierarchy.add(base_key) - base = type_dict.get(base_key) - new_entry.base_keys = hierarchy - - # find the first (sub-)subclass and insert before that - for j in range(i): - entry = type_list[j] - if key in entry.base_keys: - type_list.insert(j, new_entry) + base_key = getkey(base) + subclasses[base_key].append(key) + base_entry = type_dict.get(base_key) + if base_entry is None: break - else: - type_list.append(new_entry) - return type_list + base = base_entry.type.base_type + + # Simple topological sort using recursive DFS, based on + # https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search + seen = set() + result = [] + def dfs(u): + if u in seen: + return + seen.add(u) + for v in subclasses[getkey(u.type)]: + dfs(type_dict[v]) + result.append(u) + + for key in reversed(type_order): + dfs(type_dict[key]) + + result.reverse() + return result def sort_type_hierarchy(self, module_list, env): # poor developer's OrderedDict @@ -619,12 +757,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for module in modules: defined_here = module is env modulecode.putln("") - modulecode.putln("/* Module declarations from '%s' */" % module.qualified_name) - self.generate_c_class_declarations(module, modulecode, defined_here) + modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal()) + self.generate_c_class_declarations(module, modulecode, defined_here, globalstate) self.generate_cvariable_declarations(module, modulecode, defined_here) self.generate_cfunction_declarations(module, modulecode, defined_here) - def _put_setup_code(self, code, name): + @staticmethod + def _put_setup_code(code, name): code.put(UtilityCode.load_as_string(name, "ModuleSetupCode.c")[1]) def generate_module_preamble(self, env, options, cimported_modules, metadata, code): @@ -638,6 +777,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef PY_SSIZE_T_CLEAN") code.putln("#define PY_SSIZE_T_CLEAN") code.putln("#endif /* PY_SSIZE_T_CLEAN */") + self._put_setup_code(code, "InitLimitedAPI") for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.INITIAL: @@ -645,14 +785,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#ifndef Py_PYTHON_H") code.putln(" #error Python headers needed to compile C extensions, " "please install development version of Python.") - code.putln("#elif PY_VERSION_HEX < 0x02060000 || " + code.putln("#elif PY_VERSION_HEX < 0x02070000 || " "(0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03030000)") - code.putln(" #error Cython requires Python 2.6+ or Python 3.3+.") + code.putln(" #error Cython requires Python 2.7+ or Python 3.3+.") code.putln("#else") code.globalstate["end"].putln("#endif /* Py_PYTHON_H */") from .. import __version__ code.putln('#define CYTHON_ABI "%s"' % __version__.replace('.', '_')) + code.putln('#define __PYX_ABI_MODULE_NAME "_cython_" CYTHON_ABI') + code.putln('#define __PYX_TYPE_MODULE_PREFIX __PYX_ABI_MODULE_NAME "."') code.putln('#define CYTHON_HEX_VERSION %s' % build_hex_version(__version__)) code.putln("#define CYTHON_FUTURE_DIVISION %d" % ( Future.division in env.context.future_directives)) @@ -680,11 +822,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(" { __PYX_MARK_ERR_POS(f_index, lineno) goto Ln_error; }") code.putln("") - self.generate_extern_c_macro_definition(code) + self.generate_extern_c_macro_definition(code, env.is_cpp()) code.putln("") - code.putln("#define %s" % Naming.h_guard_prefix + self.api_name(env)) - code.putln("#define %s" % Naming.api_guard_prefix + self.api_name(env)) + code.putln("#define %s" % self.api_name(Naming.h_guard_prefix, env)) + code.putln("#define %s" % self.api_name(Naming.api_guard_prefix, env)) code.putln("/* Early includes */") self.generate_includes(env, cimported_modules, code, late=False) code.putln("") @@ -721,6 +863,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('#define __Pyx_PyObject_FromString __Pyx_Py%s_FromString' % c_string_func_name) code.putln('#define __Pyx_PyObject_FromStringAndSize __Pyx_Py%s_FromStringAndSize' % c_string_func_name) code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[0]) + env.use_utility_code(UtilityCode.load_cached("FormatTypeName", "ObjectHandling.c")) # These utility functions are assumed to exist and used elsewhere. PyrexTypes.c_long_type.create_to_py_utility_code(env) @@ -730,32 +873,42 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put(Nodes.branch_prediction_macros) code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }') code.putln('') + code.putln('#if !CYTHON_USE_MODULE_STATE') code.putln('static PyObject *%s = NULL;' % env.module_cname) - code.putln('static PyObject *%s;' % env.module_dict_cname) - code.putln('static PyObject *%s;' % Naming.builtins_cname) - code.putln('static PyObject *%s = NULL;' % Naming.cython_runtime_cname) - code.putln('static PyObject *%s;' % Naming.empty_tuple) - code.putln('static PyObject *%s;' % Naming.empty_bytes) - code.putln('static PyObject *%s;' % Naming.empty_unicode) if Options.pre_import is not None: code.putln('static PyObject *%s;' % Naming.preimport_cname) + code.putln('#endif') + code.putln('static int %s;' % Naming.lineno_cname) code.putln('static int %s = 0;' % Naming.clineno_cname) - code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) + code.putln('static const char * %s = %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) code.putln('static const char *%s;' % Naming.filename_cname) env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) if has_np_pythran(env): env.use_utility_code(UtilityCode.load_cached("PythranConversion", "CppSupport.cpp")) - def generate_extern_c_macro_definition(self, code): + def generate_extern_c_macro_definition(self, code, is_cpp): name = Naming.extern_c_macro - code.putln("#ifndef %s" % name) - code.putln(" #ifdef __cplusplus") - code.putln(' #define %s extern "C"' % name) - code.putln(" #else") - code.putln(" #define %s extern" % name) - code.putln(" #endif") + code.putln("#ifdef CYTHON_EXTERN_C") + # make sure that user overrides always take precedence + code.putln(' #undef %s' % name) + code.putln(' #define %s CYTHON_EXTERN_C' % name) + code.putln("#elif defined(%s)" % name) + code.putln(" #ifdef _MSC_VER") + code.putln(" #pragma message (\"Please do not define the '%s' macro externally. Use 'CYTHON_EXTERN_C' instead.\")" % name) + code.putln(" #else") + code.putln(" #warning Please do not define the '%s' macro externally. Use 'CYTHON_EXTERN_C' instead." % name) + code.putln(" #endif") + code.putln("#else") + if is_cpp: + code.putln(' #define %s extern "C++"' % name) + else: + code.putln(" #ifdef __cplusplus") + code.putln(' #define %s extern "C"' % name) + code.putln(" #else") + code.putln(" #define %s extern" % name) + code.putln(" #endif") code.putln("#endif") def generate_dl_import_macro(self, code): @@ -764,7 +917,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") def generate_includes(self, env, cimported_modules, code, early=True, late=True): - includes = [] for inc in sorted(env.c_includes.values(), key=IncludeCode.sortkey): if inc.location == inc.EARLY: if early: @@ -785,7 +937,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if isabs(file_path): file_path = basename(file_path) # never include absolute paths escaped_filename = file_path.replace("\\", "\\\\").replace('"', r'\"') - code.putln('"%s",' % escaped_filename) + escaped_filename = as_encoded_filename(escaped_filename) + code.putln('%s,' % escaped_filename.as_c_string_literal()) else: # Some C compilers don't like an empty array code.putln("0") @@ -802,7 +955,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! pass elif type.is_struct_or_union or type.is_cpp_class: self.generate_struct_union_predeclaration(entry, code) @@ -815,9 +968,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if not entry.in_cinclude: #print "generate_type_header_code:", entry.name, repr(entry.type) ### type = entry.type - if type.is_typedef: # Must test this first! + if type.is_typedef: # Must test this first! self.generate_typedef(entry, code) - elif type.is_enum: + elif type.is_enum or type.is_cpp_enum: self.generate_enum_definition(entry, code) elif type.is_struct_or_union: self.generate_struct_union_definition(entry, code) @@ -844,7 +997,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_typedef(self, entry, code): base_type = entry.type.typedef_base_type - if base_type.is_numeric: + enclosing_scope = entry.scope + if base_type.is_numeric and not enclosing_scope.is_cpp_class_scope: try: writer = code.globalstate['numeric_typedefs'] except KeyError: @@ -894,8 +1048,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln(header) var_entries = scope.var_entries - if not var_entries: - error(entry.pos, "Empty struct or union definition not allowed outside a 'cdef extern from' block") for attr in var_entries: code.putln( "%s;" % attr.type.declaration_code(attr.cname)) @@ -922,6 +1074,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): [base_class.empty_declaration_code() for base_class in type.base_classes]) code.put(" : public %s" % base_class_decl) code.putln(" {") + self.generate_type_header_code(scope.type_entries, code) py_attrs = [e for e in scope.entries.values() if e.type.is_pyobject and not e.is_inherited] has_virtual_methods = False @@ -956,67 +1109,69 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): arg_decls = ["void"] arg_names = [] if is_implementing: - code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) - if py_attrs: - code.put_ensure_gil() - for attr in py_attrs: - code.put_init_var_to_py_none(attr, nanny=False); - if constructor: - code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) - if py_attrs: - code.put_release_ensured_gil() - code.putln("}") + code.putln("%s(%s) {" % (type.cname, ", ".join(arg_decls))) + if py_attrs: + code.put_ensure_gil() + for attr in py_attrs: + code.put_init_var_to_py_none(attr, nanny=False) + if constructor: + code.putln("%s(%s);" % (constructor.cname, ", ".join(arg_names))) + if py_attrs: + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) + code.putln("%s(%s);" % (type.cname, ", ".join(arg_decls))) if destructor or py_attrs or has_virtual_methods: if has_virtual_methods: code.put("virtual ") if is_implementing: - code.putln("~%s() {" % type.cname) - if py_attrs: - code.put_ensure_gil() - if destructor: - code.putln("%s();" % destructor.cname) - if py_attrs: - for attr in py_attrs: - code.put_var_xdecref(attr, nanny=False); - code.put_release_ensured_gil() - code.putln("}") + code.putln("~%s() {" % type.cname) + if py_attrs: + code.put_ensure_gil() + if destructor: + code.putln("%s();" % destructor.cname) + if py_attrs: + for attr in py_attrs: + code.put_var_xdecref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") else: - code.putln("~%s();" % type.cname) + code.putln("~%s();" % type.cname) if py_attrs: # Also need copy constructor and assignment operators. if is_implementing: - code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) - code.putln("if (this != &__Pyx_other) {") - code.put_ensure_gil() - for attr in scope.var_entries: - if not attr.type.is_cfunction: - code.put_var_xdecref(attr, nanny=False); - code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) - code.put_var_incref(attr, nanny=False) - code.put_release_ensured_gil() - code.putln("}") - code.putln("return *this;") - code.putln("}") + code.putln("%s(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("%s& operator=(const %s& __Pyx_other) {" % (type.cname, type.cname)) + code.putln("if (this != &__Pyx_other) {") + code.put_ensure_gil() + for attr in scope.var_entries: + if not attr.type.is_cfunction: + code.put_var_xdecref(attr, nanny=False) + code.putln("%s = __Pyx_other.%s;" % (attr.cname, attr.cname)) + code.put_var_incref(attr, nanny=False) + code.put_release_ensured_gil() + code.putln("}") + code.putln("return *this;") + code.putln("}") else: - code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) - code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s(const %s& __Pyx_other);" % (type.cname, type.cname)) + code.putln("%s& operator=(const %s& __Pyx_other);" % (type.cname, type.cname)) code.putln("};") def generate_enum_definition(self, entry, code): code.mark_pos(entry.pos) type = entry.type name = entry.cname or entry.name or "" - header, footer = self.sue_header_footer(type, "enum", name) + + kind = "enum class" if entry.type.is_cpp_enum else "enum" + header, footer = self.sue_header_footer(type, kind, name) code.putln(header) enum_values = entry.enum_values if not enum_values: @@ -1030,18 +1185,20 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for value_entry in enum_values: if value_entry.value_node is None: - value_code = value_entry.cname + value_code = value_entry.cname.split("::")[-1] else: value_code = ("%s = %s" % ( - value_entry.cname, + value_entry.cname.split("::")[-1], value_entry.value_node.result())) if value_entry is not last_entry: value_code += "," code.putln(value_code) code.putln(footer) - if entry.type.typedef_flag: - # Not pre-declared. - code.putln("typedef enum %s %s;" % (name, name)) + + if entry.type.is_enum: + if entry.type.typedef_flag: + # Not pre-declared. + code.putln("typedef enum %s %s;" % (name, name)) def generate_typeobj_predeclaration(self, entry, code): code.putln("") @@ -1120,7 +1277,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Generate object struct definition for an # extension type. if not type.scope: - return # Forward declared but never defined + return # Forward declared but never defined header, footer = \ self.sue_header_footer(type, "struct", type.objstruct_cname) code.putln(header) @@ -1148,18 +1305,53 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): attr_type = py_object_type else: attr_type = attr.type - code.putln( - "%s;" % attr_type.declaration_code(attr.cname)) + if attr.is_cpp_optional: + decl = attr_type.cpp_optional_declaration_code(attr.cname) + else: + decl = attr_type.declaration_code(attr.cname) + type.scope.use_entry_utility_code(attr) + code.putln("%s;" % decl) code.putln(footer) if type.objtypedef_cname is not None: # Only for exposing public typedef name. code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname)) - def generate_c_class_declarations(self, env, code, definition): + def generate_c_class_declarations(self, env, code, definition, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state_typeobj = module_state.insertion_point() + module_state_defines_typeobj = module_state_defines.insertion_point() + for writer in [module_state_typeobj, module_state_defines_typeobj]: + writer.putln("#if CYTHON_USE_MODULE_STATE") for entry in env.c_class_entries: if definition or entry.defined_in_pxd: - code.putln("static PyTypeObject *%s = 0;" % ( + module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeptr_cname, + Naming.modulestateglobal_cname, entry.type.typeptr_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % + entry.type.typeptr_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeptr_cname) + if entry.type.typeobj_cname is not None: + module_state_typeobj.putln("PyObject *%s;" % entry.type.typeobj_cname) + module_state_defines_typeobj.putln("#define %s %s->%s" % ( + entry.type.typeobj_cname, + Naming.modulestateglobal_cname, + entry.type.typeobj_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % ( + entry.type.typeobj_cname)) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % ( + entry.type.typeobj_cname)) + for writer in [module_state_typeobj, module_state_defines_typeobj]: + writer.putln("#endif") def generate_cvariable_declarations(self, env, code, definition): if env.is_cython_builtin: @@ -1199,17 +1391,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if storage_class: code.put("%s " % storage_class) - code.put(type.declaration_code( - cname, dll_linkage=dll_linkage)) + if entry.is_cpp_optional: + code.put(type.cpp_optional_declaration_code( + cname, dll_linkage=dll_linkage)) + else: + code.put(type.declaration_code( + cname, dll_linkage=dll_linkage)) if init is not None: code.put_safe(" = %s" % init) code.putln(";") if entry.cname != cname: code.putln("#define %s (*%s)" % (entry.cname, cname)) + env.use_entry_utility_code(entry) def generate_cfunction_declarations(self, env, code, definition): for entry in env.cfunc_entries: - if entry.used or (entry.visibility == 'public' or entry.api): + from_pyx = Options.cimport_from_pyx and not entry.visibility == 'extern' + if (entry.used + or entry.visibility == 'public' + or entry.api + or from_pyx): generate_cfunction_declaration(entry, env, code, definition) def generate_variable_definitions(self, env, code): @@ -1229,14 +1430,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if entry.visibility != 'extern': type = entry.type scope = type.scope - if scope: # could be None if there was an error - if not scope.directives['c_api_binop_methods']: - error(self.pos, - "The 'c_api_binop_methods' directive is only supported for forward compatibility" - " and must be True.") + if scope: # could be None if there was an error self.generate_exttype_vtable(scope, code) self.generate_new_function(scope, code, entry) + self.generate_del_function(scope, code) self.generate_dealloc_function(scope, code) + if scope.needs_gc(): self.generate_traverse_function(scope, code, entry) if scope.needs_tp_clear(): @@ -1264,12 +1463,26 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_descr_set_function(scope, code) if not scope.is_closure_class_scope and scope.defines_any(["__dict__"]): self.generate_dict_getter_function(scope, code) + if scope.defines_any_special(TypeSlots.richcmp_special_methods): self.generate_richcmp_function(scope, code) + elif 'total_ordering' in scope.directives: + # Warn if this is used when it can't have any effect. + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + + for slot in TypeSlots.get_slot_table(code.globalstate.directives).PyNumberMethods: + if slot.is_binop and scope.defines_any_special(slot.user_methods): + self.generate_binop_function(scope, slot, code, entry.pos) + self.generate_property_accessors(scope, code) self.generate_method_table(scope, code) self.generate_getset_table(scope, code) + code.putln("#if CYTHON_USE_TYPE_SPECS") + self.generate_typeobj_spec(entry, code) + code.putln("#else") self.generate_typeobj_definition(full_module_name, entry, code) + code.putln("#endif") def generate_exttype_vtable(self, scope, code): # Generate the definition of an extension type's vtable. @@ -1287,8 +1500,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.empty_declaration_code())) def generate_new_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__') + tp_slot = TypeSlots.ConstructorSlot("tp_new", "__cinit__") slot_func = scope.mangle_internal("tp_new") + if tp_slot.slot_code(scope) != slot_func: + return # never used + type = scope.parent_type base_type = type.base_type @@ -1298,12 +1514,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if scope.is_internal: # internal classes (should) never need None inits, normal zeroing will do py_attrs = [] - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_constructable_attrs = [entry for entry in scope.var_entries if entry.type.needs_cpp_construction] + + cinit_func_entry = scope.lookup_here("__cinit__") + if cinit_func_entry and not cinit_func_entry.is_special: + cinit_func_entry = None - new_func_entry = scope.lookup_here("__new__") - if base_type or (new_func_entry and new_func_entry.is_special - and not new_func_entry.trivial_signature): + if base_type or (cinit_func_entry and not cinit_func_entry.trivial_signature): unused_marker = '' else: unused_marker = 'CYTHON_UNUSED ' @@ -1331,26 +1548,30 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): need_self_cast = (type.vtabslot_cname or (py_buffers or memoryview_slices or py_attrs) or - cpp_class_attrs) + cpp_constructable_attrs) if need_self_cast: code.putln("%s;" % scope.parent_type.declaration_code("p")) if base_type: tp_new = TypeSlots.get_base_slot_function(scope, tp_slot) if tp_new is None: - tp_new = "%s->tp_new" % base_type.typeptr_cname + tp_new = "__Pyx_PyType_GetSlot(%s, tp_new, newfunc)" % base_type.typeptr_cname code.putln("PyObject *o = %s(t, a, k);" % tp_new) else: code.putln("PyObject *o;") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);") + code.putln("o = alloc_func(t, 0);") + code.putln("#else") if freelist_size: code.globalstate.use_utility_code( UtilityCode.load_cached("IncludeStringH", "StringTools.c")) if is_final_type: type_safety_check = '' else: - type_safety_check = ' & ((t->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)' + type_safety_check = ' & (int)(!__Pyx_PyType_HasFeature(t, (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))' obj_struct = type.declaration_code("", deref=True) code.putln( - "if (CYTHON_COMPILING_IN_CPYTHON && likely((%s > 0) & (t->tp_basicsize == sizeof(%s))%s)) {" % ( + "if (CYTHON_COMPILING_IN_CPYTHON && likely((int)(%s > 0) & (int)(t->tp_basicsize == sizeof(%s))%s)) {" % ( freecount_name, obj_struct, type_safety_check)) code.putln("o = (PyObject*)%s[--%s];" % ( freelist_name, freecount_name)) @@ -1360,7 +1581,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyObject_GC_Track(o);") code.putln("} else {") if not is_final_type: - code.putln("if (likely((t->tp_flags & Py_TPFLAGS_IS_ABSTRACT) == 0)) {") + code.putln("if (likely(!__Pyx_PyType_HasFeature(t, Py_TPFLAGS_IS_ABSTRACT))) {") code.putln("o = (*t->tp_alloc)(t, 0);") if not is_final_type: code.putln("} else {") @@ -1369,6 +1590,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("if (unlikely(!o)) return 0;") if freelist_size and not base_type: code.putln('}') + if not base_type: + code.putln("#endif") if need_self_cast: code.putln("p = %s;" % type.cast_code("o")) #if need_self_cast: @@ -1389,9 +1612,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type.vtabslot_cname, struct_type_cast, type.vtabptr_cname)) - for entry in cpp_class_attrs: + for entry in cpp_constructable_attrs: + if entry.is_cpp_optional: + decl_code = entry.type.cpp_optional_declaration_code("") + else: + decl_code = entry.type.empty_declaration_code() code.putln("new((void*)&(p->%s)) %s();" % ( - entry.cname, entry.type.empty_declaration_code())) + entry.cname, decl_code)) for entry in py_attrs: if entry.name == "__dict__": @@ -1411,14 +1638,14 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if cclass_entry.cname == '__pyx_memoryviewslice': code.putln("p->from_slice.memview = NULL;") - if new_func_entry and new_func_entry.is_special: - if new_func_entry.trivial_signature: + if cinit_func_entry: + if cinit_func_entry.trivial_signature: cinit_args = "o, %s, NULL" % Naming.empty_tuple else: cinit_args = "o, a, k" needs_error_cleanup = True code.putln("if (unlikely(%s(%s) < 0)) goto bad;" % ( - new_func_entry.func_cname, cinit_args)) + cinit_func_entry.func_cname, cinit_args)) code.putln( "return o;") @@ -1429,6 +1656,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_del_function(self, scope, code): + tp_slot = TypeSlots.get_slot_by_name("tp_finalize", scope.directives) + slot_func_cname = scope.mangle_internal("tp_finalize") + if tp_slot.slot_code(scope) != slot_func_cname: + return # never used + + entry = scope.lookup_here("__del__") + if entry is None or not entry.is_special: + return # nothing to wrap + code.putln("") + + if tp_slot.used_ifdef: + code.putln("#if %s" % tp_slot.used_ifdef) + code.putln("static void %s(PyObject *o) {" % slot_func_cname) + code.putln("PyObject *etype, *eval, *etb;") + code.putln("PyErr_Fetch(&etype, &eval, &etb);") + code.putln("%s(o);" % entry.func_cname) + code.putln("PyErr_Restore(etype, eval, etb);") + code.putln("}") + if tp_slot.used_ifdef: + code.putln("#endif") + def generate_dealloc_function(self, scope, code): tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__') slot_func = scope.mangle_internal("tp_dealloc") @@ -1443,6 +1692,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): is_final_type = scope.parent_type.is_final_type needs_gc = scope.needs_gc() + needs_trashcan = scope.needs_trashcan() weakref_slot = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_slot not in scope.var_entries: @@ -1453,13 +1703,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): dict_slot = None _, (py_attrs, _, memoryview_slices) = scope.get_refcounted_entries() - cpp_class_attrs = [entry for entry in scope.var_entries - if entry.type.is_cpp_class] + cpp_destructable_attrs = [entry for entry in scope.var_entries + if entry.type.needs_cpp_construction] - if py_attrs or cpp_class_attrs or memoryview_slices or weakref_slot or dict_slot: + if py_attrs or cpp_destructable_attrs or memoryview_slices or weakref_slot or dict_slot: self.generate_self_cast(scope, code) - if not is_final_type: + if not is_final_type or scope.may_have_finalize(): # in Py3.4+, call tp_finalize() as early as possible code.putln("#if CYTHON_USE_TP_FINALIZE") if needs_gc: @@ -1468,11 +1718,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): finalised_check = ( '(!PyType_IS_GC(Py_TYPE(o)) || !_PyGC_FINALIZED(o))') code.putln( - "if (unlikely(PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE)" - " && Py_TYPE(o)->tp_finalize) && %s) {" % finalised_check) + "if (unlikely(" + "(PY_VERSION_HEX >= 0x03080000 || __Pyx_PyType_HasFeature(Py_TYPE(o), Py_TPFLAGS_HAVE_FINALIZE))" + " && __Pyx_PyObject_GetSlot(o, tp_finalize, destructor)) && %s) {" % finalised_check) + + code.putln("if (__Pyx_PyObject_GetSlot(o, tp_dealloc, destructor) == %s) {" % slot_func_cname) # if instance was resurrected by finaliser, return code.putln("if (PyObject_CallFinalizerFromDealloc(o)) return;") code.putln("}") + code.putln("}") code.putln("#endif") if needs_gc: @@ -1481,33 +1735,40 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # running this destructor. code.putln("PyObject_GC_UnTrack(o);") - # call the user's __dealloc__ - self.generate_usr_dealloc_call(scope, code) + if needs_trashcan: + code.globalstate.use_utility_code( + UtilityCode.load_cached("PyTrashcan", "ExtensionTypes.c")) + code.putln("__Pyx_TRASHCAN_BEGIN(o, %s)" % slot_func_cname) if weakref_slot: + # We must clean the weakreferences before calling the user's __dealloc__ + # because if the __dealloc__ releases the GIL, a weakref can be + # dereferenced accessing the object in an inconsistent state or + # resurrecting it. code.putln("if (p->__weakref__) PyObject_ClearWeakRefs(o);") + # call the user's __dealloc__ + self.generate_usr_dealloc_call(scope, code) + if dict_slot: code.putln("if (p->__dict__) PyDict_Clear(p->__dict__);") - for entry in cpp_class_attrs: + for entry in cpp_destructable_attrs: code.putln("__Pyx_call_destructor(p->%s);" % entry.cname) - for entry in py_attrs: + for entry in (py_attrs + memoryview_slices): code.put_xdecref_clear("p->%s" % entry.cname, entry.type, nanny=False, - clear_before_decref=True) - - for entry in memoryview_slices: - code.put_xdecref_memoryviewslice("p->%s" % entry.cname, - have_gil=True) + clear_before_decref=True, have_gil=True) if base_type: base_cname = base_type.typeptr_cname if needs_gc: # The base class deallocator probably expects this to be tracked, # so undo the untracking above. - if base_type.scope and base_type.scope.needs_gc(): - code.putln("PyObject_GC_Track(o);") + if base_type.scope: + # Assume that we know whether the base class uses GC or not. + if base_type.scope.needs_gc(): + code.putln("PyObject_GC_Track(o);") else: code.putln("if (PyType_IS_GC(%s)) PyObject_GC_Track(o);" % base_cname) @@ -1515,13 +1776,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if tp_dealloc is not None: code.putln("%s(o);" % tp_dealloc) elif base_type.is_builtin_type: - code.putln("%s->tp_dealloc(o);" % base_cname) + code.putln("__Pyx_PyType_GetSlot(%s, tp_dealloc, destructor)(o);" % base_cname) else: # This is an externally defined type. Calling through the # cimported base type pointer directly interacts badly with # the module cleanup, which may already have cleared it. # In that case, fall back to traversing the type hierarchy. - code.putln("if (likely(%s)) %s->tp_dealloc(o); " + code.putln("if (likely(%s)) __Pyx_PyType_GetSlot(%s, tp_dealloc, destructor)(o); " "else __Pyx_call_next_tp_dealloc(o, %s);" % ( base_cname, base_cname, slot_func_cname)) code.globalstate.use_utility_code( @@ -1536,11 +1797,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): type_safety_check = '' else: type_safety_check = ( - ' & ((Py_TYPE(o)->tp_flags & (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)) == 0)') + ' & (int)(!__Pyx_PyType_HasFeature(Py_TYPE(o), (Py_TPFLAGS_IS_ABSTRACT | Py_TPFLAGS_HEAPTYPE)))') type = scope.parent_type code.putln( - "if (CYTHON_COMPILING_IN_CPYTHON && ((%s < %d) & (Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % ( + "if (CYTHON_COMPILING_IN_CPYTHON && ((int)(%s < %d) & (int)(Py_TYPE(o)->tp_basicsize == sizeof(%s))%s)) {" % ( freecount_name, freelist_size, type.declaration_code("", deref=True), @@ -1551,12 +1812,16 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("(*Py_TYPE(o)->tp_free)(o);") if freelist_size: code.putln("}") + + if needs_trashcan: + code.putln("__Pyx_TRASHCAN_END") + code.putln( "}") def generate_usr_dealloc_call(self, scope, code): entry = scope.lookup_here("__dealloc__") - if not entry: + if not entry or not entry.is_special: return code.putln("{") @@ -1632,11 +1897,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") def generate_clear_function(self, scope, code, cclass_entry): - tp_slot = TypeSlots.get_slot_by_name("tp_clear") + tp_slot = TypeSlots.get_slot_by_name("tp_clear", scope.directives) slot_func = scope.mangle_internal("tp_clear") base_type = scope.parent_type.base_type if tp_slot.slot_code(scope) != slot_func: - return # never used + return # never used have_entries, (py_attrs, py_buffers, memoryview_slices) = ( scope.get_refcounted_entries(include_gc_simple=False)) @@ -1694,7 +1959,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("Py_CLEAR(p->%s.obj);" % entry.cname) if cclass_entry.cname == '__pyx_memoryviewslice': - code.putln("__PYX_XDEC_MEMVIEW(&p->from_slice, 1);") + code.putln("__PYX_XCLEAR_MEMVIEW(&p->from_slice, 1);") code.putln("return 0;") code.putln("}") @@ -1735,12 +2000,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if set_entry: code.putln("return %s(o, i, v);" % set_entry.func_cname) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1752,12 +2023,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_mapping", "mp_ass_subscript", "o, i, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "Subscript deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "Subscript deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1802,12 +2079,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j, v);" % ( set_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice assignment not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice assignment not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1819,12 +2102,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "return %s(o, i, j);" % ( del_entry.func_cname)) else: + code.putln( + "__Pyx_TypeName o_type_name;") self.generate_guarded_basetype_call( base_type, "tp_as_sequence", "sq_ass_slice", "o, i, j, v", code) code.putln( + "o_type_name = __Pyx_PyType_GetName(Py_TYPE(o));") + code.putln( "PyErr_Format(PyExc_NotImplementedError,") code.putln( - ' "2-element slice deletion not supported by %.200s", Py_TYPE(o)->tp_name);') + ' "2-element slice deletion not supported by " __Pyx_FMT_TYPENAME, o_type_name);') + code.putln( + "__Pyx_DECREF_TypeName(o_type_name);") code.putln( "return -1;") code.putln( @@ -1854,37 +2143,112 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # need to call up into base classes as we may not know all implemented comparison methods extern_parent = cls if cls.typeptr_cname else scope.parent_type.base_type - eq_entry = None - has_ne = False + total_ordering = 'total_ordering' in scope.directives + + comp_entry = {} + for cmp_method in TypeSlots.richcmp_special_methods: for class_scope in class_scopes: entry = class_scope.lookup_here(cmp_method) if entry is not None: + comp_entry[cmp_method] = entry break + + if total_ordering: + # Check this is valid - we must have at least 1 operation defined. + comp_names = [from_name for from_name, to_name in TOTAL_ORDERING if from_name in comp_entry] + if not comp_names: + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison and equality methods defined") + else: + warning(scope.parent_type.pos, + "total_ordering directive used, but no comparison methods defined") + total_ordering = False else: - continue + if '__eq__' not in comp_entry and '__ne__' not in comp_entry: + warning(scope.parent_type.pos, "total_ordering directive used, but no equality method defined") + total_ordering = False + + # Same priority as functools, prefers + # __lt__ to __le__ to __gt__ to __ge__ + ordering_source = max(comp_names) + for cmp_method in TypeSlots.richcmp_special_methods: cmp_type = cmp_method.strip('_').upper() # e.g. "__eq__" -> EQ + entry = comp_entry.get(cmp_method) + if entry is None and (not total_ordering or cmp_type in ('NE', 'EQ')): + # No definition, fall back to superclasses. + # eq/ne methods shouldn't use the total_ordering code. + continue + code.putln("case Py_%s: {" % cmp_type) - if cmp_method == '__eq__': - eq_entry = entry - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_True);") - elif cmp_method == '__ne__': - has_ne = True - # Python itself does not do this optimisation, it seems... - #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("return %s(o1, o2);" % entry.func_cname) - code.putln("}") + if entry is None: + assert total_ordering + # We need to generate this from the other methods. + invert_comp, comp_op, invert_equals = TOTAL_ORDERING[ordering_source, cmp_method] + + # First we always do the comparison. + code.putln("PyObject *ret;") + code.putln("ret = %s(o1, o2);" % comp_entry[ordering_source].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int order_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(order_res < 0)) return NULL;") + # We may need to check equality too. For some combos it's never required. + if invert_equals is not None: + # Implement the and/or check with an if. + if comp_op == '&&': + code.putln("if (%s order_res) {" % ('!!' if invert_comp else '!')) + code.putln("ret = __Pyx_NewRef(Py_False);") + code.putln("} else {") + elif comp_op == '||': + code.putln("if (%s order_res) {" % ('!' if invert_comp else '')) + code.putln("ret = __Pyx_NewRef(Py_True);") + code.putln("} else {") + else: + raise AssertionError('Unknown op %s' % (comp_op, )) + if '__eq__' in comp_entry: + eq_func = '__eq__' + else: + # Fall back to NE, which is defined here. + eq_func = '__ne__' + invert_equals = not invert_equals + + code.putln("ret = %s(o1, o2);" % comp_entry[eq_func].func_cname) + code.putln("if (likely(ret && ret != Py_NotImplemented)) {") + code.putln("int eq_res = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") + code.putln("if (unlikely(eq_res < 0)) return NULL;") + if invert_equals: + code.putln("ret = eq_res ? Py_False : Py_True;") + else: + code.putln("ret = eq_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # equals success + code.putln("}") # Needs to try equals + else: + # Convert direct to a boolean. + if invert_comp: + code.putln("ret = order_res ? Py_False : Py_True;") + else: + code.putln("ret = order_res ? Py_True : Py_False;") + code.putln("Py_INCREF(ret);") + code.putln("}") # comp_op + code.putln("return ret;") + else: + code.putln("return %s(o1, o2);" % entry.func_cname) + code.putln("}") # Case - if eq_entry and not has_ne and not extern_parent: + if '__eq__' in comp_entry and '__ne__' not in comp_entry and not extern_parent: code.putln("case Py_NE: {") code.putln("PyObject *ret;") # Python itself does not do this optimisation, it seems... #code.putln("if (o1 == o2) return __Pyx_NewRef(Py_False);") - code.putln("ret = %s(o1, o2);" % eq_entry.func_cname) + code.putln("ret = %s(o1, o2);" % comp_entry['__eq__'].func_cname) code.putln("if (likely(ret && ret != Py_NotImplemented)) {") - code.putln("int b = __Pyx_PyObject_IsTrue(ret); Py_DECREF(ret);") + code.putln("int b = __Pyx_PyObject_IsTrue(ret);") + code.putln("Py_DECREF(ret);") code.putln("if (unlikely(b < 0)) return NULL;") code.putln("ret = (b) ? Py_False : Py_True;") code.putln("Py_INCREF(ret);") @@ -1902,6 +2266,73 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("}") # switch code.putln("}") + def generate_binop_function(self, scope, slot, code, pos): + func_name = scope.mangle_internal(slot.slot_name) + if scope.directives['c_api_binop_methods']: + code.putln('#define %s %s' % (func_name, slot.left_slot.slot_code(scope))) + return + + code.putln() + preprocessor_guard = slot.preprocessor_guard_code() + if preprocessor_guard: + code.putln(preprocessor_guard) + + if slot.left_slot.signature in (TypeSlots.binaryfunc, TypeSlots.ibinaryfunc): + slot_type = 'binaryfunc' + extra_arg = extra_arg_decl = '' + elif slot.left_slot.signature in (TypeSlots.powternaryfunc, TypeSlots.ipowternaryfunc): + slot_type = 'ternaryfunc' + extra_arg = ', extra_arg' + extra_arg_decl = ', PyObject* extra_arg' + else: + error(pos, "Unexpected type slot signature: %s" % slot) + return + + def get_slot_method_cname(method_name): + entry = scope.lookup(method_name) + return entry.func_cname if entry and entry.is_special else None + + def call_slot_method(method_name, reverse): + func_cname = get_slot_method_cname(method_name) + if func_cname: + return "%s(%s%s)" % ( + func_cname, + "right, left" if reverse else "left, right", + extra_arg) + else: + return '%s_maybe_call_slot(__Pyx_PyType_GetSlot(%s, tp_base, PyTypeObject*), left, right %s)' % ( + func_name, + scope.parent_type.typeptr_cname, + extra_arg) + + if get_slot_method_cname(slot.left_slot.method_name) and not get_slot_method_cname(slot.right_slot.method_name): + warning(pos, "Extension type implements %s() but not %s(). " + "The behaviour has changed from previous Cython versions to match Python semantics. " + "You can implement both special methods in a backwards compatible way." % ( + slot.left_slot.method_name, + slot.right_slot.method_name, + )) + + overloads_left = int(bool(get_slot_method_cname(slot.left_slot.method_name))) + overloads_right = int(bool(get_slot_method_cname(slot.right_slot.method_name))) + code.putln( + TempitaUtilityCode.load_as_string( + "BinopSlot", "ExtensionTypes.c", + context={ + "func_name": func_name, + "slot_name": slot.slot_name, + "overloads_left": overloads_left, + "overloads_right": overloads_right, + "call_left": call_slot_method(slot.left_slot.method_name, reverse=False), + "call_right": call_slot_method(slot.right_slot.method_name, reverse=True), + "type_cname": scope.parent_type.typeptr_cname, + "slot_type": slot_type, + "extra_arg": extra_arg, + "extra_arg_decl": extra_arg_decl, + })[1]) + if preprocessor_guard: + code.putln("#endif") + def generate_getattro_function(self, scope, code): # First try to get the attribute using __getattribute__, if defined, or # PyObject_GenericGetAttr. @@ -2136,10 +2567,42 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_typeobj_spec(self, entry, code): + ext_type = entry.type + scope = ext_type.scope + + members_slot = TypeSlots.get_slot_by_name("tp_members", code.globalstate.directives) + members_slot.generate_substructure_spec(scope, code) + + buffer_slot = TypeSlots.get_slot_by_name("tp_as_buffer", code.globalstate.directives) + if not buffer_slot.is_empty(scope): + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + buffer_slot.generate_substructure(scope, code) + code.putln("#endif") + + code.putln("static PyType_Slot %s_slots[] = {" % ext_type.typeobj_cname) + for slot in TypeSlots.get_slot_table(code.globalstate.directives): + slot.generate_spec(scope, code) + code.putln("{0, 0},") + code.putln("};") + + if ext_type.typedef_flag: + objstruct = ext_type.objstruct_cname + else: + objstruct = "struct %s" % ext_type.objstruct_cname + classname = scope.class_name.as_c_string_literal() + code.putln("static PyType_Spec %s_spec = {" % ext_type.typeobj_cname) + code.putln('"%s.%s",' % (self.full_module_name, classname.replace('"', ''))) + code.putln("sizeof(%s)," % objstruct) + code.putln("0,") + code.putln("%s," % TypeSlots.get_slot_by_name("tp_flags", scope.directives).slot_code(scope)) + code.putln("%s_slots," % ext_type.typeobj_cname) + code.putln("};") + def generate_typeobj_definition(self, modname, entry, code): type = entry.type scope = type.scope - for suite in TypeSlots.substructures: + for suite in TypeSlots.get_slot_table(code.globalstate.directives).substructures: suite.generate_substructure(scope, code) code.putln("") if entry.visibility == 'public': @@ -2150,9 +2613,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(header % type.typeobj_cname) code.putln( "PyVarObject_HEAD_INIT(0, 0)") + classname = scope.class_name.as_c_string_literal() code.putln( - '"%s.%s", /*tp_name*/' % ( - self.full_module_name, scope.class_name)) + '"%s."%s, /*tp_name*/' % ( + self.full_module_name, + classname)) if type.typedef_flag: objstruct = type.objstruct_cname else: @@ -2161,7 +2626,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): "sizeof(%s), /*tp_basicsize*/" % objstruct) code.putln( "0, /*tp_itemsize*/") - for slot in TypeSlots.slot_table: + for slot in TypeSlots.get_slot_table(code.globalstate.directives): slot.generate(scope, code) code.putln( "};") @@ -2215,12 +2680,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if doc: if doc.is_unicode: doc = doc.as_utf8_string() - doc_code = doc.as_c_string_literal() + doc_code = "PyDoc_STR(%s)" % doc.as_c_string_literal() else: doc_code = "0" code.putln( - '{(char *)"%s", %s, %s, (char *)%s, 0},' % ( - entry.name, + '{(char *)%s, %s, %s, (char *)%s, 0},' % ( + entry.name.as_c_string_literal(), entry.getter_cname or "0", entry.setter_cname or "0", doc_code)) @@ -2296,7 +2761,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if code.label_used(code.error_label): code.put_label(code.error_label) # This helps locate the offending name. - code.put_add_traceback(self.full_module_name) + code.put_add_traceback(EncodedString(self.full_module_name)) code.error_label = old_error_label code.putln("bad:") code.putln("return -1;") @@ -2305,20 +2770,213 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1]) code.exit_cfunc_scope() # done with labels + def generate_module_state_start(self, env, code): + # TODO: Refactor to move module state struct decl closer to the static decl + code.putln('typedef struct {') + code.putln('PyObject *%s;' % env.module_dict_cname) + code.putln('PyObject *%s;' % Naming.builtins_cname) + code.putln('PyObject *%s;' % Naming.cython_runtime_cname) + code.putln('PyObject *%s;' % Naming.empty_tuple) + code.putln('PyObject *%s;' % Naming.empty_bytes) + code.putln('PyObject *%s;' % Naming.empty_unicode) + if Options.pre_import is not None: + code.putln('PyObject *%s;' % Naming.preimport_cname) + for type_cname, used_name in Naming.used_types_and_macros: + code.putln('#ifdef %s' % used_name) + code.putln('PyTypeObject *%s;' % type_cname) + code.putln('#endif') + + def generate_module_state_end(self, env, modules, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + module_state.putln('} %s;' % Naming.modulestate_cname) + module_state.putln('') + module_state.putln("#if CYTHON_USE_MODULE_STATE") + module_state.putln('#ifdef __cplusplus') + module_state.putln('namespace {') + module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('} /* anonymous namespace */') + module_state.putln('#else') + module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('#endif') + module_state.putln('') + module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + module_state.putln('') + module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % ( + Naming.modulestateglobal_cname, + Naming.modulestate_cname, + Naming.pymoduledef_cname)) + module_state.putln('') + module_state.putln('#define %s (PyState_FindModule(&%s))' % ( + env.module_cname, + Naming.pymoduledef_cname)) + module_state.putln("#else") + module_state.putln('static %s %s_static =' % ( + Naming.modulestate_cname, + Naming.modulestateglobal_cname + )) + module_state.putln('#ifdef __cplusplus') + # C++ likes to be initialized with {} to avoid "missing initializer" warnings + # but it isn't valid C + module_state.putln(' {};') + module_state.putln('#else') + module_state.putln(' {0};') + module_state.putln('#endif') + module_state.putln('static %s *%s = &%s_static;' % ( + Naming.modulestate_cname, + Naming.modulestateglobal_cname, + Naming.modulestateglobal_cname + )) + module_state.putln("#endif") + module_state_clear.putln("return 0;") + module_state_clear.putln("}") + module_state_clear.putln("#endif") + module_state_traverse.putln("return 0;") + module_state_traverse.putln("}") + module_state_traverse.putln("#endif") + + def generate_module_state_defines(self, env, code): + code.putln('#define %s %s->%s' % ( + env.module_dict_cname, + Naming.modulestateglobal_cname, + env.module_dict_cname)) + code.putln('#define %s %s->%s' % ( + Naming.builtins_cname, + Naming.modulestateglobal_cname, + Naming.builtins_cname)) + code.putln('#define %s %s->%s' % ( + Naming.cython_runtime_cname, + Naming.modulestateglobal_cname, + Naming.cython_runtime_cname)) + code.putln('#define %s %s->%s' % ( + Naming.empty_tuple, + Naming.modulestateglobal_cname, + Naming.empty_tuple)) + code.putln('#define %s %s->%s' % ( + Naming.empty_bytes, + Naming.modulestateglobal_cname, + Naming.empty_bytes)) + code.putln('#define %s %s->%s' % ( + Naming.empty_unicode, + Naming.modulestateglobal_cname, + Naming.empty_unicode)) + if Options.pre_import is not None: + code.putln('#define %s %s->%s' % ( + Naming.preimport_cname, + Naming.modulestateglobal_cname, + Naming.preimport_cname)) + for cname, used_name in Naming.used_types_and_macros: + code.putln('#ifdef %s' % used_name) + code.putln('#define %s %s->%s' % ( + cname, + Naming.modulestateglobal_cname, + cname)) + code.putln('#endif') + + def generate_module_state_clear(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname) + code.putln("%s *clear_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!clear_module_state) return 0;") + code.putln('Py_CLEAR(clear_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + + def generate_module_state_traverse(self, env, code): + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname) + code.putln("%s *traverse_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!traverse_module_state) return 0;") + code.putln('Py_VISIT(traverse_module_state->%s);' % + env.module_dict_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_unicode) + code.putln('#ifdef __Pyx_CyFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cyfunction_type_cname) + code.putln('#endif') + code.putln('#ifdef __Pyx_FusedFunction_USED') + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.fusedfunction_type_cname) + code.putln('#endif') + def generate_module_init_func(self, imported_modules, env, code): subfunction = self.mod_init_subfunction(self.pos, self.scope, code) + self.generate_pymoduledef_struct(env, code) + code.enter_cfunc_scope(self.scope) code.putln("") code.putln(UtilityCode.load_as_string("PyModInitFuncType", "ModuleSetupCode.c")[0]) - header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % env.module_name + if env.module_name.isascii(): + py2_mod_name = env.module_name + fail_compilation_in_py2 = False + else: + fail_compilation_in_py2 = True + # at this point py2_mod_name is largely a placeholder and the value doesn't matter + py2_mod_name = env.module_name.encode("ascii", errors="ignore").decode("utf8") + + header2 = "__Pyx_PyMODINIT_FUNC init%s(void)" % py2_mod_name header3 = "__Pyx_PyMODINIT_FUNC %s(void)" % self.mod_init_func_cname('PyInit', env) + header3 = EncodedString(header3) code.putln("#if PY_MAJOR_VERSION < 3") # Optimise for small code size as the module init function is only executed once. code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header2) + if fail_compilation_in_py2: + code.putln('#error "Unicode module names are not supported in Python 2";') + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC init__init__(void) { init%s(); }" % py2_mod_name) + code.putln("#endif") code.putln(header2) code.putln("#else") code.putln("%s CYTHON_SMALL_CODE; /*proto*/" % header3) + if self.scope.is_package: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("__Pyx_PyMODINIT_FUNC PyInit___init__(void) { return %s(); }" % ( + self.mod_init_func_cname('PyInit', env))) + code.putln("#endif") + # Hack for a distutils bug - https://bugs.python.org/issue39432 + # distutils attempts to make visible a slightly wrong PyInitU module name. Just create a dummy + # function to keep it quiet + wrong_punycode_module_name = self.wrong_punycode_module_name(env.module_name) + if wrong_punycode_module_name: + code.putln("#if !defined(CYTHON_NO_PYINIT_EXPORT) && (defined(_WIN32) || defined(WIN32) || defined(MS_WINDOWS))") + code.putln("void %s(void) {} /* workaround for https://bugs.python.org/issue39432 */" % wrong_punycode_module_name) + code.putln("#endif") code.putln(header3) # CPython 3.5+ supports multi-phase module initialisation (gives access to __spec__, __file__, etc.) @@ -2333,7 +2991,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") # main module init code lives in Py_mod_exec function, not in PyInit function code.putln("static CYTHON_SMALL_CODE int %s(PyObject *%s)" % ( - self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env), + self.module_init_func_cname(), Naming.pymodinit_module_arg)) code.putln("#endif") # PEP489 @@ -2341,12 +2999,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # start of module init/exec function (pre/post PEP 489) code.putln("{") + code.putln('int stringtab_initialized = 0;') + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln('int pystate_addmodule_run = 0;') + code.putln("#endif") tempdecl_code = code.insertion_point() profile = code.globalstate.directives['profile'] linetrace = code.globalstate.directives['linetrace'] if profile or linetrace: + if linetrace: + code.use_fast_gil_utility_code() code.globalstate.use_utility_code(UtilityCode.load_cached("Profile", "Profile.c")) code.put_declare_refcount_context() @@ -2361,7 +3025,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln('PyErr_SetString(PyExc_RuntimeError,' ' "Module \'%s\' has already been imported. Re-initialisation is not supported.");' % - env.module_name) + env.module_name.as_c_string_literal()[1:-1]) code.putln("return -1;") code.putln("}") code.putln("#elif PY_MAJOR_VERSION >= 3") @@ -2372,6 +3036,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln("#endif") + code.putln("/*--- Module creation code ---*/") + self.generate_module_creation_code(env, code) + if profile or linetrace: tempdecl_code.put_trace_declarations() code.put_trace_frame_init() @@ -2395,7 +3062,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for ext_type in ('CyFunction', 'FusedFunction', 'Coroutine', 'Generator', 'AsyncGen', 'StopAsyncIteration'): code.putln("#ifdef __Pyx_%s_USED" % ext_type) - code.put_error_if_neg(self.pos, "__pyx_%s_init()" % ext_type) + code.put_error_if_neg(self.pos, "__pyx_%s_init(%s)" % (ext_type, env.module_cname)) code.putln("#endif") code.putln("/*--- Library function declarations ---*/") @@ -2408,18 +3075,18 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyEval_InitThreads();") code.putln("#endif") - code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) - code.putln("/*--- Initialize various global constants etc. ---*/") - code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") + code.put_error_if_neg(self.pos, "__Pyx_InitConstants()") + code.putln("stringtab_initialized = 1;") + code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") # calls any utility code + code.putln("#if PY_MAJOR_VERSION < 3 && (__PYX_DEFAULT_STRING_ENCODING_IS_ASCII || " "__PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT)") code.put_error_if_neg(self.pos, "__Pyx_init_sys_getdefaultencoding_params()") code.putln("#endif") - code.putln("if (%s%s) {" % (Naming.module_is_main, self.full_module_name.replace('.', '__'))) + code.putln("if (%s) {" % self.is_main_module_flag_cname()) code.put_error_if_neg(self.pos, 'PyObject_SetAttr(%s, %s, %s)' % ( env.module_cname, code.intern_identifier(EncodedString("__name__")), @@ -2474,7 +3141,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put_trace_call(header3, self.pos, nogil=not code.funcstate.gil_owned) code.funcstate.can_trace = True + code.mark_pos(None) self.body.generate_execution_code(code) + code.mark_pos(None) if profile or linetrace: code.funcstate.can_trace = False @@ -2495,8 +3164,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for cname, type in code.funcstate.all_managed_temps(): code.put_xdecref(cname, type) code.putln('if (%s) {' % env.module_cname) - code.putln('if (%s) {' % env.module_dict_cname) - code.put_add_traceback("init %s" % env.qualified_name) + code.putln('if (%s && stringtab_initialized) {' % env.module_dict_cname) + # We can run into errors before the module or stringtab are initialized. + # In this case it is not safe to add a traceback (because it uses the stringtab) + code.put_add_traceback(EncodedString("init %s" % env.qualified_name)) code.globalstate.use_utility_code(Nodes.traceback_utility_code) # Module reference and module dict are in global variables which might still be needed # for cleanup, atexit code, etc., so leaking is better than crashing. @@ -2504,9 +3175,24 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # user code in atexit or other global registries. ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False) code.putln('}') + code.putln("#if !CYTHON_USE_MODULE_STATE") code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#else") + # This section is mainly for the limited API. env.module_cname still owns a reference so + # decrement that + code.put_decref(env.module_cname, py_object_type, nanny=False) + # Also remove the failed module from the module state lookup + # fetch/restore the error indicator because PyState_RemvoeModule might fail itself + code.putln("if (pystate_addmodule_run) {") + code.putln("PyObject *tp, *value, *tb;") + code.putln("PyErr_Fetch(&tp, &value, &tb);") + code.putln("PyState_RemoveModule(&%s);" % Naming.pymoduledef_cname) + code.putln("PyErr_Restore(tp, value, tb);") + code.putln("}") + code.putln("#endif") code.putln('} else if (!PyErr_Occurred()) {') - code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name) + code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % + env.qualified_name.as_c_string_literal()[1:-1]) code.putln('}') code.put_label(code.return_label) @@ -2556,7 +3242,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("static int %s(void) {" % self.cfunc_name) code.put_declare_refcount_context() self.tempdecl_code = code.insertion_point() - code.put_setup_refcount_context(self.cfunc_name) + code.put_setup_refcount_context(EncodedString(self.cfunc_name)) # Leave a grepable marker that makes it easy to find the generator source. code.putln("/*--- %s ---*/" % self.description) return code @@ -2613,7 +3299,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): EncodedString(decode_filename( os.path.dirname(module_path)))).cname, code.error_goto_if_null(temp, self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) code.putln( 'if (PyObject_SetAttrString(%s, "__path__", %s) < 0) %s;' % ( env.module_cname, temp, code.error_goto(self.pos))) @@ -2637,14 +3323,15 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # CPython may not have put us into sys.modules yet, but relative imports and reimports require it fq_module_name = self.full_module_name if fq_module_name.endswith('.__init__'): - fq_module_name = fq_module_name[:-len('.__init__')] + fq_module_name = EncodedString(fq_module_name[:-len('.__init__')]) + fq_module_name_cstring = fq_module_name.as_c_string_literal() code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("{") code.putln("PyObject *modules = PyImport_GetModuleDict(); %s" % code.error_goto_if_null("modules", self.pos)) - code.putln('if (!PyDict_GetItemString(modules, "%s")) {' % fq_module_name) - code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, "%s", %s)' % ( - fq_module_name, env.module_cname), self.pos)) + code.putln('if (!PyDict_GetItemString(modules, %s)) {' % fq_module_name_cstring) + code.putln(code.error_goto_if_neg('PyDict_SetItemString(modules, %s, %s)' % ( + fq_module_name_cstring, env.module_cname), self.pos)) code.putln("}") code.putln("}") code.putln("#endif") @@ -2717,11 +3404,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if Options.pre_import is not None: code.put_decref_clear(Naming.preimport_cname, py_object_type, nanny=False, clear_before_decref=True) - for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]: + for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]: code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True) + code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True) def generate_main_method(self, env, code): - module_is_main = "%s%s" % (Naming.module_is_main, self.full_module_name.replace('.', '__')) + module_is_main = self.is_main_module_flag_cname() if Options.embed == "main": wmain = "wmain" else: @@ -2734,8 +3422,33 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): main_method=Options.embed, wmain_method=wmain)) + def punycode_module_name(self, prefix, name): + # adapted from PEP483 + try: + name = '_' + name.encode('ascii').decode('ascii') + except UnicodeEncodeError: + name = 'U_' + name.encode('punycode').replace(b'-', b'_').decode('ascii') + return "%s%s" % (prefix, name) + + def wrong_punycode_module_name(self, name): + # to work around a distutils bug by also generating an incorrect symbol... + try: + name.encode("ascii") + return None # workaround is not needed + except UnicodeEncodeError: + return "PyInitU" + (u"_"+name).encode('punycode').replace(b'-', b'_').decode('ascii') + def mod_init_func_cname(self, prefix, env): - return '%s_%s' % (prefix, env.module_name) + # from PEP483 + return self.punycode_module_name(prefix, env.module_name) + + # Returns the name of the C-function that corresponds to the module initialisation. + # (module initialisation == the cython code outside of functions) + # Note that this should never be the name of a wrapper and always the name of the + # function containing the actual code. Otherwise, cygdb will experience problems. + def module_init_func_cname(self): + env = self.scope + return self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) def generate_pymoduledef_struct(self, env, code): if env.doc: @@ -2750,7 +3463,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") code.putln("#if PY_MAJOR_VERSION >= 3") code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") - exec_func_cname = self.mod_init_func_cname(Naming.pymodule_exec_func_cname, env) + exec_func_cname = self.module_init_func_cname() code.putln("static PyObject* %s(PyObject *spec, PyModuleDef *def); /*proto*/" % Naming.pymodule_create_func_cname) code.putln("static int %s(PyObject* module); /*proto*/" % exec_func_cname) @@ -2760,15 +3473,27 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("{Py_mod_exec, (void*)%s}," % exec_func_cname) code.putln("{0, NULL}") code.putln("};") + if not env.module_name.isascii(): + code.putln("#else /* CYTHON_PEP489_MULTI_PHASE_INIT */") + code.putln('#error "Unicode module names are only supported with multi-phase init' + ' as per PEP489"') code.putln("#endif") code.putln("") - code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#ifdef __cplusplus') + code.putln('namespace {') + code.putln("struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#else') + code.putln("static struct PyModuleDef %s =" % Naming.pymoduledef_cname) + code.putln('#endif') + code.putln('{') code.putln(" PyModuleDef_HEAD_INIT,") - code.putln(' "%s",' % env.module_name) + code.putln(' %s,' % env.module_name.as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") + code.putln("#elif CYTHON_USE_MODULE_STATE") # FIXME: should allow combination with PEP-489 + code.putln(" sizeof(%s), /* m_size */" % Naming.modulestate_cname) code.putln("#else") code.putln(" -1, /* m_size */") code.putln("#endif") @@ -2778,10 +3503,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln(" NULL, /* m_reload */") code.putln("#endif") + code.putln("#if CYTHON_USE_MODULE_STATE") + code.putln(" %s_traverse, /* m_traverse */" % Naming.module_cname) + code.putln(" %s_clear, /* m_clear */" % Naming.module_cname) + code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#else") code.putln(" NULL, /* m_traverse */") code.putln(" NULL, /* m_clear */") code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#endif") code.putln("};") + code.putln('#ifdef __cplusplus') + code.putln('} /* anonymous namespace */') + code.putln('#endif') code.putln("#endif") def generate_module_creation_code(self, env, code): @@ -2800,20 +3534,44 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln("#if PY_MAJOR_VERSION < 3") code.putln( - '%s = Py_InitModule4("%s", %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( + '%s = Py_InitModule4(%s, %s, %s, 0, PYTHON_API_VERSION); Py_XINCREF(%s);' % ( env.module_cname, - env.module_name, + env.module_name.as_c_string_literal(), env.method_table_cname, doc, env.module_cname)) - code.putln("#else") + code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#elif CYTHON_USE_MODULE_STATE") + # manage_ref is False (and refnanny calls are omitted) because refnanny isn't yet initialized + module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=False) + code.putln( + "%s = PyModule_Create(&%s); %s" % ( + module_temp, + Naming.pymoduledef_cname, + code.error_goto_if_null(module_temp, self.pos))) + code.putln("{") + # So that PyState_FindModule works in the init function: + code.putln("int add_module_result = PyState_AddModule(%s, &%s);" % ( + module_temp, Naming.pymoduledef_cname)) + code.putln("%s = 0; /* transfer ownership from %s to %s pseudovariable */" % ( + module_temp, module_temp, env.module_name + )) + # At this stage the module likely has a refcount of 2 - one owned by the list + # inside PyState_AddModule and one owned by "__pyx_m" (and returned from this + # function as a new reference). + code.putln(code.error_goto_if_neg("add_module_result", self.pos)) + code.putln("pystate_addmodule_run = 1;") + code.putln("}") + code.funcstate.release_temp(module_temp) + code.putln('#else') code.putln( "%s = PyModule_Create(&%s);" % ( env.module_cname, Naming.pymoduledef_cname)) - code.putln("#endif") code.putln(code.error_goto_if_null(env.module_cname, self.pos)) + code.putln("#endif") code.putln("#endif") # CYTHON_PEP489_MULTI_PHASE_INIT + code.putln("CYTHON_UNUSED_VAR(%s);" % module_temp) # only used in limited API code.putln( "%s = PyModule_GetDict(%s); %s" % ( @@ -2903,8 +3661,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # investigation shows that the resulting binary is smaller with repeated functions calls. for entry in entries: signature = entry.type.signature_string() - code.putln('if (__Pyx_ExportFunction("%s", (void (*)(void))%s, "%s") < 0) %s' % ( - entry.name, + code.putln('if (__Pyx_ExportFunction(%s, (void (*)(void))%s, "%s") < 0) %s' % ( + entry.name.as_c_string_literal(), entry.cname, signature, code.error_goto(self.pos))) @@ -2946,7 +3704,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: if env is module: cname = entry.cname @@ -2977,13 +3735,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module.qualified_name, temp, code.error_goto(self.pos))) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) for entry in entries: code.putln( - 'if (__Pyx_ImportFunction_%s(%s, "%s", (void (**)(void))&%s, "%s") < 0) %s' % ( + 'if (__Pyx_ImportFunction_%s(%s, %s, (void (**)(void))&%s, "%s") < 0) %s' % ( Naming.cyversion, temp, - entry.name, + entry.name.as_c_string_literal(), entry.cname, entry.type.signature_string(), code.error_goto(self.pos))) @@ -3006,7 +3764,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_base_type_import_code(self, env, entry, code, import_generator): base_type = entry.type.base_type if (base_type and base_type.module_name != env.qualified_name and not - base_type.is_builtin_type and not entry.utility_code_definition): + (base_type.is_builtin_type or base_type.is_cython_builtin_type) + and not entry.utility_code_definition): self.generate_type_import_code(env, base_type, self.pos, code, import_generator) def generate_type_import_code(self, env, type, pos, code, import_generator): @@ -3023,7 +3782,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if type.vtabptr_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('GetVTable', 'ImportExport.c')) - code.putln("%s = (struct %s*)__Pyx_GetVtable(%s->tp_dict); %s" % ( + code.putln("%s = (struct %s*)__Pyx_GetVtable(%s); %s" % ( type.vtabptr_cname, type.vtabstruct_cname, type.typeptr_cname, @@ -3064,15 +3823,17 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): module, module_name)) + type_name = type.name.as_c_string_literal() + if condition and replacement: code.putln("") # start in new line code.putln("#if %s" % condition) code.putln('"%s",' % replacement) code.putln("#else") - code.putln('"%s",' % type.name) + code.putln('%s,' % type_name) code.putln("#endif") else: - code.put(' "%s", ' % type.name) + code.put(' %s, ' % type_name) if sizeof_objstruct != objstruct: if not condition: @@ -3080,6 +3841,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000") code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( objstruct, Naming.cyversion, objstruct)) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( + objstruct, Naming.cyversion, objstruct)) code.putln("#else") code.putln('sizeof(%s), __PYX_GET_STRUCT_ALIGNMENT_%s(%s),' % ( sizeof_objstruct, Naming.cyversion, sizeof_objstruct)) @@ -3104,6 +3868,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): def generate_type_ready_code(self, entry, code): Nodes.CClassDefNode.generate_type_ready_code(entry, code) + def is_main_module_flag_cname(self): + full_module_name = self.full_module_name.replace('.', '__') + return self.punycode_module_name(Naming.module_is_main, full_module_name) + def generate_exttype_vtable_init_code(self, entry, code): # Generate code to initialise the C method table of an # extension type. @@ -3156,7 +3924,7 @@ class ModuleImportGenerator(object): self.temps.append(temp) code.putln('%s = PyImport_ImportModule(%s); if (unlikely(!%s)) %s' % ( temp, module_name_string, temp, error_code)) - code.put_gotref(temp) + code.put_gotref(temp, py_object_type) self.imported[module_name_string] = temp return temp @@ -3216,5 +3984,3 @@ packed_struct_utility_code = UtilityCode(proto=""" #define __Pyx_PACKED #endif """, impl="", proto_block='utility_code_proto_before_types') - -capsule_utility_code = UtilityCode.load("Capsule") |