diff options
author | Eddie Elizondo <eduardo.elizondorueda@gmail.com> | 2020-01-12 00:51:41 -0800 |
---|---|---|
committer | Stefan Behnel <stefan_ml@behnel.de> | 2020-01-12 09:51:41 +0100 |
commit | 84473513e5f709089cf201ac950350b359dc2453 (patch) | |
tree | b591fb01c9ff1aed34c79531069e953aceae2cbf | |
parent | 640aab7408d6fa1cf139b0b175b60fddae56a487 (diff) | |
download | cython-84473513e5f709089cf201ac950350b359dc2453.tar.gz |
Add LIMITED_API support and remove static state (GH-3223)
Blacklists failing test for now
-rw-r--r-- | .travis.yml | 13 | ||||
-rw-r--r-- | Cython/Compiler/Code.py | 59 | ||||
-rw-r--r-- | Cython/Compiler/ExprNodes.py | 31 | ||||
-rw-r--r-- | Cython/Compiler/ModuleNode.py | 290 | ||||
-rw-r--r-- | Cython/Compiler/Naming.py | 2 | ||||
-rw-r--r-- | Cython/Compiler/Nodes.py | 40 | ||||
-rw-r--r-- | Cython/Compiler/PyrexTypes.py | 2 | ||||
-rw-r--r-- | Cython/Compiler/TypeSlots.py | 21 | ||||
-rw-r--r-- | Cython/Utility/Exceptions.c | 12 | ||||
-rw-r--r-- | Cython/Utility/ExtensionTypes.c | 6 | ||||
-rw-r--r-- | Cython/Utility/ImportExport.c | 18 | ||||
-rw-r--r-- | Cython/Utility/ModuleSetupCode.c | 117 | ||||
-rw-r--r-- | Cython/Utility/ObjectHandling.c | 8 | ||||
-rw-r--r-- | Cython/Utility/Optimize.c | 12 | ||||
-rw-r--r-- | Cython/Utility/StringTools.c | 50 | ||||
-rw-r--r-- | Cython/Utility/TypeConversion.c | 22 | ||||
-rwxr-xr-x | runtests.py | 7 | ||||
-rw-r--r-- | tests/limited_api_bugs.txt | 25 |
18 files changed, 693 insertions, 42 deletions
diff --git a/.travis.yml b/.travis.yml index 09164dc1b..1c84f7121 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,6 +51,10 @@ matrix: dist: xenial # Required for Python 3.7 sudo: required # travis-ci/travis-ci#9069 env: TEST_CODE_STYLE=1 + - python: 3.7 + dist: xenial # Required for Python 3.7 + sudo: required # travis-ci/travis-ci#9069 + env: LIMITED_API=--limited-api EXCLUDE=--no-file - python: 3.4 env: BACKEND=c - python: 3.4 @@ -63,6 +67,9 @@ matrix: env: BACKEND=c - python: 3.6 env: BACKEND=cpp + - python: 3.6 + sudo: required # travis-ci/travis-ci#9069 + env: LIMITED_API=--limited-api EXCLUDE=--no-file - python: 3.8 dist: xenial # Required for Python 3.7 sudo: required # travis-ci/travis-ci#9069 @@ -71,6 +78,10 @@ matrix: dist: xenial # Required for Python 3.7 sudo: required # travis-ci/travis-ci#9069 env: BACKEND=cpp + - python: 3.8-dev + dist: xenial # Required for Python 3.8 + sudo: required # travis-ci/travis-ci#9069 + env: LIMITED_API=--limited-api EXCLUDE=--no-file - os: osx osx_image: xcode6.4 env: PY=2 @@ -154,5 +165,5 @@ script: # Need to clear the ccache? Try something like this: # - if [ -n "${BACKEND##*cpp*}" -a -z "${TRAVIS_PYTHON_VERSION##*3.4}" ]; then ccache -C || true; fi - if [ "$COVERAGE" != "1" ]; then CFLAGS="-O2 -ggdb -Wall -Wextra $(python -c 'import sys; print("-fno-strict-aliasing" if sys.version_info[0] == 2 else "")')" python setup.py build_ext -i; fi - - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) + - CFLAGS="-O0 -ggdb -Wall -Wextra" python runtests.py -vv $STYLE_ARGS -x Debugger --backends=$BACKEND $LIMITED_API $EXCLUDE $(if [ "$COVERAGE" == "1" ]; then echo " --coverage"; fi) $(if [ -z "$TEST_CODE_STYLE" ]; then echo " -j7 "; fi) - ccache -s || true diff --git a/Cython/Compiler/Code.py b/Cython/Compiler/Code.py index 97983b5bd..c7b228905 100644 --- a/Cython/Compiler/Code.py +++ b/Cython/Compiler/Code.py @@ -1080,6 +1080,10 @@ class GlobalState(object): 'h_code', 'filename_table', 'utility_code_proto_before_types', + 'module_state', + 'module_state_clear', + 'module_state_traverse', + 'module_state_defines', 'numeric_typedefs', # Let these detailed individual parts stay!, 'complex_type_declarations', # as the proper solution is to make a full DAG... 'type_declarations', # More coarse-grained blocks would simply hide @@ -1420,9 +1424,18 @@ class GlobalState(object): for c in self.py_constants] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") for _, cname, c in consts: + self.parts['module_state'].putln("%s;" % c.type.declaration_code(cname)) + self.parts['module_state_defines'].putln( + "#define %s %s->%s" % (cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln( "static %s;" % c.type.declaration_code(cname)) + decls_writer.putln("#endif") def generate_cached_methods_decls(self): if not self.cached_cmethods: @@ -1476,13 +1489,15 @@ class GlobalState(object): decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array)) decls_writer.putln("#endif") + init_globals = self.parts['init_globals'] if py_strings: self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c")) py_strings.sort() w = self.parts['pystring_table'] w.putln("") w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname) - for c_cname, _, py_string in py_strings: + for idx, py_string_args in enumerate(py_strings): + c_cname, _, py_string = py_string_args if not py_string.is_str or not py_string.encoding or \ py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII', 'UTF8', 'UTF-8'): @@ -1490,8 +1505,19 @@ class GlobalState(object): else: encoding = '"%s"' % py_string.encoding.lower() + self.parts['module_state'].putln("PyObject *%s;" % py_string.cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + py_string.cname, + Naming.modulestateglobal_cname, + py_string.cname)) + self.parts['module_state_clear'].putln("Py_CLEAR(clear_module_state->%s);" % + py_string.cname) + self.parts['module_state_traverse'].putln("Py_VISIT(traverse_module_state->%s);" % + py_string.cname) + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") decls_writer.putln( "static PyObject *%s;" % py_string.cname) + decls_writer.putln("#endif") if py_string.py3str_cstring: w.putln("#if PY_MAJOR_VERSION >= 3") w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( @@ -1502,6 +1528,17 @@ class GlobalState(object): py_string.intern )) w.putln("#else") + + w.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + w.putln("{0, %s, sizeof(%s), %s, %d, %d, %d}," % ( + c_cname, + c_cname, + encoding, + py_string.is_unicode, + py_string.is_str, + py_string.intern + )) + w.putln("#else") w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % ( py_string.cname, c_cname, @@ -1511,25 +1548,42 @@ class GlobalState(object): py_string.is_str, py_string.intern )) + w.putln("#endif") + init_globals.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + init_globals.putln("if (__Pyx_InitString(%s[%d], &%s) < 0) %s;" % ( + Naming.stringtab_cname, + idx, + py_string.cname, + init_globals.error_goto(self.module_pos))) + init_globals.putln("#endif") if py_string.py3str_cstring: w.putln("#endif") w.putln("{0, 0, 0, 0, 0, 0, 0}") w.putln("};") - init_globals = self.parts['init_globals'] + init_globals.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") init_globals.putln( "if (__Pyx_InitStrings(%s) < 0) %s;" % ( Naming.stringtab_cname, init_globals.error_goto(self.module_pos))) + init_globals.putln("#endif") def generate_num_constants(self): consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c) for c in self.num_const_index.values()] consts.sort() decls_writer = self.parts['decls'] + decls_writer.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") init_globals = self.parts['init_globals'] for py_type, _, _, value, value_code, c in consts: cname = c.cname + self.parts['module_state'].putln("PyObject *%s;" % cname) + self.parts['module_state_defines'].putln("#define %s %s->%s" % ( + cname, Naming.modulestateglobal_cname, cname)) + self.parts['module_state_clear'].putln( + "Py_CLEAR(clear_module_state->%s);" % cname) + self.parts['module_state_traverse'].putln( + "Py_VISIT(traverse_module_state->%s);" % cname) decls_writer.putln("static PyObject *%s;" % cname) if py_type == 'float': function = 'PyFloat_FromDouble(%s)' @@ -1544,6 +1598,7 @@ class GlobalState(object): init_globals.putln('%s = %s; %s' % ( cname, function % value_code, init_globals.error_goto_if_null(cname, self.module_pos))) + decls_writer.putln("#endif") # The functions below are there in a transition phase only # and will be deprecated. They are called from Nodes.BlockNode. diff --git a/Cython/Compiler/ExprNodes.py b/Cython/Compiler/ExprNodes.py index fc65696a7..5957e3eed 100644 --- a/Cython/Compiler/ExprNodes.py +++ b/Cython/Compiler/ExprNodes.py @@ -2301,6 +2301,14 @@ class NameNode(AtomicExprNode): setter = '__Pyx_' + n else: assert False, repr(entry) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.put_incref(rhs.py_result(), py_object_type) + code.put_error_if_neg(self.pos, '%s(%s, %s, %s)' % ( + "PyModule_AddObject", + Naming.module_cname, + code.get_string_const(self.entry.name), + rhs.py_result())) + code.putln("#else") code.put_error_if_neg( self.pos, '%s(%s, %s, %s)' % ( @@ -2308,6 +2316,7 @@ class NameNode(AtomicExprNode): namespace, interned_cname, rhs.py_result())) + code.putln("#endif") if debug_disposal_code: print("NameNode.generate_assignment_code:") print("...generating disposal code for %s" % rhs) @@ -9370,6 +9379,27 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): else: flags = '0' + code.putln('#if CYTHON_COMPILING_IN_LIMITED_API') + dict_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln('%s = PyDict_New(); %s' % ( + dict_temp, + code.error_goto_if_null(dict_temp, self.pos))) + code.put_gotref(dict_temp) + code.putln( + '%s = %s(&%s, %s, %s, %s, %s, %s, %s); %s' % ( + self.result(), + constructor, + self.pymethdef_cname, + flags, + self.get_py_qualified_name(code), + self.closure_result_code(), + self.get_py_mod_name(code), + dict_temp, + code_object_result, + code.error_goto_if_null(self.result(), self.pos))) + code.put_decref_clear(dict_temp, type=py_object_type) + code.funcstate.release_temp(dict_temp) + code.putln('#else') code.putln( '%s = %s(&%s, %s, %s, %s, %s, %s, %s); %s' % ( self.result(), @@ -9382,6 +9412,7 @@ class PyCFunctionNode(ExprNode, ModuleNameMixin): Naming.moddict_cname, code_object_result, code.error_goto_if_null(self.result(), self.pos))) + code.putln('#endif') code.put_gotref(self.py_result()) diff --git a/Cython/Compiler/ModuleNode.py b/Cython/Compiler/ModuleNode.py index 586035604..81627212f 100644 --- a/Cython/Compiler/ModuleNode.py +++ b/Cython/Compiler/ModuleNode.py @@ -418,6 +418,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # initialise the macro to reduce the code size of one-time functionality code.putln(UtilityCode.load_as_string("SmallCodeConfig", "ModuleSetupCode.c")[0].strip()) + self.generate_module_state_start(env, globalstate['module_state']) + self.generate_module_state_defines(env, globalstate['module_state_defines']) + self.generate_module_state_clear(env, globalstate['module_state_clear']) + self.generate_module_state_traverse(env, globalstate['module_state_traverse']) + # init_globals is inserted before this self.generate_module_init_func(modules[:-1], env, globalstate['init_module']) self.generate_module_cleanup_func(env, globalstate['cleanup_module']) @@ -432,6 +437,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): globalstate.use_utility_code(utilcode) globalstate.finalize_main_c_code() + self.generate_module_state_end(env, modules, globalstate) + f = open_new_file(result.c_file) try: rootwriter.copyto(f) @@ -638,7 +645,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): defined_here = module is env modulecode.putln("") modulecode.putln("/* Module declarations from %s */" % module.qualified_name.as_c_string_literal()) - self.generate_c_class_declarations(module, modulecode, defined_here) + self.generate_c_class_declarations(module, modulecode, defined_here, globalstate) self.generate_cvariable_declarations(module, modulecode, defined_here) self.generate_cfunction_declarations(module, modulecode, defined_here) @@ -742,6 +749,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.put(Nodes.branch_prediction_macros) code.putln('static CYTHON_INLINE void __Pyx_pretend_to_initialize(void* ptr) { (void)ptr; }') code.putln('') + code.putln('#if !CYTHON_COMPILING_IN_LIMITED_API') code.putln('static PyObject *%s = NULL;' % env.module_cname) code.putln('static PyObject *%s;' % env.module_dict_cname) code.putln('static PyObject *%s;' % Naming.builtins_cname) @@ -755,6 +763,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln('static int %s = 0;' % Naming.clineno_cname) code.putln('static const char * %s= %s;' % (Naming.cfilenm_cname, Naming.file_c_macro)) code.putln('static const char *%s;' % Naming.filename_cname) + code.putln('#endif') env.use_utility_code(UtilityCode.load_cached("FastTypeChecks", "ModuleSetupCode.c")) if has_np_pythran(env): @@ -1167,11 +1176,28 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # Only for exposing public typedef name. code.putln("typedef struct %s %s;" % (type.objstruct_cname, type.objtypedef_cname)) - def generate_c_class_declarations(self, env, code, definition): + def generate_c_class_declarations(self, env, code, definition, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") for entry in env.c_class_entries: if definition or entry.defined_in_pxd: code.putln("static PyTypeObject *%s = 0;" % ( entry.type.typeptr_cname)) + module_state.putln("PyTypeObject *%s;" % entry.type.typeptr_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeptr_cname, + Naming.modulestateglobal_cname, + entry.type.typeptr_cname)) + module_state_clear.putln( + "Py_CLEAR(clear_module_state->%s);" % + entry.type.typeptr_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeptr_cname) + code.putln("#endif") def generate_cvariable_declarations(self, env, code, definition): if env.is_cython_builtin: @@ -1277,7 +1303,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): self.generate_property_accessors(scope, code) self.generate_method_table(scope, code) self.generate_getset_table(scope, code) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + self.generate_typeobj_spec(full_module_name, entry, code) + code.putln("#else") self.generate_typeobj_definition(full_module_name, entry, code) + code.putln("#endif") def generate_exttype_vtable(self, scope, code): # Generate the definition of an extension type's vtable. @@ -1329,8 +1359,12 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): freecount_name = scope.mangle_internal(Naming.freecount_name) decls = code.globalstate['decls'] + if cinit_func_entry is None: + decls.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") decls.putln("static PyObject *%s(PyTypeObject *t, PyObject *a, PyObject *k); /*proto*/" % slot_func) + if cinit_func_entry is None: + decls.putln("#endif") code.putln("") if freelist_size: code.putln("static %s[%d];" % ( @@ -1338,6 +1372,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): freelist_size)) code.putln("static int %s = 0;" % freecount_name) code.putln("") + if cinit_func_entry is None: + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln( "static PyObject *%s(PyTypeObject *t, %sPyObject *a, %sPyObject *k) {" % ( slot_func, unused_marker, unused_marker)) @@ -1354,6 +1390,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("PyObject *o = %s(t, a, k);" % tp_new) else: code.putln("PyObject *o;") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("allocfunc alloc_func = (allocfunc)PyType_GetSlot(t, Py_tp_alloc);") + code.putln("o = alloc_func(t, 0);") + code.putln("#else") if freelist_size: code.globalstate.use_utility_code( UtilityCode.load_cached("IncludeStringH", "StringTools.c")) @@ -1379,6 +1419,7 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("} else {") code.putln("o = (PyObject *) PyBaseObject_Type.tp_new(t, %s, 0);" % Naming.empty_tuple) code.putln("}") + code.putln("#endif") code.putln("if (unlikely(!o)) return 0;") if freelist_size and not base_type: code.putln('}') @@ -1441,6 +1482,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("return NULL;") code.putln( "}") + if cinit_func_entry is None: + code.putln("#endif") def generate_dealloc_function(self, scope, code): tp_slot = TypeSlots.ConstructorSlot("tp_dealloc", '__dealloc__') @@ -1451,6 +1494,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): slot_func_cname = scope.mangle_internal("tp_dealloc") code.putln("") + cdealloc_func_entry = scope.lookup_here("__dealloc__") + if cdealloc_func_entry and not cdealloc_func_entry.is_special: + cdealloc_func_entry = None + if cdealloc_func_entry is None: + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln( "static void %s(PyObject *o) {" % slot_func_cname) @@ -1584,6 +1632,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + if cdealloc_func_entry is None: + code.putln("#endif") def generate_usr_dealloc_call(self, scope, code): entry = scope.lookup_here("__dealloc__") @@ -2165,6 +2215,52 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln( "}") + def generate_typeobj_spec(self, modname, entry, code): + type = entry.type + scope = type.scope + if type.typedef_flag: + objstruct = type.objstruct_cname + else: + objstruct = "struct %s" % type.objstruct_cname + classname = scope.class_name.as_c_string_literal() + code.putln("static PyType_Slot %s_slots[] = {" % type.typeobj_cname) + has_tp_getattro = False + for slot in TypeSlots.slot_table: + if slot.slot_name == "tp_flags": + continue + if slot.slot_name == "tp_new" and scope.lookup_here("__cinit__") is None: + continue + if slot.slot_name == "tp_dealloc" and scope.lookup_here("__dealloc__") is None: + continue + if slot.slot_name == "tp_getattro": + has_tp_getattro = True + if slot.slot_name == "tp_as_number": + slot.generate_substructure_spec(scope, code) + continue + if slot.slot_name == "tp_as_sequence": + slot.generate_substructure_spec(scope, code) + continue + if slot.slot_name == "tp_as_mapping": + slot.generate_substructure_spec(scope, code) + continue + if slot.slot_name == "tp_as_buffer": # Can't support tp_as_buffer + continue + v = TypeSlots.get_slot_by_name(slot.slot_name).spec_slot_value(scope) + if v is not None: + code.putln(" {Py_%s, (void *)%s}," % (slot.slot_name, v)) + if not has_tp_getattro: + code.putln(" {Py_tp_getattro, __Pyx_PyObject_GenericGetAttr},") + code.putln(" {0, 0},") + code.putln("};") + + code.putln("static PyType_Spec %s_spec = {" % type.typeobj_cname) + code.putln(" \"%s.%s\"," % (self.full_module_name, classname.replace("\"", ""))) + code.putln(" sizeof(%s)," % objstruct) + code.putln(" 0,") + code.putln(" %s," % TypeSlots.get_slot_by_name("tp_flags").spec_slot_value(scope)) + code.putln(" %s_slots," % type.typeobj_cname) + code.putln("};") + def generate_typeobj_definition(self, modname, entry, code): type = entry.type scope = type.scope @@ -2336,6 +2432,147 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(UtilityCode.load_as_string("ImportStar", "ImportExport.c")[1]) code.exit_cfunc_scope() # done with labels + def generate_module_state_start(self, env, code): + # TODO: Reactor LIMITED_API struct decl closer to the static decl + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln('typedef struct {') + code.putln('PyObject *%s;' % Naming.builtins_cname) + code.putln('PyObject *%s;' % Naming.cython_runtime_cname) + code.putln('PyObject *%s;' % Naming.empty_tuple) + code.putln('PyObject *%s;' % Naming.empty_bytes) + code.putln('PyObject *%s;' % Naming.empty_unicode) + if Options.pre_import is not None: + code.putln('PyObject *%s;' % Naming.preimport_cname) + code.putln('int %s;' % Naming.lineno_cname) + code.putln('int %s;' % Naming.clineno_cname) + code.putln('const char *%s;' % Naming.filename_cname) + + def generate_module_state_end(self, env, modules, globalstate): + module_state = globalstate['module_state'] + module_state_defines = globalstate['module_state_defines'] + module_state_clear = globalstate['module_state_clear'] + module_state_traverse = globalstate['module_state_traverse'] + for module in modules: + definition = module is env + for entry in env.c_class_entries: + if definition or entry.defined_in_pxd: + module_state.putln("PyObject *%s;" % entry.type.typeobj_cname) + module_state_defines.putln("#define %s %s->%s" % ( + entry.type.typeobj_cname, + Naming.modulestateglobal_cname, + entry.type.typeobj_cname)) + module_state_clear.putln("Py_CLEAR(clear_module_state->%s);" % + entry.type.typeobj_cname) + module_state_traverse.putln( + "Py_VISIT(traverse_module_state->%s);" % + entry.type.typeobj_cname) + module_state.putln('} %s;' % Naming.modulestate_cname) + module_state.putln('') + module_state.putln('#ifdef __cplusplus') + module_state.putln('namespace {') + module_state.putln('extern struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('} /* anonymous namespace */') + module_state.putln('#else') + module_state.putln('static struct PyModuleDef %s;' % Naming.pymoduledef_cname) + module_state.putln('#endif') + module_state.putln('') + module_state.putln('#define %s(o) ((%s *)__Pyx_PyModule_GetState(o))' % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + module_state.putln('') + module_state.putln('#define %s (%s(PyState_FindModule(&%s)))' % ( + Naming.modulestateglobal_cname, + Naming.modulestate_cname, + Naming.pymoduledef_cname)) + module_state.putln('') + module_state.putln('#define %s (PyState_FindModule(&%s))' % ( + env.module_cname, + Naming.pymoduledef_cname)) + module_state.putln("#endif") + module_state_defines.putln("#endif") + module_state_clear.putln("return 0;") + module_state_clear.putln("};") + module_state_clear.putln("#endif") + module_state_traverse.putln("return 0;") + module_state_traverse.putln("};") + module_state_traverse.putln("#endif") + + def generate_module_state_defines(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln('#define %s %s->%s' % ( + Naming.builtins_cname, + Naming.modulestateglobal_cname, + Naming.builtins_cname)) + code.putln('#define %s %s->%s' % ( + Naming.cython_runtime_cname, + Naming.modulestateglobal_cname, + Naming.cython_runtime_cname)) + code.putln('#define %s %s->%s' % ( + Naming.empty_tuple, + Naming.modulestateglobal_cname, + Naming.empty_tuple)) + code.putln('#define %s %s->%s' % ( + Naming.empty_bytes, + Naming.modulestateglobal_cname, + Naming.empty_bytes)) + code.putln('#define %s %s->%s' % ( + Naming.empty_unicode, + Naming.modulestateglobal_cname, + Naming.empty_unicode)) + if Options.pre_import is not None: + code.putln('#define %s %s->%s' % ( + Naming.preimport_cname, + Naming.modulestateglobal_cname, + Naming.preimport_cname)) + code.putln('#define %s %s->%s' % ( + Naming.lineno_cname, + Naming.modulestateglobal_cname, + Naming.lineno_cname)) + code.putln('#define %s %s->%s' % ( + Naming.clineno_cname, + Naming.modulestateglobal_cname, + Naming.clineno_cname)) + code.putln('#define %s %s->%s' % ( + Naming.filename_cname, + Naming.modulestateglobal_cname, + Naming.filename_cname)) + + def generate_module_state_clear(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("static int %s_clear(PyObject *m) {" % Naming.module_cname) + code.putln("%s *clear_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!clear_module_state) return 0;") + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_CLEAR(clear_module_state->%s);' % + Naming.empty_unicode) + + def generate_module_state_traverse(self, env, code): + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("static int %s_traverse(PyObject *m, visitproc visit, void *arg) {" % Naming.module_cname) + code.putln("%s *traverse_module_state = %s(m);" % ( + Naming.modulestate_cname, + Naming.modulestate_cname)) + code.putln("if (!traverse_module_state) return 0;") + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.builtins_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.cython_runtime_cname) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_tuple) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_bytes) + code.putln('Py_VISIT(traverse_module_state->%s);' % + Naming.empty_unicode) + def generate_module_init_func(self, imported_modules, env, code): subfunction = self.mod_init_subfunction(self.scope, code) @@ -2425,6 +2662,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): )) code.putln("#endif") + code.putln("/*--- Module creation code ---*/") + self.generate_module_creation_code(env, code) + if profile or linetrace: tempdecl_code.put_trace_declarations() code.put_trace_frame_init() @@ -2462,9 +2702,6 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln("#endif") - code.putln("/*--- Module creation code ---*/") - self.generate_module_creation_code(env, code) - code.putln("/*--- Initialize various global constants etc. ---*/") code.put_error_if_neg(self.pos, "__Pyx_InitGlobals()") @@ -2549,7 +2786,9 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): for cname, type in code.funcstate.all_managed_temps(): code.put_xdecref(cname, type) code.putln('if (%s) {' % env.module_cname) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln('if (%s) {' % env.module_dict_cname) + code.putln("#endif") code.put_add_traceback(EncodedString("init %s" % env.qualified_name)) code.globalstate.use_utility_code(Nodes.traceback_utility_code) # Module reference and module dict are in global variables which might still be needed @@ -2557,8 +2796,10 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): # At least clearing the module dict here might be a good idea, but could still break # user code in atexit or other global registries. ##code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln('}') code.put_decref_clear(env.module_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#endif") code.putln('} else if (!PyErr_Occurred()) {') code.putln('PyErr_SetString(PyExc_ImportError, "init %s");' % env.qualified_name.as_c_string_literal()[1:-1]) @@ -2778,8 +3019,11 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): if Options.pre_import is not None: code.put_decref_clear(Naming.preimport_cname, py_object_type, nanny=False, clear_before_decref=True) - for cname in [env.module_dict_cname, Naming.cython_runtime_cname, Naming.builtins_cname]: + for cname in [Naming.cython_runtime_cname, Naming.builtins_cname]: code.put_decref_clear(cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") + code.put_decref_clear(env.module_dict_cname, py_object_type, nanny=False, clear_before_decref=True) + code.putln("#endif") def generate_main_method(self, env, code): module_is_main = self.is_main_module_flag_cname() @@ -2837,12 +3081,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#endif") code.putln("") + code.putln('#ifdef __cplusplus') + code.putln('namespace {') + code.putln("struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#else') code.putln("static struct PyModuleDef %s = {" % Naming.pymoduledef_cname) + code.putln('#endif') code.putln(" PyModuleDef_HEAD_INIT,") code.putln(' %s,' % env.module_name.as_c_string_literal()) code.putln(" %s, /* m_doc */" % doc) code.putln("#if CYTHON_PEP489_MULTI_PHASE_INIT") code.putln(" 0, /* m_size */") + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln(" sizeof(%s), /* m_size */" % Naming.modulestate_cname) code.putln("#else") code.putln(" -1, /* m_size */") code.putln("#endif") @@ -2852,10 +3103,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("#else") code.putln(" NULL, /* m_reload */") code.putln("#endif") + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln(" %s_traverse, /* m_traverse */" % Naming.module_cname) + code.putln(" %s_clear, /* m_clear */" % Naming.module_cname) + code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#else") code.putln(" NULL, /* m_traverse */") code.putln(" NULL, /* m_clear */") code.putln(" %s /* m_free */" % cleanup_func) + code.putln("#endif") code.putln("};") + code.putln('#ifdef __cplusplus') + code.putln('} /* anonymous namespace */') + code.putln('#endif') code.putln("#endif") def generate_module_creation_code(self, env, code): @@ -2880,7 +3140,19 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): env.method_table_cname, doc, env.module_cname)) - code.putln("#else") + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + module_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True) + code.putln( + "%s = PyModule_Create(&%s); %s" % ( + module_temp, + Naming.pymoduledef_cname, + code.error_goto_if_null(module_temp, self.pos))) + code.put_gotref(module_temp) + code.putln(code.error_goto_if_neg("PyState_AddModule(%s, &%s)" % ( + module_temp, Naming.pymoduledef_cname), self.pos)) + code.put_decref_clear(module_temp, type=py_object_type) + code.funcstate.release_temp(module_temp) + code.putln('#else') code.putln( "%s = PyModule_Create(&%s);" % ( env.module_cname, @@ -2889,11 +3161,13 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln(code.error_goto_if_null(env.module_cname, self.pos)) code.putln("#endif") # CYTHON_PEP489_MULTI_PHASE_INIT + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln( "%s = PyModule_GetDict(%s); %s" % ( env.module_dict_cname, env.module_cname, code.error_goto_if_null(env.module_dict_cname, self.pos))) code.put_incref(env.module_dict_cname, py_object_type, nanny=False) + code.putln("#endif") code.putln( '%s = PyImport_AddModule(__Pyx_BUILTIN_MODULE_NAME); %s' % ( @@ -3148,6 +3422,8 @@ class ModuleNode(Nodes.Node, Nodes.BlockNode): code.putln("") # start in new line code.putln("#if defined(PYPY_VERSION_NUM) && PYPY_VERSION_NUM < 0x050B0000") code.putln('sizeof(%s),' % objstruct) + code.putln("#elif CYTHON_COMPILING_IN_LIMITED_API") + code.putln('sizeof(%s),' % objstruct) code.putln("#else") code.putln('sizeof(%s),' % sizeof_objstruct) code.putln("#endif") diff --git a/Cython/Compiler/Naming.py b/Cython/Compiler/Naming.py index 304ef9188..f1902a77d 100644 --- a/Cython/Compiler/Naming.py +++ b/Cython/Compiler/Naming.py @@ -96,6 +96,8 @@ clineno_cname = pyrex_prefix + "clineno" cfilenm_cname = pyrex_prefix + "cfilenm" local_tstate_cname = pyrex_prefix + "tstate" module_cname = pyrex_prefix + "m" +modulestate_cname = pyrex_prefix + "mstate" +modulestateglobal_cname = pyrex_prefix + "mstate_global" moddoc_cname = pyrex_prefix + "mdoc" methtable_cname = pyrex_prefix + "methods" retval_cname = pyrex_prefix + "r" diff --git a/Cython/Compiler/Nodes.py b/Cython/Compiler/Nodes.py index b3525584d..da568310a 100644 --- a/Cython/Compiler/Nodes.py +++ b/Cython/Compiler/Nodes.py @@ -3715,10 +3715,17 @@ class DefNodeWrapper(FuncDefNode): code.putln('{') all_args = tuple(positional_args) + tuple(kw_only_args) non_posonly_args = [arg for arg in all_args if not arg.pos_only] + non_pos_args_id = ','.join( + ['&%s' % code.intern_identifier(arg.name) for arg in non_posonly_args] + ['0']) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln("PyObject **%s[] = {%s};" % ( + Naming.pykwdlist_cname, + non_pos_args_id)) + code.putln("#else") code.putln("static PyObject **%s[] = {%s};" % ( Naming.pykwdlist_cname, - ','.join(['&%s' % code.intern_identifier(arg.name) - for arg in non_posonly_args] + ['0']))) + non_pos_args_id)) + code.putln("#endif") # Before being converted and assigned to the target variables, # borrowed references to all unpacked argument values are @@ -5070,6 +5077,13 @@ class CClassDefNode(ClassDefNode): if not scope: # could be None if there was an error return if entry.visibility != 'extern': + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + "%s = PyType_FromSpec(&%s_spec); %s" % ( + typeobj_cname, + typeobj_cname, + code.error_goto_if_null(typeobj_cname, entry.pos))) + code.putln("#else") for slot in TypeSlots.slot_table: slot.generate_dynamic_init_code(scope, code) if heap_type_bases: @@ -5108,6 +5122,7 @@ class CClassDefNode(ClassDefNode): code.putln("%s.tp_getattro = %s;" % ( typeobj_cname, py_cfunc)) code.putln("}") + code.putln("#endif") # Fix special method docstrings. This is a bit of a hack, but # unless we let PyType_Ready create the slot wrappers we have @@ -5145,11 +5160,19 @@ class CClassDefNode(ClassDefNode): if type.vtable_cname: code.globalstate.use_utility_code( UtilityCode.load_cached('SetVTable', 'ImportExport.c')) + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + "if (__Pyx_SetVtable(%s, %s) < 0) %s" % ( + typeobj_cname, + type.vtabptr_cname, + code.error_goto(entry.pos))) + code.putln("#else") code.putln( "if (__Pyx_SetVtable(%s.tp_dict, %s) < 0) %s" % ( typeobj_cname, type.vtabptr_cname, code.error_goto(entry.pos))) + code.putln("#endif") if heap_type_bases: code.globalstate.use_utility_code( UtilityCode.load_cached('MergeVTables', 'ImportExport.c')) @@ -5160,12 +5183,21 @@ class CClassDefNode(ClassDefNode): # scope.is_internal is set for types defined by # Cython (such as closures), the 'internal' # directive is set by users + code.putln("#if CYTHON_COMPILING_IN_LIMITED_API") + code.putln( + 'if (PyObject_SetAttr(%s, %s, %s) < 0) %s' % ( + Naming.module_cname, + code.intern_identifier(scope.class_name), + typeobj_cname, + code.error_goto(entry.pos))) + code.putln("#else") code.putln( 'if (PyObject_SetAttr(%s, %s, (PyObject *)&%s) < 0) %s' % ( Naming.module_cname, code.intern_identifier(scope.class_name), typeobj_cname, code.error_goto(entry.pos))) + code.putln("#endif") weakref_entry = scope.lookup_here("__weakref__") if not scope.is_closure_class_scope else None if weakref_entry: if weakref_entry.type is py_object_type: @@ -5187,15 +5219,19 @@ class CClassDefNode(ClassDefNode): # do so at runtime. code.globalstate.use_utility_code( UtilityCode.load_cached('SetupReduce', 'ExtensionTypes.c')) + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln('if (__Pyx_setup_reduce((PyObject*)&%s) < 0) %s' % ( typeobj_cname, code.error_goto(entry.pos))) + code.putln("#endif") # Generate code to initialise the typeptr of an extension # type defined in this module to point to its type object. if type.typeobj_cname: + code.putln("#if !CYTHON_COMPILING_IN_LIMITED_API") code.putln( "%s = &%s;" % ( type.typeptr_cname, type.typeobj_cname)) + code.putln("#endif") def annotate(self, code): if self.type_init_args: diff --git a/Cython/Compiler/PyrexTypes.py b/Cython/Compiler/PyrexTypes.py index 77935c926..8381dccff 100644 --- a/Cython/Compiler/PyrexTypes.py +++ b/Cython/Compiler/PyrexTypes.py @@ -1310,7 +1310,7 @@ class BuiltinObjectType(PyObjectType): name = '"%s"' % self.name # avoid wasting too much space but limit number of different format strings space_for_name = (len(self.name) // 16 + 1) * 16 - error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, Py_TYPE(%s)->tp_name), 0)' % ( + error = '(PyErr_Format(PyExc_TypeError, "Expected %%.%ds, got %%.200s", %s, __Pyx_PyType_Name(Py_TYPE(%s))), 0)' % ( space_for_name, name, arg) return check + '||' + error diff --git a/Cython/Compiler/TypeSlots.py b/Cython/Compiler/TypeSlots.py index 5a84bc89e..1e16c7572 100644 --- a/Cython/Compiler/TypeSlots.py +++ b/Cython/Compiler/TypeSlots.py @@ -242,7 +242,15 @@ class SlotDescriptor(object): guard = ("#if PY_MAJOR_VERSION >= 3") return guard - def generate(self, scope, code): + def spec_slot_value(self, scope): + if self.is_initialised_dynamically: + return None + result = self.slot_code(scope) + if result == "0": + return None + return result + + def generate(self, scope, code, spec=False): preprocessor_guard = self.preprocessor_guard_code() if preprocessor_guard: code.putln(preprocessor_guard) @@ -271,7 +279,11 @@ class SlotDescriptor(object): code.putln("#else") end_pypy_guard = True - code.putln("%s, /*%s*/" % (value, self.slot_name)) + if spec: + if value != "0": + code.putln("{Py_%s, (void *)%s}," % (self.slot_name, value)) + else: + code.putln("%s, /*%s*/" % (value, self.slot_name)) if end_pypy_guard: code.putln("#endif") @@ -555,6 +567,11 @@ class SuiteSlot(SlotDescriptor): if self.ifdef: code.putln("#endif") + def generate_substructure_spec(self, scope, code): + if not self.is_empty(scope): + for slot in self.sub_slots: + slot.generate(scope, code, spec=True) + substructures = [] # List of all SuiteSlot instances class MethodTableSlot(SlotDescriptor): diff --git a/Cython/Utility/Exceptions.c b/Cython/Utility/Exceptions.c index caff02068..6912ef485 100644 --- a/Cython/Utility/Exceptions.c +++ b/Cython/Utility/Exceptions.c @@ -284,7 +284,7 @@ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject PyErr_SetObject(type, value); if (tb) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API PyObject *tmp_type, *tmp_value, *tmp_tb; PyErr_Fetch(&tmp_type, &tmp_value, &tmp_tb); Py_INCREF(tb); @@ -705,6 +705,15 @@ static void __Pyx_AddTraceback(const char *funcname, int c_line, #include "frameobject.h" #include "traceback.h" +#if CYTHON_COMPILING_IN_LIMITED_API +static void __Pyx_AddTraceback(const char *funcname, int c_line, + int py_line, const char *filename) { + if (c_line) { + c_line = __Pyx_CLineForTraceback(__Pyx_PyThreadState_Current, c_line); + } + _PyTraceback_Add(funcname, filename, c_line ? -c_line : py_line); +} +#else static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( const char *funcname, int c_line, int py_line, const char *filename) { @@ -791,3 +800,4 @@ bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } +#endif diff --git a/Cython/Utility/ExtensionTypes.c b/Cython/Utility/ExtensionTypes.c index ca2adbe20..2dc3f9d44 100644 --- a/Cython/Utility/ExtensionTypes.c +++ b/Cython/Utility/ExtensionTypes.c @@ -172,13 +172,16 @@ static void __Pyx_call_next_tp_clear(PyObject* obj, inquiry current_tp_clear) { /////////////// SetupReduce.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce(PyObject* type_obj); +#endif /////////////// SetupReduce /////////////// //@requires: ObjectHandling.c::PyObjectGetAttrStrNoError //@requires: ObjectHandling.c::PyObjectGetAttrStr //@substitute: naming +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_setup_reduce_is_named(PyObject* meth, PyObject* name) { int ret; PyObject *name_attr; @@ -250,7 +253,7 @@ static int __Pyx_setup_reduce(PyObject* type_obj) { BAD: if (!PyErr_Occurred()) - PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", ((PyTypeObject*)type_obj)->tp_name); + PyErr_Format(PyExc_RuntimeError, "Unable to initialize pickling for %s", __Pyx_PyType_Name(type_obj)); ret = -1; GOOD: #if !CYTHON_USE_PYTYPE_LOOKUP @@ -264,3 +267,4 @@ GOOD: Py_XDECREF(setstate_cython); return ret; } +#endif diff --git a/Cython/Utility/ImportExport.c b/Cython/Utility/ImportExport.c index 4c8877925..a350aab6d 100644 --- a/Cython/Utility/ImportExport.c +++ b/Cython/Utility/ImportExport.c @@ -182,8 +182,13 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { if (level == -1) { if (strchr(__Pyx_MODULE_NAME, '.')) { /* try package relative import first */ + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, 1); + #else module = PyImport_ImportModuleLevelObject( name, $moddict_cname, empty_dict, from_list, 1); + #endif if (unlikely(!module)) { if (unlikely(!PyErr_ExceptionMatches(PyExc_ImportError))) goto bad; @@ -202,9 +207,14 @@ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { name, $moddict_cname, empty_dict, from_list, py_level, (PyObject *)NULL); Py_DECREF(py_level); #else + #if CYTHON_COMPILING_IN_LIMITED_API + module = PyImport_ImportModuleLevelObject( + name, empty_dict, empty_dict, from_list, level); + #else module = PyImport_ImportModuleLevelObject( name, $moddict_cname, empty_dict, from_list, level); #endif + #endif } } bad: @@ -686,11 +696,19 @@ static int __Pyx_SetVtable(PyObject *dict, void *vtable); /*proto*/ /////////////// SetVTable /////////////// +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_SetVtable(PyObject *type, void *vtable) { +#else static int __Pyx_SetVtable(PyObject *dict, void *vtable) { +#endif PyObject *ob = PyCapsule_New(vtable, 0, 0); if (!ob) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + if (PyObject_SetAttr(type, PYIDENT("__pyx_vtable__"), ob) < 0) +#else if (PyDict_SetItem(dict, PYIDENT("__pyx_vtable__"), ob) < 0) +#endif goto bad; Py_DECREF(ob); return 0; diff --git a/Cython/Utility/ModuleSetupCode.c b/Cython/Utility/ModuleSetupCode.c index dc12bac3d..b0e18f4a1 100644 --- a/Cython/Utility/ModuleSetupCode.c +++ b/Cython/Utility/ModuleSetupCode.c @@ -44,6 +44,7 @@ #define CYTHON_COMPILING_IN_PYPY 1 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #undef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 0 @@ -90,6 +91,7 @@ #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 @@ -133,10 +135,53 @@ #undef CYTHON_USE_EXC_INFO_STACK #define CYTHON_USE_EXC_INFO_STACK 0 +#elif defined(CYTHON_LIMITED_API) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #define CYTHON_COMPILING_IN_LIMITED_API 1 + + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_PYTYPE_LOOKUP + #define CYTHON_USE_PYTYPE_LOOKUP 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_ASYNC_SLOTS 0 + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 1 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_GIL + #define CYTHON_FAST_GIL 0 + #undef CYTHON_METH_FASTCALL + #define CYTHON_METH_FASTCALL 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 + #undef CYTHON_PEP489_MULTI_PHASE_INIT + #define CYTHON_PEP489_MULTI_PHASE_INIT 0 + #undef CYTHON_USE_TP_FINALIZE + #define CYTHON_USE_TP_FINALIZE 1 + #undef CYTHON_USE_DICT_VERSIONS + #define CYTHON_USE_DICT_VERSIONS 0 + #undef CYTHON_USE_EXC_INFO_STACK + #define CYTHON_USE_EXC_INFO_STACK 0 + #else #define CYTHON_COMPILING_IN_PYPY 0 #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #define CYTHON_COMPILING_IN_LIMITED_API 0 #ifndef CYTHON_USE_TYPE_SLOTS #define CYTHON_USE_TYPE_SLOTS 1 @@ -495,12 +540,17 @@ class __Pyx_FakeReference { // special C-API functions only in Pyston #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#elif CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) #else #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) #endif -#if !CYTHON_FAST_THREAD_STATE +#if CYTHON_COMPILING_IN_LIMITED_API + #define __Pyx_PyThreadState_Current PyThreadState_Get() +#elif !CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_Current PyThreadState_GET() #elif PY_VERSION_HEX >= 0x03060000 //#elif PY_VERSION_HEX >= 0x03050200 @@ -512,6 +562,18 @@ class __Pyx_FakeReference { #define __Pyx_PyThreadState_Current _PyThreadState_Current #endif +#if CYTHON_COMPILING_IN_LIMITED_API +static inline void *__Pyx_PyModule_GetState(PyObject *op) +{ + void *result; + + result = PyModule_GetState(op); + if (!result) + Py_FatalError("Couldn't find the module state"); + return result; +} +#endif + // TSS (Thread Specific Storage) API #if PY_VERSION_HEX < 0x030700A2 && !defined(PyThread_tss_create) && !defined(Py_tss_NEEDS_INIT) #include "pythread.h" @@ -582,8 +644,39 @@ static CYTHON_INLINE PyObject * __Pyx_PyDict_GetItemStrWithError(PyObject *dict, #define __Pyx_PyDict_GetItemStr PyDict_GetItem #endif -/* new Py3.3 unicode type (PEP 393) */ -#if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) +/* Type slots */ +#if CYTHON_COMPILING_IN_LIMITED_API + #if defined(_PyType_Name) + #define __Pyx_PyType_Name(tp) (_PyType_Name((PyTypeObject *)tp)) + #else + #define __Pyx_PyType_Name(tp) (((PyTypeObject *)tp)->tp_name) + #endif + #define __Pyx_PyType_GetFlags(tp) (PyType_GetFlags((PyTypeObject *)tp)) +#else + #define __Pyx_PyType_Name(tp) (((PyTypeObject *)tp)->tp_name) + #define __Pyx_PyType_GetFlags(tp) (((PyTypeObject *)tp)->tp_flags) +#endif + +#if CYTHON_COMPILING_IN_LIMITED_API + #if !defined(PyUnicode_GET_SIZE) + #define PyUnicode_GET_SIZE(u) PyUnicode_GetSize(u) + #endif + #define CYTHON_PEP393_ENABLED 1 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 + #define __Pyx_PyUnicode_READY(op) (0) + #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GetSize(u) + #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AsUnicode(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(wchar_t) == 2) ? 65535 : 1114111) + #define __Pyx_PyUnicode_KIND(u) (sizeof(wchar_t)) + #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AsUnicode(u)) + /* (void)(k) => avoid unused variable warning due to macro: */ + #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((wchar_t*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((wchar_t*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GetSize(u)) +#elif PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) + /* new Py3.3 unicode type (PEP 393) */ #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ? \ 0 : _PyUnicode_Ready((PyObject *)(op))) @@ -987,12 +1080,20 @@ static CYTHON_SMALL_CODE int __Pyx_check_single_interpreter(void) { return 0; } +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *module, const char* from_name, const char* to_name, int allow_none) { +#else static CYTHON_SMALL_CODE int __Pyx_copy_spec_to_module(PyObject *spec, PyObject *moddict, const char* from_name, const char* to_name, int allow_none) { +#endif PyObject *value = PyObject_GetAttrString(spec, from_name); int result = 0; if (likely(value)) { if (allow_none || value != Py_None) { +#if CYTHON_COMPILING_IN_LIMITED_API + result = PyModule_AddObject(module, to_name, value); +#else result = PyDict_SetItemString(moddict, to_name, value); +#endif } Py_DECREF(value); } else if (PyErr_ExceptionMatches(PyExc_AttributeError)) { @@ -1019,9 +1120,13 @@ static CYTHON_SMALL_CODE PyObject* ${pymodule_create_func_cname}(PyObject *spec, Py_DECREF(modname); if (unlikely(!module)) goto bad; +#if CYTHON_COMPILING_IN_LIMITED_API + moddict = module; +#else moddict = PyModule_GetDict(module); if (unlikely(!moddict)) goto bad; // moddict is a borrowed reference +#endif if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "loader", "__loader__", 1) < 0)) goto bad; if (unlikely(__Pyx_copy_spec_to_module(spec, moddict, "origin", "__file__", 1) < 0)) goto bad; @@ -1038,6 +1143,7 @@ bad: /////////////// CodeObjectCache.proto /////////////// +#if !CYTHON_COMPILING_IN_LIMITED_API typedef struct { PyCodeObject* code_object; int code_line; @@ -1054,11 +1160,13 @@ static struct __Pyx_CodeObjectCache __pyx_code_cache = {0,0,NULL}; static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line); static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +#endif /////////////// CodeObjectCache /////////////// // Note that errors are simply ignored in the code below. // This is just a cache, if a lookup or insertion fails - so what? +#if !CYTHON_COMPILING_IN_LIMITED_API static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { @@ -1139,9 +1247,11 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { __pyx_code_cache.count++; Py_INCREF(code_object); } +#endif /////////////// CodeObjectCache.cleanup /////////////// + #if !CYTHON_COMPILING_IN_LIMITED_API if (__pyx_code_cache.entries) { __Pyx_CodeObjectCacheEntry* entries = __pyx_code_cache.entries; int i, count = __pyx_code_cache.count; @@ -1153,6 +1263,7 @@ static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object) { } PyMem_Free(entries); } + #endif /////////////// CheckBinaryVersion.proto /////////////// diff --git a/Cython/Utility/ObjectHandling.c b/Cython/Utility/ObjectHandling.c index bb062f6c4..5f63b24ab 100644 --- a/Cython/Utility/ObjectHandling.c +++ b/Cython/Utility/ObjectHandling.c @@ -1289,6 +1289,14 @@ static CYTHON_INLINE PyObject *__Pyx__GetModuleGlobalName(PyObject *name) } else if (unlikely(PyErr_Occurred())) { return NULL; } +#elif CYTHON_COMPILING_IN_LIMITED_API + if (unlikely(!$module_cname)) { + return NULL; + } + result = PyObject_GetItem($module_cname, name); + if (likely(result)) { + return result; + } #else result = PyDict_GetItem($moddict_cname, name); __PYX_UPDATE_DICT_CACHE($moddict_cname, result, *dict_cached_value, *dict_version) diff --git a/Cython/Utility/Optimize.c b/Cython/Utility/Optimize.c index 1242d528b..e1d3ec085 100644 --- a/Cython/Utility/Optimize.c +++ b/Cython/Utility/Optimize.c @@ -776,7 +776,11 @@ static CYTHON_INLINE {{c_ret_type}} __Pyx_PyInt_{{'' if ret_type.is_pyobject els if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{return_compare('(double)a', '(double)b', c_op)}} } @@ -1064,7 +1068,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, CYTHON_UNUSED {{if c_op in '+-*' or op in ('TrueDivide', 'Eq', 'Ne')}} if (PyFloat_CheckExact({{pyval}})) { const long {{'a' if order == 'CObj' else 'b'}} = intval; +#if CYTHON_COMPILING_IN_LIMITED_API + double {{ival}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else double {{ival}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{if op in ('Eq', 'Ne')}} if ((double)a {{c_op}} (double)b) { {{return_true}}; @@ -1143,7 +1151,11 @@ static {{c_ret_type}} {{cfunc_name}}(PyObject *op1, PyObject *op2, double floatv {{endif}} if (likely(PyFloat_CheckExact({{pyval}}))) { +#if CYTHON_COMPILING_IN_LIMITED_API + {{fval}} = __pyx_PyFloat_AsDouble({{pyval}}); +#else {{fval}} = PyFloat_AS_DOUBLE({{pyval}}); +#endif {{zerodiv_check(fval)}} } else diff --git a/Cython/Utility/StringTools.c b/Cython/Utility/StringTools.c index 040ad2fb1..ed56c7006 100644 --- a/Cython/Utility/StringTools.c +++ b/Cython/Utility/StringTools.c @@ -43,13 +43,42 @@ static CYTHON_INLINE Py_ssize_t __Pyx_Py_UNICODE_ssize_strlen(const Py_UNICODE * //////////////////// InitStrings.proto //////////////////// +#if CYTHON_COMPILING_IN_LIMITED_API +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str); /*proto*/ +#else static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); /*proto*/ +#endif //////////////////// InitStrings //////////////////// +#if PY_MAJOR_VERSION >= 3 +static int __Pyx_InitString(__Pyx_StringTabEntry t, PyObject **str) { + if (t.is_unicode | t.is_str) { + if (t.intern) { + *str = PyUnicode_InternFromString(t.s); + } else if (t.encoding) { + *str = PyUnicode_Decode(t.s, t.n - 1, t.encoding, NULL); + } else { + *str = PyUnicode_FromStringAndSize(t.s, t.n - 1); + } + } else { + *str = PyBytes_FromStringAndSize(t.s, t.n - 1); + } + if (!*str) + return -1; + // initialise cached hash value + if (PyObject_Hash(*str) == -1) + return -1; + return 0; +} +#endif + +#if !CYTHON_COMPILING_IN_LIMITED_API static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { - #if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION >= 3 /* Python 3+ has unicode identifiers */ + __Pyx_InitString(*t, t->p); + #else if (t->is_unicode) { *t->p = PyUnicode_DecodeUTF8(t->s, t->n - 1, NULL); } else if (t->intern) { @@ -57,28 +86,17 @@ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { } else { *t->p = PyString_FromStringAndSize(t->s, t->n - 1); } - #else /* Python 3+ has unicode identifiers */ - if (t->is_unicode | t->is_str) { - if (t->intern) { - *t->p = PyUnicode_InternFromString(t->s); - } else if (t->encoding) { - *t->p = PyUnicode_Decode(t->s, t->n - 1, t->encoding, NULL); - } else { - *t->p = PyUnicode_FromStringAndSize(t->s, t->n - 1); - } - } else { - *t->p = PyBytes_FromStringAndSize(t->s, t->n - 1); - } - #endif if (!*t->p) return -1; // initialise cached hash value if (PyObject_Hash(*t->p) == -1) return -1; + #endif ++t; } return 0; } +#endif //////////////////// BytesContains.proto //////////////////// @@ -189,7 +207,7 @@ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int //@requires: BytesEquals static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else #if PY_MAJOR_VERSION < 3 @@ -300,7 +318,7 @@ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int eq //@requires: IncludeStringH static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { -#if CYTHON_COMPILING_IN_PYPY +#if CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API return PyObject_RichCompareBool(s1, s2, equals); #else if (s1 == s2) { diff --git a/Cython/Utility/TypeConversion.c b/Cython/Utility/TypeConversion.c index 166ff44cb..0a95d2f3c 100644 --- a/Cython/Utility/TypeConversion.c +++ b/Cython/Utility/TypeConversion.c @@ -80,11 +80,21 @@ static CYTHON_INLINE PyObject* __Pyx_PyUnicode_FromString(const char*); #define __Pyx_PyUnicode_FromCString(s) __Pyx_PyUnicode_FromString((const char*)s) // There used to be a Py_UNICODE_strlen() in CPython 3.x, but it is deprecated since Py3.3. -static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) { +#if CYTHON_COMPILING_IN_LIMITED_API +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const wchar_t *u) +{ + const wchar_t *u_end = u; + while (*u_end++) ; + return (size_t)(u_end - u - 1); +} +#else +static CYTHON_INLINE size_t __Pyx_Py_UNICODE_strlen(const Py_UNICODE *u) +{ const Py_UNICODE *u_end = u; while (*u_end++) ; return (size_t)(u_end - u - 1); } +#endif #define __Pyx_PyUnicode_FromUnicode(u) PyUnicode_FromUnicode(u, __Pyx_Py_UNICODE_strlen(u)) #define __Pyx_PyUnicode_FromUnicodeAndLength PyUnicode_FromUnicode @@ -271,7 +281,7 @@ static CYTHON_INLINE const char* __Pyx_PyObject_AsStringAndSize(PyObject* o, Py_ } else #endif /* __PYX_DEFAULT_STRING_ENCODING_IS_ASCII || __PYX_DEFAULT_STRING_ENCODING_IS_DEFAULT */ -#if (!CYTHON_COMPILING_IN_PYPY) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) +#if (!CYTHON_COMPILING_IN_PYPY && !CYTHON_COMPILING_IN_LIMITED_API) || (defined(PyByteArray_AS_STRING) && defined(PyByteArray_GET_SIZE)) if (PyByteArray_Check(o)) { *length = PyByteArray_GET_SIZE(o); return PyByteArray_AS_STRING(o); @@ -311,7 +321,7 @@ static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const "__int__ returned non-int (type %.200s). " "The ability to return an instance of a strict subclass of int " "is deprecated, and may be removed in a future version of Python.", - Py_TYPE(result)->tp_name)) { + __Pyx_PyType_Name(Py_TYPE(result)))) { Py_DECREF(result); return NULL; } @@ -320,7 +330,7 @@ static PyObject* __Pyx_PyNumber_IntOrLongWrongResultType(PyObject* result, const #endif PyErr_Format(PyExc_TypeError, "__%.4s__ returned non-%.4s (type %.200s)", - type_name, type_name, Py_TYPE(result)->tp_name); + type_name, type_name, __Pyx_PyType_Name(Py_TYPE(result))); Py_DECREF(result); return NULL; } @@ -947,9 +957,9 @@ static CYTHON_INLINE {{TYPE}} {{FROM_PY_FUNCTION}}(PyObject *x) { } } { -#if CYTHON_COMPILING_IN_PYPY && !defined(_PyLong_AsByteArray) +#if (CYTHON_COMPILING_IN_PYPY || CYTHON_COMPILING_IN_LIMITED_API) && !defined(_PyLong_AsByteArray) PyErr_SetString(PyExc_RuntimeError, - "_PyLong_AsByteArray() not available in PyPy, cannot convert large numbers"); + "_PyLong_AsByteArray() not available, cannot convert large numbers"); #else {{TYPE}} val; PyObject *v = __Pyx_PyNumber_IntOrLong(x); diff --git a/runtests.py b/runtests.py index ff78cb80b..66c834f43 100755 --- a/runtests.py +++ b/runtests.py @@ -2149,6 +2149,8 @@ def main(): help="specify Pythran include directory. This will run the C++ tests using Pythran backend for Numpy") parser.add_option("--no-capture", dest="capture", default=True, action="store_false", help="do not capture stdout, stderr in srctree tests. Makes pdb.set_trace interactive") + parser.add_option("--limited-api", dest="limited_api", default=False, action="store_true", + help="Compiles Cython using CPython's LIMITED_API") options, cmd_args = parser.parse_args(args) @@ -2368,6 +2370,10 @@ def runtests(options, cmd_args, coverage=None): sys.path.insert(0, os.path.split(libpath)[0]) CFLAGS.append("-DCYTHON_REFNANNY=1") + if options.limited_api: + CFLAGS.append("-DCYTHON_LIMITED_API=1") + + if xml_output_dir and options.fork: # doesn't currently work together sys.stderr.write("Disabling forked testing to support XML test output\n") @@ -2426,6 +2432,7 @@ def runtests(options, cmd_args, coverage=None): bug_files = [ ('bugs.txt', True), ('pypy_bugs.txt', IS_PYPY), + ('limited_api_bugs.txt', options.limited_api), ('windows_bugs.txt', sys.platform == 'win32'), ('cygwin_bugs.txt', sys.platform == 'cygwin') ] diff --git a/tests/limited_api_bugs.txt b/tests/limited_api_bugs.txt new file mode 100644 index 000000000..dd779ac6d --- /dev/null +++ b/tests/limited_api_bugs.txt @@ -0,0 +1,25 @@ +# This file contains tests corresponding to unresolved bugs using CPython's +# Limited API which will be skipped in the normal testing run. + +calc_pi_2 +calc_pi_3 +calc_pi_4 +clone +convolve2 +cpdef_sin +dict_animal +integrate +landscaping +mymodule +not_none +overhead +profile +profile_2 +queue3 +resize +safe_usage +shrubbery_2 +sin_of_square +test_queue +unsafe_usage +wave_function |